@codeyam/codeyam-cli 0.1.0-staging.1 → 0.1.0-staging.1669d45

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/README.md

@@ -0,0 +1,294 @@
+# ScopeDataStructure Helpers
+
+This directory contains extracted helper modules from `ScopeDataStructure.ts`, the core engine for CodeYam's static analysis.
+
+## Architecture Overview
+
+```
+┌─────────────────────────────────────────────────────────────────────────┐
+│                        ScopeDataStructure                               │
+│                                                                         │
+│  Two-Phase Analysis:                                                    │
+│  ┌─────────────────────┐      ┌──────────────────────────────────────┐ │
+│  │ Phase 1: Discovery  │ ──▶  │ Phase 2: Schema Building            │ │
+│  │ (onlyEquivalencies) │      │ (captureCompleteSchema)              │ │
+│  └─────────────────────┘      └──────────────────────────────────────┘ │
+│                                                                         │
+│  Core Data Structures:                                                  │
+│  ┌──────────────┐  ┌───────────────────┐  ┌────────────────────────┐  │
+│  │ scopeNodes   │  │ equivalencyDB     │  │ externalFunctionCalls  │  │
+│  │ (per-scope)  │  │ (cross-scope)     │  │ (API tracking)         │  │
+│  └──────────────┘  └───────────────────┘  └────────────────────────┘  │
+└─────────────────────────────────────────────────────────────────────────┘
+
+Helper Modules:
+┌─────────────┐  ┌─────────────┐  ┌─────────────────┐  ┌──────────────┐
+│ PathManager │  │ UniqueId    │  │ VisitedTracker  │  │ ScopeTree    │
+│             │  │ Utils       │  │                 │  │ Manager      │
+└─────────────┘  └─────────────┘  └─────────────────┘  └──────────────┘
+```
+
+## Module Index
+
+### PathManager.ts
+
+**Purpose**: Path string manipulation with caching.
+
+**Key Functions**:
+
+- `splitPath(path)` - Split `user.profile.name` → `['user', 'profile', 'name']`
+- `joinPathParts(parts)` - Join `['user', 'id']` → `user.id`
+- `isValidPath(path)` - Validate path format
+
+**When to Use**: Whenever you need to parse or construct schema paths.
+
+---
+
+### uniqueIdUtils.ts
+
+**Purpose**: Create unique identifiers for scope/path combinations.
+
+**Key Functions**:
+
+- `uniqueId({scopeNodeName, schemaPath, value})` - Create lookup key
+- `uniqueScopeAndPaths(array)` - Deduplicate by scope+path
+
+**Key Format**: `scopeNodeName::schemaPath[::value]`
+
+**When to Use**: Building cache keys, deduplicating equivalencies.
+
+---
+
+### VisitedTracker.ts
+
+**Purpose**: Prevent infinite loops during recursive schema traversal.
+
+**Key Methods**:
+
+- `checkAndMarkGlobalVisited(scope, path, value)` - Fine-grained (includes type)
+- `checkAndMarkComplexSourceVisited(scope, path)` - Coarse (ignores type)
+- `resetGlobalVisited()` - Called between analysis phases
+
+**When to Use**: Inside recursive methods like `addToSchema()` or `followEquivalencies()`.
+
+---
+
+### ScopeTreeManager.ts
+
+**Purpose**: Track parent-child relationships between function scopes.
+
+**Key Methods**:
+
+- `addPath(['outer', 'inner'])` - Build tree as scopes are discovered
+- `findNode(name)` - Look up scope in hierarchy
+- `getTree()` - Get full tree structure
+
+**When to Use**: Understanding scope nesting, debugging scope resolution.
+
+---
+
+### selectBestValue.ts
+
+**Purpose**: Choose the most specific type when merging schemas.
+
+**Logic**: Prefers `'string'` over `'unknown'`, handles union types.
+
+**When to Use**: Merging type information from multiple sources.
+
+---
+
+### DebugTracer.ts
+
+**Purpose**: Structured debug/trace system for selective tracing without code changes.
+
+**Key Features**:
+
+- `trace(label, context)` - Simple trace point
+- `traceEnter/traceExit(label)` - Track call depth
+- Path/scope pattern filtering - Only trace what you care about
+- Max depth limiting - Prevent runaway output
+
+**Example Usage**:
+
+```typescript
+const tracer = new DebugTracer({
+  enabled: true,
+  pathPatterns: [/user\.id/], // Only trace paths matching this pattern
+  maxDepth: 10,
+});
+
+tracer.traceEnter('addToSchema', { path, scope: scopeNode.name });
+// ... do work ...
+tracer.traceExit('addToSchema', { wrote: true });
+```
+
+**When to Use**: Debugging specific paths through the analysis without modifying code.
+
+---
+
+## Key Concepts for Debugging
+
+### Equivalencies
+
+An **equivalency** connects two data paths that refer to the same data:
+
+```
+// In code:
+const { user } = props;
+return <Name name={user.name} />;
+
+// Creates equivalencies:
+props.user.name ←→ user.name ←→ <Name>.props.name
+```
+
+Equivalencies are stored in `scopeNode.equivalencies` as:
+
+```typescript
+{
+  'user.name': [
+    { scopeNodeName: 'MyComponent', schemaPath: 'signature[0].user.name', ... }
+  ]
+}
+```
+
+### Schema Paths
+
+Schema paths describe the shape of data:
+
+| Path                             | Meaning                              |
+| -------------------------------- | ------------------------------------ |
+| `signature[0]`                   | First function argument              |
+| `signature[0].user.id`           | `id` property of `user` in first arg |
+| `items[].name`                   | `name` property of array elements    |
+| `returnValue`                    | Function return value                |
+| `functionCallReturnValue(fetch)` | Return value from `fetch()` call     |
+
+### The Two Phases
+
+1. **Phase 1 (Discovery)**: `onlyEquivalencies = true`
+   - Processes all scopes
+   - Builds equivalency relationships
+   - Does NOT populate schemas
+
+2. **Phase 2 (Schema Building)**: `captureCompleteSchema()`
+   - Resets visited tracker
+   - Follows equivalencies to build complete schemas
+   - Populates `equivalencyDatabase` for source/usage tracking
+
+## ScopeDataStructure Debugging Methods
+
+ScopeDataStructure includes built-in debugging helpers. These methods are accessible on any ScopeDataStructure instance:
+
+### `explainPath(scopeName, path)`
+
+Traces a path through equivalencies and shows where data comes from:
+
+```typescript
+const explanation = sds.explainPath('MyComponent', 'user.id');
+// {
+//   path: 'user.id',
+//   scope: 'MyComponent',
+//   schemaValue: 'string',
+//   equivalencies: [{ scope: 'MyComponent', path: 'signature[0].user.id' }],
+//   source: { scope: 'fetchUser', path: 'functionCallReturnValue(fetch).data.user.id' }
+// }
+```
+
+### `traceEquivalencyChain(scopeName, path, maxDepth?)`
+
+Shows the full chain of equivalencies for a path:
+
+```typescript
+const chain = sds.traceEquivalencyChain('MyComponent', 'user.id');
+// Returns array of steps showing how the data is traced back to its source
+```
+
+### `findPaths(scopeName, pattern)`
+
+Finds all paths matching a regex pattern:
+
+```typescript
+const paths = sds.findPaths('MyComponent', /user/);
+// Returns paths containing "user" with their values and equivalency status
+```
+
+### `diagnoseMissingPath(scopeName, path)`
+
+Explains why a path might be missing from the schema:
+
+```typescript
+const diagnosis = sds.diagnoseMissingPath('MyComponent', 'user.id');
+// { exists: false, possibleReasons: [...], suggestions: [...] }
+```
+
+### `dumpState(options?)`
+
+Dumps the current analysis state for inspection:
+
+```typescript
+const state = sds.dumpState({ includeSchemas: true, scopeFilter: 'MyComp' });
+// { scopeCount, metrics, scopes: [...] }
+```
+
+---
+
+## Common Debugging Patterns
+
+### "Why is this path missing from the schema?"
+
+**Quick approach**: Use `diagnoseMissingPath()`:
+
+```typescript
+const diagnosis = sds.diagnoseMissingPath('MyComponent', 'user.id');
+console.log(diagnosis.possibleReasons);
+console.log(diagnosis.suggestions);
+```
+
+**Manual investigation**:
+
+1. Check if the path exists in equivalencies: `scopeNode.equivalencies[path]`
+2. Check if it was visited: Enable debug logging in `addToSchema()`
+3. Check if it was filtered: Look at `isValidPath()` rejections
+
+### "Why is this type wrong?"
+
+**Quick approach**: Use `traceEquivalencyChain()`:
+
+```typescript
+const chain = sds.traceEquivalencyChain('MyComponent', 'user.id');
+// Each step shows the schemaValue - find where the wrong type appears
+```
+
+**Manual investigation**:
+
+1. Find where the type is set: Search for the path in `addToSchema()` calls
+2. Check `selectBestValue()` logic: Multiple sources may be merged
+3. Check equivalency chain: Type may come from a linked path
+
+### "Infinite loop or hang"
+
+1. Check `VisitedTracker` usage: Missing cycle detection?
+2. Look for recursive equivalencies: A → B → C → A
+3. Enable metrics: `maxEquivalencyChainDepth` tracks depth
+
+## Testing
+
+Each module has tests in `__tests__/`:
+
+- `PathManager.test.ts`
+- `uniqueIdUtils.test.ts`
+- `VisitedTracker.test.ts`
+- `ScopeTreeManager.test.ts`
+- `selectBestValue.test.ts`
+
+Run with: `npx jest packages/ai/src/lib/dataStructure/helpers`
+
+## Adding New Helpers
+
+When extracting new functionality:
+
+1. **Single Responsibility**: One concept per module
+2. **Document Context**: Explain WHY, not just WHAT
+3. **Link Related Modules**: Help readers navigate
+4. **Write Tests First**: Tests are documentation
+5. **Keep Interfaces Stable**: ScopeDataStructure's public API shouldn't change

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/ScopeTreeManager.ts

@@ -0,0 +1,161 @@
+/**
+ * ScopeTreeManager
+ *
+ * Manages the hierarchical tree structure of scopes in the analysis.
+ * Tracks parent-child relationships between function scopes.
+ *
+ * ## Purpose
+ *
+ * When analyzing nested functions, we need to track which functions
+ * are defined inside other functions. This affects:
+ * - Variable scope inheritance (inner functions can access outer variables)
+ * - Equivalency propagation (data flows between parent and child scopes)
+ * - Schema merging (combining schemas from nested call hierarchies)
+ *
+ * ## Tree Structure Example
+ *
+ * For code like:
+ * ```typescript
+ * function outer(data) {
+ *   function inner(item) {
+ *     return item.name;
+ *   }
+ *   return data.map(inner);
+ * }
+ * ```
+ *
+ * The tree would be:
+ * ```
+ * outer
+ *   └── inner
+ * ```
+ *
+ * ## Key Operations
+ *
+ * - `addPath(['outer', 'inner'])` - Builds tree as scopes are discovered
+ * - `findNode('inner')` - Locates scope node for equivalency lookups
+ * - `getTree()` - Returns full tree for debugging/visualization
+ *
+ * ## Related Modules
+ *
+ * - `ScopeDataStructure.addScopeAnalysis()` - Adds paths when processing scopes
+ * - `ScopeDataStructure.getAnalysisTree()` - Exposes tree to callers
+ */
+
+export interface ScopeTreeNode {
+  name: string;
+  children: ScopeTreeNode[];
+}
+
+export const ROOT_SCOPE_NAME = 'root';
+
+export class ScopeTreeManager {
+  private tree: ScopeTreeNode;
+
+  constructor(rootName: string = ROOT_SCOPE_NAME) {
+    this.tree = { name: rootName, children: [] };
+  }
+
+  /**
+   * Gets the name of the root scope.
+   * This is frequently used as the default scope name throughout the analysis.
+   */
+  getRootName(): string {
+    return this.tree.name;
+  }
+
+  /**
+   * Sets the name of the root scope.
+   * Called during initialization when the actual root scope name is known.
+   */
+  setRootName(name: string): void {
+    this.tree.name = name;
+  }
+
+  /**
+   * Gets the entire tree structure.
+   * Used by getAnalysisTree() to return the scope hierarchy.
+   */
+  getTree(): ScopeTreeNode {
+    return this.tree;
+  }
+
+  /**
+   * Adds a scope path to the tree.
+   * Creates intermediate nodes as needed.
+   *
+   * @param pathParts - Array of scope names representing the path from root
+   *
+   * @example
+   * // For a nested function: root -> parentFunc -> childFunc
+   * addPath(['root', 'parentFunc', 'childFunc'])
+   */
+  addPath(pathParts: string[]): void {
+    let currentNode: { children: ScopeTreeNode[] } = { children: [this.tree] };
+
+    for (const pathPart of pathParts) {
+      const existingChild = currentNode.children.find(
+        (child) => child.name === pathPart,
+      );
+
+      if (existingChild) {
+        currentNode = existingChild;
+      } else {
+        const newChild: ScopeTreeNode = {
+          name: pathPart,
+          children: [],
+        };
+        currentNode.children.push(newChild);
+        currentNode = newChild;
+      }
+    }
+  }
+
+  /**
+   * Finds a node by name in the tree (breadth-first search).
+   * Returns undefined if not found.
+   */
+  findNode(name: string): ScopeTreeNode | undefined {
+    const queue: ScopeTreeNode[] = [this.tree];
+
+    while (queue.length > 0) {
+      const node = queue.shift()!;
+      if (node.name === name) {
+        return node;
+      }
+      queue.push(...node.children);
+    }
+
+    return undefined;
+  }
+
+  /**
+   * Gets all scope names in the tree (flattened).
+   */
+  getAllNames(): string[] {
+    const names: string[] = [];
+    const queue: ScopeTreeNode[] = [this.tree];
+
+    while (queue.length > 0) {
+      const node = queue.shift()!;
+      names.push(node.name);
+      queue.push(...node.children);
+    }
+
+    return names;
+  }
+
+  /**
+   * Returns the depth of the tree (for debugging/metrics).
+   */
+  getDepth(): number {
+    const getNodeDepth = (node: ScopeTreeNode): number => {
+      if (node.children.length === 0) {
+        return 1;
+      }
+      return 1 + Math.max(...node.children.map(getNodeDepth));
+    };
+
+    return getNodeDepth(this.tree);
+  }
+}

package/analyzer-template/packages/ai/src/lib/dataStructure/helpers/VisitedTracker.ts

@@ -0,0 +1,235 @@
+/**
+ * VisitedTracker
+ *
+ * Manages visited state tracking for the ScopeDataStructure analysis.
+ * Prevents infinite loops and redundant processing by tracking which
+ * scope/path/value combinations have already been processed.
+ *
+ * ## Why This Exists
+ *
+ * The schema building process recursively follows equivalencies between
+ * data paths. Without cycle detection, circular references like:
+ *   `user.profile` → `currentUser` → `user.profile`
+ * would cause infinite loops.
+ *
+ * ## Two Tracking Modes
+ *
+ * ### 1. Global Visited (scope::path::value)
+ * Fine-grained deduplication for `addToSchema()`. Tracks the complete
+ * tuple so the same path can be visited with different type values.
+ *
+ * Example: `myFunc::user.id::string` and `myFunc::user.id::number`
+ * are tracked separately, allowing both types to be recorded.
+ *
+ * ### 2. Complex Source Visited (scope::path)
+ * Coarser deduplication for `addComplexSourcePathVariables()`. Ignores
+ * the type value because complex source processing should happen once
+ * per path regardless of type.
+ *
+ * ## Lifecycle
+ *
+ * - Global visited is **reset** at the start of `captureCompleteSchema()`
+ *   to allow re-processing in the schema building phase.
+ * - Complex source visited is **never reset** during normal operation.
+ *
+ * ## Related Modules
+ *
+ * - `ScopeDataStructure.addToSchema()` - Primary consumer of global visited
+ * - `ScopeDataStructure.addComplexSourcePathVariables()` - Uses complex source visited
+ */
+
+/**
+ * Tracks visited state to prevent infinite loops during schema building.
+ */
+export class VisitedTracker {
+  /**
+   * Fine-grained visited tracking using scope::path::value.
+   * Used by addToSchema to skip already-processed combinations.
+   */
+  private globalVisited: Set<string> = new Set();
+
+  /**
+   * Coarser visited tracking using scope::path only (without value).
+   * Used by addComplexSourcePathVariables to allow each unique
+   * scope+path combo to be processed once, regardless of value.
+   */
+  private complexSourceVisited: Set<string> = new Set();
+
+  /**
+   * Creates a unique key for global visited tracking.
+   * @param scopeNodeName - Name of the scope
+   * @param path - Schema path
+   * @param value - Type value
+   */
+  private makeGlobalKey(
+    scopeNodeName: string,
+    path: string,
+    value: string,
+  ): string {
+    return `${scopeNodeName}::${path}::${value}`;
+  }
+
+  /**
+   * Creates a unique key for complex source visited tracking.
+   * @param scopeNodeName - Name of the scope
+   * @param path - Schema path
+   */
+  private makeComplexKey(scopeNodeName: string, path: string): string {
+    return `${scopeNodeName}::${path}`;
+  }
+
+  /**
+   * Checks if a scope/path/value combination has been globally visited.
+   * Does NOT mark it as visited - use markGlobalVisited for that.
+   *
+   * @returns true if already visited
+   */
+  hasGlobalVisited(
+    scopeNodeName: string,
+    path: string,
+    value: string,
+  ): boolean {
+    return this.globalVisited.has(
+      this.makeGlobalKey(scopeNodeName, path, value),
+    );
+  }
+
+  /**
+   * Marks a scope/path/value combination as globally visited.
+   */
+  markGlobalVisited(scopeNodeName: string, path: string, value: string): void {
+    this.globalVisited.add(this.makeGlobalKey(scopeNodeName, path, value));
+  }
+
+  /**
+   * Checks and marks a scope/path/value combination as globally visited.
+   * This is a convenience method that combines check and mark.
+   *
+   * @returns true if was already visited (should skip processing)
+   */
+  checkAndMarkGlobalVisited(
+    scopeNodeName: string,
+    path: string,
+    value: string,
+  ): boolean {
+    const key = this.makeGlobalKey(scopeNodeName, path, value);
+    if (this.globalVisited.has(key)) {
+      return true; // Already visited
+    }
+    this.globalVisited.add(key);
+    return false; // Not visited, now marked
+  }
+
+  /**
+   * Checks if a scope/path combination has been visited for complex sources.
+   * Does NOT mark it as visited - use markComplexSourceVisited for that.
+   *
+   * @returns true if already visited
+   */
+  hasComplexSourceVisited(scopeNodeName: string, path: string): boolean {
+    return this.complexSourceVisited.has(
+      this.makeComplexKey(scopeNodeName, path),
+    );
+  }
+
+  /**
+   * Marks a scope/path combination as visited for complex sources.
+   */
+  markComplexSourceVisited(scopeNodeName: string, path: string): void {
+    this.complexSourceVisited.add(this.makeComplexKey(scopeNodeName, path));
+  }
+
+  /**
+   * Checks and marks a scope/path combination as visited for complex sources.
+   * This is a convenience method that combines check and mark.
+   *
+   * @returns true if was already visited (should skip processing)
+   */
+  checkAndMarkComplexSourceVisited(
+    scopeNodeName: string,
+    path: string,
+  ): boolean {
+    const key = this.makeComplexKey(scopeNodeName, path);
+    if (this.complexSourceVisited.has(key)) {
+      return true; // Already visited
+    }
+    this.complexSourceVisited.add(key);
+    return false; // Not visited, now marked
+  }
+
+  /**
+   * Resets the global visited set.
+   * Called at the start of captureCompleteSchema() to allow
+   * re-processing in the second phase.
+   */
+  resetGlobalVisited(): void {
+    this.globalVisited.clear();
+  }
+
+  /**
+   * Resets the complex source visited set.
+   * Note: In the original code, this was never reset.
+   */
+  resetComplexSourceVisited(): void {
+    this.complexSourceVisited.clear();
+  }
+
+  /**
+   * Resets all visited tracking.
+   */
+  resetAll(): void {
+    this.globalVisited.clear();
+    this.complexSourceVisited.clear();
+  }
+
+  /**
+   * Returns statistics about the visited sets for debugging/monitoring.
+   */
+  getStats(): { globalVisited: number; complexSourceVisited: number } {
+    return {
+      globalVisited: this.globalVisited.size,
+      complexSourceVisited: this.complexSourceVisited.size,
+    };
+  }
+
+  /**
+   * Checks if ANY variant of a scope/path was ever visited (ignores value).
+   * Useful for debugging to see if a path was processed at all.
+   *
+   * @returns true if any variant of this scope/path was visited
+   */
+  wasVisited(scopeNodeName: string, path: string): boolean {
+    const prefix = `${scopeNodeName}::${path}::`;
+    for (const key of this.globalVisited) {
+      if (key.startsWith(prefix)) {
+        return true;
+      }
+    }
+    return this.hasComplexSourceVisited(scopeNodeName, path);
+  }
+
+  /**
+   * Gets all visited entries for a scope (for debugging).
+   * Returns an array of {path, value} objects.
+   */
+  getVisitedForScope(
+    scopeNodeName: string,
+  ): Array<{ path: string; value: string }> {
+    const prefix = `${scopeNodeName}::`;
+    const results: Array<{ path: string; value: string }> = [];
+
+    for (const key of this.globalVisited) {
+      if (key.startsWith(prefix)) {
+        const parts = key.slice(prefix.length).split('::');
+        if (parts.length >= 2) {
+          results.push({
+            path: parts.slice(0, -1).join('::'), // Everything except last part
+            value: parts[parts.length - 1], // Last part is value
+          });
+        }
+      }
+    }
+
+    return results;
+  }
+}