@arcteninc/core 0.0.176 → 0.0.177

package/scripts/tree-sitter-discover.ts

@@ -1,526 +0,0 @@
-#!/usr/bin/env node
-/**
- * Fast function discovery using tree-sitter
- * Scans TypeScript/TSX files for ALL functions (exported + private)
- * 5-10x faster than TypeScript compiler for initial discovery
- *
- * Key: Discovers both exported and non-exported functions
- * - isExported: true = can be imported from anywhere
- * - isExported: false = only accessible within the file
- */
-
-import * as fs from 'fs';
-import * as path from 'path';
-import { glob } from 'glob';
-
-// tree-sitter types
-interface TreeSitterParser {
-  parse(input: string): SyntaxTree;
-  setLanguage(language: Language): void;
-}
-
-interface Language {}
-
-interface SyntaxTree {
-  rootNode: SyntaxNode;
-}
-
-interface SyntaxNode {
-  type: string;
-  text: string;
-  children: SyntaxNode[];
-  namedChildren: SyntaxNode[];
-  childForFieldName(name: string): SyntaxNode | null;
-  descendantsOfType(type: string | string[]): SyntaxNode[];
-  firstChild: SyntaxNode | null;
-  nextSibling: SyntaxNode | null;
-  startPosition: { row: number; column: number };
-  endPosition: { row: number; column: number };
-}
-
-export interface DiscoveredFunction {
-  name: string;
-  kind: 'function' | 'arrow' | 'variable' | 'class';
-  file: string;
-  line: number;
-  hasParameters: boolean;
-  isAsync: boolean;
-  isExported: boolean; // true = can be imported, false = private to file
-  jsDocComment?: string;
-}
-
-// Backwards compatibility alias
-export type DiscoveredExport = DiscoveredFunction;
-
-let Parser: any = null;
-let TypeScriptLanguage: Language | null = null;
-let TSXLanguage: Language | null = null;
-
-async function initTreeSitter(): Promise<boolean> {
-  if (Parser) return true;
-
-  // Use web-tree-sitter (WASM) - most compatible across Node/Bun/Windows
-  try {
-    const TreeSitterModule = await import('web-tree-sitter');
-    const TreeSitter = TreeSitterModule.default || TreeSitterModule;
-
-    // In web-tree-sitter 0.25+, init is a static method on Parser class
-    const ParserClass = TreeSitter.Parser || TreeSitter;
-
-    if (typeof ParserClass.init === 'function') {
-      await ParserClass.init();
-    }
-
-    Parser = ParserClass;
-
-    // Load TypeScript and TSX language WASM files
-    // First try local wasm files, then CDN
-    const scriptDir = typeof __dirname !== 'undefined'
-      ? __dirname
-      : path.dirname(new URL(import.meta.url).pathname).replace(/^\/([A-Z]:)/, '$1'); // Fix Windows paths
-
-    const wasmDir = path.join(scriptDir, 'wasm');
-    const Language = TreeSitter.Language || Parser.Language;
-
-    const tsWasmPath = path.join(wasmDir, 'tree-sitter-typescript.wasm');
-    const tsxWasmPath = path.join(wasmDir, 'tree-sitter-tsx.wasm');
-
-    if (fs.existsSync(tsWasmPath) && fs.existsSync(tsxWasmPath)) {
-      TypeScriptLanguage = await Language.load(tsWasmPath);
-      TSXLanguage = await Language.load(tsxWasmPath);
-    } else {
-      // Download WASM files from CDN
-      console.log('⏳ Downloading tree-sitter language files...');
-
-      const tsWasmUrl = 'https://unpkg.com/tree-sitter-typescript@0.23.2/tree-sitter-typescript.wasm';
-      const tsxWasmUrl = 'https://unpkg.com/tree-sitter-typescript@0.23.2/tree-sitter-tsx.wasm';
-
-      // Create wasm directory if it doesn't exist
-      if (!fs.existsSync(wasmDir)) {
-        fs.mkdirSync(wasmDir, { recursive: true });
-      }
-
-      // Download and save WASM files
-      const tsResponse = await fetch(tsWasmUrl);
-      const tsxResponse = await fetch(tsxWasmUrl);
-
-      if (!tsResponse.ok || !tsxResponse.ok) {
-        throw new Error('Failed to download WASM files');
-      }
-
-      const tsBuffer = Buffer.from(await tsResponse.arrayBuffer());
-      const tsxBuffer = Buffer.from(await tsxResponse.arrayBuffer());
-
-      fs.writeFileSync(tsWasmPath, tsBuffer);
-      fs.writeFileSync(tsxWasmPath, tsxBuffer);
-
-      console.log('✓ Tree-sitter languages downloaded');
-
-      TypeScriptLanguage = await Language.load(tsWasmPath);
-      TSXLanguage = await Language.load(tsxWasmPath);
-    }
-
-    return true;
-  } catch (error) {
-    console.error('Failed to initialize tree-sitter:', error);
-  }
-
-  return false;
-}
-
-function getParser(isTsx: boolean): TreeSitterParser {
-  const parser = new Parser();
-  parser.setLanguage(isTsx ? TSXLanguage : TypeScriptLanguage);
-  return parser;
-}
-
-/**
- * Extract JSDoc comment from a node if present
- */
-function extractJsDoc(node: SyntaxNode): string | undefined {
-  let sibling = node.firstChild;
-  // Look for comment nodes before the declaration
-  const comments: string[] = [];
-
-  // Actually need to look at previous siblings
-  let prev = node;
-  while (prev) {
-    const parent = prev;
-    // In tree-sitter, comments are usually siblings
-    break;
-  }
-
-  // Look for comment child
-  for (const child of node.children) {
-    if (child.type === 'comment') {
-      const text = child.text;
-      if (text.startsWith('/**')) {
-        return text
-          .replace(/^\/\*\*\s*/, '')
-          .replace(/\s*\*\/$/, '')
-          .replace(/^\s*\*\s?/gm, '')
-          .trim();
-      }
-    }
-  }
-
-  return undefined;
-}
-
-/**
- * Check if a node has an export modifier
- */
-function hasExportKeyword(node: SyntaxNode): boolean {
-  for (const child of node.children) {
-    if (child.type === 'export' || child.text === 'export') {
-      return true;
-    }
-  }
-  return false;
-}
-
-/**
- * Check if a function is async
- */
-function isAsyncFunction(node: SyntaxNode): boolean {
-  for (const child of node.children) {
-    if (child.type === 'async' || child.text === 'async') {
-      return true;
-    }
-  }
-  return false;
-}
-
-/**
- * Check if a function has parameters (excluding empty params)
- */
-function hasParameters(node: SyntaxNode): boolean {
-  const params = node.childForFieldName('parameters');
-  if (!params) return false;
-
-  // Filter out just parentheses
-  const paramChildren = params.namedChildren;
-  return paramChildren.length > 0;
-}
-
-/**
- * Extract function name from various declaration patterns
- */
-function extractFunctionName(node: SyntaxNode): string | null {
-  // Direct function name
-  const nameNode = node.childForFieldName('name');
-  if (nameNode) {
-    return nameNode.text;
-  }
-
-  // For variable declarations: export const foo = ...
-  if (node.type === 'variable_declarator') {
-    const name = node.childForFieldName('name');
-    return name?.text || null;
-  }
-
-  return null;
-}
-
-/**
- * Discover ALL functions in a file using tree-sitter (exported + private)
- */
-export async function discoverFunctionsInFile(filePath: string): Promise<DiscoveredFunction[]> {
-  const initialized = await initTreeSitter();
-  if (!initialized) return [];
-
-  const content = fs.readFileSync(filePath, 'utf-8');
-  const isTsx = filePath.endsWith('.tsx');
-  const parser = getParser(isTsx);
-  const tree = parser.parse(content);
-
-  const functions: DiscoveredFunction[] = [];
-  const rootNode = tree.rootNode;
-
-  function processNode(node: SyntaxNode, isInsideExport: boolean = false) {
-    // === EXPORTED FUNCTIONS ===
-    // export function foo() {} or export const foo = () => {}
-    if (node.type === 'export_statement') {
-      const declaration = node.childForFieldName('declaration');
-      if (!declaration) {
-        // Could be export { ... } or export default - skip re-exports
-        return;
-      }
-
-      if (declaration.type === 'function_declaration') {
-        const name = extractFunctionName(declaration);
-        if (name) {
-          functions.push({
-            name,
-            kind: 'function',
-            file: filePath,
-            line: node.startPosition.row + 1,
-            hasParameters: hasParameters(declaration),
-            isAsync: isAsyncFunction(declaration),
-            isExported: true,
-            jsDocComment: extractJsDoc(node),
-          });
-        }
-      }
-
-      // export const foo = () => {} or export const foo = function() {}
-      if (declaration.type === 'lexical_declaration') {
-        for (const child of declaration.namedChildren) {
-          if (child.type === 'variable_declarator') {
-            const name = child.childForFieldName('name')?.text;
-            const value = child.childForFieldName('value');
-
-            if (name && value) {
-              const isArrow = value.type === 'arrow_function';
-              const isFunc = value.type === 'function_expression' || value.type === 'function';
-
-              if (isArrow || isFunc) {
-                functions.push({
-                  name,
-                  kind: isArrow ? 'arrow' : 'function',
-                  file: filePath,
-                  line: node.startPosition.row + 1,
-                  hasParameters: hasParameters(value),
-                  isAsync: isAsyncFunction(value),
-                  isExported: true,
-                  jsDocComment: extractJsDoc(node),
-                });
-              }
-            }
-          }
-        }
-      }
-
-      // export class Foo {}
-      if (declaration.type === 'class_declaration') {
-        const name = extractFunctionName(declaration);
-        if (name) {
-          functions.push({
-            name,
-            kind: 'class',
-            file: filePath,
-            line: node.startPosition.row + 1,
-            hasParameters: false,
-            isAsync: false,
-            isExported: true,
-            jsDocComment: extractJsDoc(node),
-          });
-        }
-      }
-
-      // Don't process children of export statements (already handled)
-      return;
-    }
-
-    // === NON-EXPORTED (PRIVATE) FUNCTIONS ===
-    // function foo() {} (without export)
-    if (node.type === 'function_declaration' && !isInsideExport) {
-      const name = extractFunctionName(node);
-      if (name) {
-        functions.push({
-          name,
-          kind: 'function',
-          file: filePath,
-          line: node.startPosition.row + 1,
-          hasParameters: hasParameters(node),
-          isAsync: isAsyncFunction(node),
-          isExported: false,
-          jsDocComment: extractJsDoc(node),
-        });
-      }
-    }
-
-    // const foo = () => {} or const foo = function() {} (without export)
-    if (node.type === 'lexical_declaration' && !isInsideExport) {
-      for (const child of node.namedChildren) {
-        if (child.type === 'variable_declarator') {
-          const name = child.childForFieldName('name')?.text;
-          const value = child.childForFieldName('value');
-
-          if (name && value) {
-            const isArrow = value.type === 'arrow_function';
-            const isFunc = value.type === 'function_expression' || value.type === 'function';
-
-            if (isArrow || isFunc) {
-              functions.push({
-                name,
-                kind: isArrow ? 'arrow' : 'function',
-                file: filePath,
-                line: node.startPosition.row + 1,
-                hasParameters: hasParameters(value),
-                isAsync: isAsyncFunction(value),
-                isExported: false,
-                jsDocComment: extractJsDoc(node),
-              });
-            }
-          }
-        }
-      }
-    }
-
-    // class Foo {} (without export)
-    if (node.type === 'class_declaration' && !isInsideExport) {
-      const name = extractFunctionName(node);
-      if (name) {
-        functions.push({
-          name,
-          kind: 'class',
-          file: filePath,
-          line: node.startPosition.row + 1,
-          hasParameters: false,
-          isAsync: false,
-          isExported: false,
-          jsDocComment: extractJsDoc(node),
-        });
-      }
-    }
-
-    // Process children (but not inside class bodies to avoid methods)
-    if (node.type !== 'class_body') {
-      for (const child of node.namedChildren) {
-        processNode(child, false);
-      }
-    }
-  }
-
-  processNode(rootNode);
-  return functions;
-}
-
-// Backwards compatibility alias
-export const discoverExportsInFile = discoverFunctionsInFile;
-
-/**
- * Discover ALL functions in a directory (exported + private)
- */
-export async function discoverFunctionsInDirectory(
-  directory: string,
-  options: {
-    ignore?: string[];
-    functionsOnly?: boolean;
-    exportedOnly?: boolean;
-  } = {}
-): Promise<DiscoveredFunction[]> {
-  const ignore = options.ignore || ['**/node_modules/**', '**/dist/**', '**/.next/**', '**/build/**'];
-
-  const pattern = path.join(directory, '**/*.{ts,tsx}').replace(/\\/g, '/');
-  const files = await glob(pattern, { ignore });
-
-  console.log(`📂 Scanning ${files.length} files with tree-sitter...`);
-
-  const startTime = Date.now();
-  const allFunctions: DiscoveredFunction[] = [];
-
-  for (const file of files) {
-    const functions = await discoverFunctionsInFile(file);
-    allFunctions.push(...functions);
-  }
-
-  const elapsed = Date.now() - startTime;
-  const exported = allFunctions.filter(f => f.isExported).length;
-  const priv = allFunctions.filter(f => !f.isExported).length;
-  console.log(`✓ Found ${allFunctions.length} functions (${exported} exported, ${priv} private) in ${elapsed}ms`);
-
-  let result = allFunctions;
-
-  // Filter to exported only if requested
-  if (options.exportedOnly) {
-    result = result.filter(f => f.isExported);
-  }
-
-  // Filter to functions only if requested
-  if (options.functionsOnly) {
-    result = result.filter(f => f.kind === 'function' || f.kind === 'arrow');
-  }
-
-  return result;
-}
-
-// Backwards compatibility alias
-export const discoverExportsInDirectory = discoverFunctionsInDirectory;
-
-/**
- * Filter exports that look like they could be tools
- * (functions with parameters, not utility helpers)
- */
-export function filterLikelyTools(exports: DiscoveredExport[]): DiscoveredExport[] {
-  return exports.filter(exp => {
-    // Must be a function or arrow function
-    if (exp.kind !== 'function' && exp.kind !== 'arrow') {
-      return false;
-    }
-
-    // Must have parameters (tools need inputs)
-    if (!exp.hasParameters) {
-      return false;
-    }
-
-    // Skip specific React/utility patterns (NOT general prefixes like create/get)
-    const name = exp.name.toLowerCase();
-
-    // Exact patterns to skip (React hooks, contexts, HOCs)
-    const skipExactPatterns = [
-      /^use[A-Z]/, // React hooks: useState, useEffect, useCallback
-      /^with[A-Z]/, // HOCs: withRouter, withAuth
-      /^create(context|store|reducer|slice|action|selector|ref|portal)$/i, // React utilities
-      /^is[A-Z][a-z]*$/, // Short type guards: isArray, isNull (but not isValidEmail)
-      /^has(own|prototype)/i, // hasOwnProperty, hasPrototype
-      /^to(string|json|array|number|boolean)$/i, // Converters
-      /^from(string|json|array|entries)$/i, // Converters
-      /^(parse|stringify)(json|int|float)?$/i, // Parsers
-      /^(assert|throw|log|debug|trace|warn|error|console)/i, // Logging/errors
-      /^(emit|dispatch|subscribe|unsubscribe)$/i, // Event system
-      /^on[A-Z]/, // Event handlers: onClick, onChange
-      /^handle[A-Z]/, // Event handlers: handleClick, handleSubmit
-      /^_/, // Private functions
-    ];
-
-    for (const pattern of skipExactPatterns) {
-      if (pattern.test(exp.name)) {
-        return false;
-      }
-    }
-
-    // Skip very short names (likely utilities)
-    if (exp.name.length < 4) {
-      return false;
-    }
-
-    return true;
-  });
-}
-
-// CLI entry point
-if (import.meta.url === `file://${process.argv[1]}` || process.argv[1]?.endsWith('tree-sitter-discover.ts')) {
-  const args = process.argv.slice(2);
-  const directory = args[0] || process.cwd();
-
-  discoverFunctionsInDirectory(directory, { functionsOnly: true })
-    .then(functions => {
-      const tools = filterLikelyTools(functions);
-
-      const exported = tools.filter(t => t.isExported);
-      const priv = tools.filter(t => !t.isExported);
-
-      console.log('\n📦 Exported functions (can be imported anywhere):');
-      if (exported.length === 0) {
-        console.log('  (none found)');
-      } else {
-        for (const tool of exported) {
-          console.log(`  ✓ ${tool.name} (${path.basename(tool.file)}:${tool.line})`);
-        }
-      }
-
-      console.log('\n🔒 Private functions (only in their file):');
-      if (priv.length === 0) {
-        console.log('  (none found)');
-      } else {
-        for (const tool of priv) {
-          console.log(`  • ${tool.name} (${path.basename(tool.file)}:${tool.line})`);
-        }
-      }
-
-      console.log(`\nTotal: ${tools.length} likely tool functions`);
-    })
-    .catch(console.error);
-}