@flisk/analyze-tracking 0.7.2 → 0.7.4

package/src/analyze/typescript/utils/function-finder.js

@@ -0,0 +1,139 @@
+/**
+ * @fileoverview Utilities for finding function context in TypeScript AST
+ * @module analyze/typescript/utils/function-finder
+ */
+
+const ts = require('typescript');
+const { isReactHookCall } = require('./type-resolver');
+
+/**
+ * Finds the name of the function that wraps a given node
+ * @param {Object} node - The TypeScript AST node to find the wrapper for
+ * @returns {string} The function name or 'global' if not in a function
+ */
+function findWrappingFunction(node) {
+  let current = node;
+
+  while (current) {
+    const functionName = extractFunctionName(current);
+
+    if (functionName) {
+      return functionName;
+    }
+
+    current = current.parent;
+  }
+
+  return 'global';
+}
+
+/**
+ * Extracts function name from different TypeScript AST node types
+ * @param {Object} node - Current TypeScript node
+ * @returns {string|null} Function name or null if not a function context
+ */
+function extractFunctionName(node) {
+  // Function declaration
+  if (ts.isFunctionDeclaration(node)) {
+    return node.name ? node.name.escapedText : 'anonymous';
+  }
+
+  // Method declaration in class
+  if (ts.isMethodDeclaration(node)) {
+    return node.name ? node.name.escapedText : 'anonymous';
+  }
+
+  // Arrow function or function expression
+  if (ts.isArrowFunction(node) || ts.isFunctionExpression(node)) {
+    return findParentFunctionName(node) || 'anonymous';
+  }
+
+  // Constructor
+  if (ts.isConstructorDeclaration(node)) {
+    return 'constructor';
+  }
+
+  // Getter/Setter
+  if (ts.isGetAccessorDeclaration(node) || ts.isSetAccessorDeclaration(node)) {
+    return node.name ? `${ts.isGetAccessorDeclaration(node) ? 'get' : 'set'} ${node.name.escapedText}` : 'anonymous';
+  }
+
+  return null;
+}
+
+/**
+ * Finds the parent function name for arrow functions or function expressions
+ * @param {Object} node - Arrow function or function expression node
+ * @returns {string|null} Parent function name or null
+ */
+function findParentFunctionName(node) {
+  const parent = node.parent;
+
+  if (!parent) return null;
+
+  if (
+    ts.isCallExpression(parent) &&
+    ts.isIdentifier(parent.expression) &&
+    isReactHookCall(parent)
+  ) {
+    if (
+      parent.parent &&
+      ts.isVariableDeclaration(parent.parent) &&
+      parent.parent.name
+    ) {
+      return `${parent.expression.escapedText}(${parent.parent.name.escapedText})`;
+    }
+    return `${parent.expression.escapedText}()`;
+  }
+
+  // Variable declaration: const myFunc = () => {}
+  if (ts.isVariableDeclaration(parent) && parent.name) {
+    // Check if initializer is a recognized React hook call
+    if (
+      parent.initializer &&
+      ts.isCallExpression(parent.initializer) &&
+      ts.isIdentifier(parent.initializer.expression) &&
+      REACT_HOOKS.has(parent.initializer.expression.escapedText)
+    ) {
+      return `${parent.initializer.expression.escapedText}(${parent.name.escapedText})`;
+    }
+    return parent.name.escapedText;
+  }
+
+  // Property assignment: { myFunc: () => {} }
+  if (ts.isPropertyAssignment(parent) && parent.name) {
+    if (ts.isIdentifier(parent.name)) {
+      return parent.name.escapedText;
+    }
+    if (ts.isStringLiteral(parent.name)) {
+      return parent.name.text;
+    }
+  }
+
+  // Method property in object literal: { myFunc() {} }
+  if (ts.isMethodDeclaration(parent) && parent.name) {
+    return parent.name.escapedText;
+  }
+
+  // Binary expression assignment: obj.myFunc = () => {}
+  if (ts.isBinaryExpression(parent) &&
+    parent.operatorToken.kind === ts.SyntaxKind.EqualsToken) {
+    if (ts.isPropertyAccessExpression(parent.left)) {
+      return parent.left.name.escapedText;
+    }
+  }
+
+  // Call expression argument: someFunc(() => {})
+  if (ts.isCallExpression(parent)) {
+    const argIndex = parent.arguments.indexOf(node);
+    if (argIndex >= 0) {
+      return `anonymous-callback-${argIndex}`;
+    }
+  }
+
+  return null;
+}
+
+module.exports = {
+  findWrappingFunction
+};

package/src/analyze/typescript/utils/type-resolver.js

@@ -0,0 +1,208 @@
+/**
+ * @fileoverview Utilities for resolving TypeScript types and identifiers
+ * @module analyze/typescript/utils/type-resolver
+ */
+
+const ts = require('typescript');
+
+/**
+ * Resolves an identifier to its initializer node
+ * @param {Object} checker - TypeScript type checker
+ * @param {Object} identifier - Identifier node to resolve
+ * @param {Object} sourceFile - Source file containing the identifier
+ * @returns {Object|null} Initializer node or null
+ */
+function resolveIdentifierToInitializer(checker, identifier, sourceFile) {
+  try {
+    const symbol = checker.getSymbolAtLocation(identifier);
+    if (!symbol || !symbol.valueDeclaration) {
+      return null;
+    }
+    
+    const declaration = symbol.valueDeclaration;
+    
+    // Handle variable declarations
+    if (ts.isVariableDeclaration(declaration) && declaration.initializer) {
+      return declaration.initializer;
+    }
+    
+    // Handle property assignments
+    if (ts.isPropertyAssignment(declaration) && declaration.initializer) {
+      return declaration.initializer;
+    }
+    
+    // Handle parameter with default value
+    if (ts.isParameter(declaration) && declaration.initializer) {
+      return declaration.initializer;
+    }
+    
+    return null;
+  } catch (error) {
+    return null;
+  }
+}
+
+/**
+ * Gets the type string for a node
+ * @param {Object} checker - TypeScript type checker
+ * @param {Object} node - AST node
+ * @returns {string} Type string
+ */
+function getTypeOfNode(checker, node) {
+  try {
+    const type = checker.getTypeAtLocation(node);
+    return checker.typeToString(type);
+  } catch (error) {
+    return 'any';
+  }
+}
+
+/**
+ * Resolves a type string to its properties structure
+ * @param {Object} checker - TypeScript type checker
+ * @param {string} typeString - Type string to resolve
+ * @param {Set} [visitedTypes] - Set of visited types to prevent cycles
+ * @returns {Object} Resolved type structure
+ */
+function resolveTypeToProperties(checker, typeString, visitedTypes = new Set()) {
+  // Prevent infinite recursion for circular references
+  if (visitedTypes.has(typeString)) {
+    return { type: 'object' };
+  }
+  
+  // Handle primitive types
+  if (['string', 'number', 'boolean', 'any', 'unknown', 'null', 'undefined', 'void', 'never'].includes(typeString)) {
+    return { type: typeString };
+  }
+  
+  // Handle array types: T[] or Array<T>
+  const arrayMatch = typeString.match(/^(.+)\[\]$/) || typeString.match(/^Array<(.+)>$/);
+  if (arrayMatch) {
+    const elementType = arrayMatch[1].trim();
+    visitedTypes.add(typeString);
+    const elementProps = resolveTypeToProperties(checker, elementType, visitedTypes);
+    return {
+      type: 'array',
+      items: elementProps
+    };
+  }
+  
+  // Handle readonly array types: readonly T[] or ReadonlyArray<T>
+  const readonlyArrayMatch = typeString.match(/^readonly (.+)\[\]$/) || typeString.match(/^ReadonlyArray<(.+)>$/);
+  if (readonlyArrayMatch) {
+    const elementType = readonlyArrayMatch[1].trim();
+    visitedTypes.add(typeString);
+    const elementProps = resolveTypeToProperties(checker, elementType, visitedTypes);
+    return {
+      type: 'array',
+      items: elementProps
+    };
+  }
+  
+  // Handle union types - preserve them as-is
+  if (typeString.includes('|')) {
+    return { type: typeString };
+  }
+  
+  // Handle intersection types
+  if (typeString.includes('&')) {
+    // For simplicity, mark intersection types as 'object'
+    return { type: 'object' };
+  }
+  
+  // Check if it looks like a custom type/interface
+  if (isCustomType(typeString)) {
+    return {
+      type: 'object',
+      __unresolved: typeString
+    };
+  }
+  
+  // Default case - preserve the type string as-is
+  return { type: typeString };
+}
+
+/**
+ * Checks if a type string represents a custom type or interface
+ * @param {string} typeString - Type string to check
+ * @returns {boolean}
+ */
+function isCustomType(typeString) {
+  // Custom types typically start with uppercase and don't contain certain characters
+  return typeString[0] === typeString[0].toUpperCase() && 
+         !typeString.includes('<') && 
+         !typeString.includes('|') && 
+         !typeString.includes('&') &&
+         !typeString.includes('(') &&
+         !typeString.includes('[');
+}
+
+/**
+ * Gets the basic type of an array element
+ * @param {Object} checker - TypeScript type checker
+ * @param {Object} element - Array element node
+ * @returns {string} Basic type string
+ */
+function getBasicTypeOfArrayElement(checker, element) {
+  if (!element) return 'any';
+  
+  // Check for literal values first
+  if (ts.isStringLiteral(element)) {
+    return 'string';
+  } else if (ts.isNumericLiteral(element)) {
+    return 'number';
+  } else if (element.kind === ts.SyntaxKind.TrueKeyword || element.kind === ts.SyntaxKind.FalseKeyword) {
+    return 'boolean';
+  } else if (ts.isObjectLiteralExpression(element)) {
+    return 'object';
+  } else if (ts.isArrayLiteralExpression(element)) {
+    return 'array';
+  } else if (element.kind === ts.SyntaxKind.NullKeyword) {
+    return 'null';
+  } else if (element.kind === ts.SyntaxKind.UndefinedKeyword) {
+    return 'undefined';
+  }
+  
+  // For identifiers and other expressions, try to get the type
+  const typeString = getTypeOfNode(checker, element);
+  
+  // Extract basic type from TypeScript type string
+  if (typeString.startsWith('"') || typeString.startsWith("'")) {
+    return 'string'; // String literal type
+  } else if (!isNaN(Number(typeString))) {
+    return 'number'; // Numeric literal type
+  } else if (typeString === 'true' || typeString === 'false') {
+    return 'boolean'; // Boolean literal type
+  } else if (typeString.includes('[]') || typeString.startsWith('Array<')) {
+    return 'array';
+  } else if (['string', 'number', 'boolean', 'object', 'null', 'undefined'].includes(typeString)) {
+    return typeString;
+  } else if (isCustomType(typeString)) {
+    return 'object';
+  }
+  
+  return 'any';
+}
+
+/**
+ * Checks if a CallExpression is a React hook (useCallback, useState, etc)
+ * @param {Object} node - CallExpression node
+ * @param {string[]} hookNames - List of hook names to check
+ * @returns {boolean}
+ */
+function isReactHookCall(node, hookNames = ['useCallback', 'useState', 'useEffect', 'useMemo', 'useReducer']) {
+  if (!node || !node.expression) return false;
+  if (ts.isIdentifier(node.expression)) {
+    return hookNames.includes(node.expression.escapedText);
+  }
+  return false;
+}
+
+module.exports = {
+  resolveIdentifierToInitializer,
+  getTypeOfNode,
+  resolveTypeToProperties,
+  isCustomType,
+  getBasicTypeOfArrayElement,
+  isReactHookCall
+};

package/src/generateDescriptions/index.js

@@ -0,0 +1,81 @@
+/**
+ * @fileoverview AI-powered description generator for analytics events
+ * @module analyze-tracking/generateDescriptions
+ */
+
+const { createPrompt } = require('./promptUtils');
+const { createEventDescriptionSchema } = require('./schemaUtils');
+const { sendPromptToLLM } = require('./llmUtils');
+
+async function generateEventDescription(eventName, event, codebaseDir, model) {
+  const properties = event.properties || {};
+  const implementations = event.implementations || [];
+
+  // Create prompt for the LLM
+  const prompt = createPrompt(eventName, properties, implementations, codebaseDir);
+
+  // Define the output schema using Zod
+  const eventDescriptionSchema = createEventDescriptionSchema(properties);
+
+  // Send prompt to the LLM and get the structured response
+  const result = await sendPromptToLLM(prompt, eventDescriptionSchema, model);
+
+  return { eventName, descriptions: result?.descriptions || null };
+}
+
+async function generateDescriptions(events, codebaseDir, model) {
+  const eventPromises = Object.entries(events).map(([eventName, event]) =>
+    generateEventDescription(eventName, event, codebaseDir, model)
+  );
+
+  console.info(`Running ${eventPromises.length} prompts in parallel...`);
+
+  const results = await Promise.all(eventPromises);
+
+  // Process results and update the events object
+  results.forEach(({ eventName, descriptions }) => {
+    if (descriptions) {
+      const event = events[eventName];
+      event.description = descriptions.eventDescription;
+
+      // Update property descriptions recursively
+      function updatePropertyDescriptions(eventProperties, descriptionProperties) {
+        for (const propName in descriptionProperties) {
+          if (eventProperties[propName]) {
+            eventProperties[propName].description = descriptionProperties[propName].description;
+            if (eventProperties[propName].properties && descriptionProperties[propName].properties) {
+              updatePropertyDescriptions(
+                eventProperties[propName].properties,
+                descriptionProperties[propName].properties
+              );
+            }
+          }
+        }
+      }
+
+      updatePropertyDescriptions(event.properties, descriptions.properties);
+
+      // Update implementations with descriptions
+      for (let i = 0; i < descriptions.implementations.length; i++) {
+        if (event.implementations[i]) {
+          if (
+            event.implementations[i].path === descriptions.implementations[i].path &&
+            event.implementations[i].line === descriptions.implementations[i].line
+          ) {
+            event.implementations[i].description = descriptions.implementations[i].description;
+          } else {
+            console.error(`Returned implementation description does not match path or line for event: ${eventName}`);
+          }
+        }
+      }
+    } else {
+      console.error(`Failed to get description for event: ${eventName}`);
+    }
+  });
+
+  return events;
+}
+
+module.exports = {
+  generateDescriptions
+};

package/src/generateDescriptions/llmUtils.js

@@ -0,0 +1,33 @@
+/**
+ * @fileoverview LLM integration utilities for generating structured responses
+ * @module analyze-tracking/generateDescriptions/llmUtils
+ */
+
+const { PromptTemplate } = require('@langchain/core/prompts');
+
+async function sendPromptToLLM(prompt, schema, model) {
+  try {
+    const promptTemplate = new PromptTemplate({
+      template: `You are an expert at structured data extraction. Generate detailed descriptions for the following analytics event, its properties, and implementations.\n{input}`,
+      inputVariables: ['input'],
+    });
+
+    const formattedPrompt = await promptTemplate.format({
+      input: prompt,
+    });
+
+    const structuredModel = model.withStructuredOutput(schema);
+    const response = await structuredModel.invoke(formattedPrompt);
+
+    return {
+      descriptions: response,
+    };
+  } catch (error) {
+    console.error('Error during LLM response parsing:', error.message);
+    return null;
+  }
+}
+
+module.exports = {
+  sendPromptToLLM
+};

package/src/generateDescriptions/promptUtils.js

@@ -0,0 +1,62 @@
+/**
+ * @fileoverview Utilities for creating LLM prompts with code snippets
+ * @module analyze-tracking/generateDescriptions/promptUtils
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+function createPrompt(eventName, properties, implementations, codebaseDir) {
+  let prompt = `Event Name: "${eventName}"\n\n`;
+  prompt += `Properties:\n`;
+
+  function appendPropertiesToPrompt(properties, indent = '') {
+    for (const propName in properties) {
+      const prop = properties[propName];
+      prompt += `${indent}- "${propName}" (type: ${prop.type})\n`;
+      if (prop.properties) {
+        prompt += `${indent}  Sub-properties:\n`;
+        appendPropertiesToPrompt(prop.properties, indent + '    ');
+      }
+    }
+  }
+
+  appendPropertiesToPrompt(properties);
+
+  // Add implementations with code snippets
+  prompt += `\nImplementations:\n`;
+  for (const impl of implementations) {
+    const codeSnippet = getCodeSnippet(path.join(codebaseDir, impl.path), impl.line);
+    prompt += `- Path: "${impl.path}", Line: ${impl.line}, Function: "${impl.function}", Destination: "${impl.destination}"\n`;
+    prompt += `Code Snippet:\n`;
+    prompt += '```\n';
+    prompt += codeSnippet + '\n';
+    prompt += '```\n';
+  }
+
+  return prompt;
+}
+
+function getCodeSnippet(filePath, lineNumber, contextLines = 5) {
+  // Extract a code snippet from the file around the specified line
+  try {
+    const fileContent = fs.readFileSync(filePath, 'utf8');
+    const lines = fileContent.split('\n');
+    const startLine = Math.max(0, lineNumber - contextLines - 1);
+    const endLine = Math.min(lines.length, lineNumber + contextLines);
+
+    const snippetLines = lines.slice(startLine, endLine);
+    return snippetLines.join('\n');
+  } catch (e) {
+    // Only log errors if not in test mode
+    if (process.env.NODE_ENV !== 'test' && !process.env.NODE_TEST_CONTEXT) {
+      console.error(`Failed to read file ${filePath}:`, e);
+    }
+    return '';
+  }
+}
+
+module.exports = {
+  createPrompt,
+  getCodeSnippet
+};

package/src/generateDescriptions/schemaUtils.js

@@ -0,0 +1,61 @@
+/**
+ * @fileoverview Zod schema utilities for structuring event descriptions
+ * @module analyze-tracking/generateDescriptions/schemaUtils
+ */
+
+const { z } = require('zod');
+
+function createEventDescriptionSchema(properties) {
+  function buildPropertySchema(prop) {
+    if (prop.properties) {
+      const subPropertiesSchema = {};
+      for (const subPropName in prop.properties) {
+        subPropertiesSchema[subPropName] = buildPropertySchema(prop.properties[subPropName]);
+      }
+      return z.object({
+        description: z
+          .string()
+          .describe('A maximum of 10 words describing the property and what it means'),
+        properties: z.object(subPropertiesSchema),
+      });
+    } else {
+      return z.object({
+        description: z
+          .string()
+          .describe('A maximum of 10 words describing the property and what it means'),
+      });
+    }
+  }
+
+  // Define the schema for properties
+  const propertiesSchema = {};
+  for (const propName in properties) {
+    propertiesSchema[propName] = buildPropertySchema(properties[propName]);
+  }
+
+  // Define the schema for implementations
+  const implementationsSchema = z.array(
+    z.object({
+      description: z
+        .string()
+        .describe('A maximum of 10 words describing how this event is triggered without using the word "triggered"'),
+      path: z.string(),
+      line: z.number(),
+    })
+  );
+
+  // Construct the full schema
+  const eventDescriptionSchema = z.object({
+    eventDescription: z
+      .string()
+      .describe('A maximum of 10 words describing the event and what it tracks without using the word "tracks"'),
+    properties: z.object(propertiesSchema),
+    implementations: implementationsSchema,
+  });
+
+  return eventDescriptionSchema;
+}
+
+module.exports = {
+  createEventDescriptionSchema
+};

package/src/index.js

@@ -1,12 +1,17 @@
+/**
+ * @fileoverview Main entry point for the analytics tracking analyzer
+ * @module analyze-tracking
+ */
+
 const { analyzeDirectory } = require('./analyze');
-const { getRepoDetails } = require('./repoDetails');
-const { generateYamlSchema } = require('./yamlGenerator');
+const { getRepoDetails } = require('./utils/repoDetails');
+const { generateYamlSchema, generateJsonSchema } = require('./utils/yamlGenerator');
 const { generateDescriptions } = require('./generateDescriptions');
 
 const { ChatOpenAI } = require('@langchain/openai');
 const { ChatVertexAI } = require('@langchain/google-vertexai');
 
-async function run(targetDir, outputPath, customFunction, customSourceDetails, generateDescription, provider, model) {
+async function run(targetDir, outputPath, customFunction, customSourceDetails, generateDescription, provider, model, stdout, format) {
   let events = await analyzeDirectory(targetDir, customFunction);
   if (generateDescription) {
     let llm;
@@ -30,7 +35,11 @@ async function run(targetDir, outputPath, customFunction, customSourceDetails, g
     events = await generateDescriptions(events, targetDir, llm);
   }
   const repoDetails = await getRepoDetails(targetDir, customSourceDetails);
-  generateYamlSchema(events, repoDetails, outputPath);
+  if (format === 'json') {
+    generateJsonSchema(events, repoDetails, outputPath, stdout);
+  } else {
+    generateYamlSchema(events, repoDetails, outputPath, stdout);
+  }
 }
 
 module.exports = { run };

package/src/{fileProcessor.js → utils/fileProcessor.js}

@@ -1,3 +1,8 @@
+/**
+ * @fileoverview File system utilities for recursively reading directories
+ * @module analyze-tracking/utils/fileProcessor
+ */
+
 const fs = require('fs');
 const path = require('path');
 

package/src/{repoDetails.js → utils/repoDetails.js}

@@ -1,3 +1,8 @@
+/**
+ * @fileoverview Git repository utilities for extracting commit and repository information
+ * @module analyze-tracking/utils/repoDetails
+ */
+
 const fs = require('fs');
 const git = require('isomorphic-git');
 const { execSync } = require('child_process');

package/src/utils/yamlGenerator.js

@@ -0,0 +1,47 @@
+/**
+ * @fileoverview YAML schema generator for analytics tracking events
+ * @module analyze-tracking/utils/yamlGenerator
+ */
+
+const fs = require('fs');
+const yaml = require('js-yaml');
+
+const VERSION = 1
+const SCHEMA_URL = "https://raw.githubusercontent.com/fliskdata/analyze-tracking/main/schema.json";
+
+function generateYamlSchema(events, repository, outputPath, stdout = false) {
+  const schema = {
+    version: VERSION,
+    source: repository,
+    events,
+  };
+  const options = {
+    noRefs: true,
+    lineWidth: -1,
+  };
+  const yamlOutput = yaml.dump(schema, options);
+  const yamlFile = `# yaml-language-server: $schema=${SCHEMA_URL}\n${yamlOutput}`;
+  if (stdout) {
+    process.stdout.write(yamlFile);
+  } else {
+    fs.writeFileSync(outputPath, yamlFile, 'utf8');
+    console.log(`Tracking schema YAML file generated: ${outputPath}`);
+  }
+}
+
+function generateJsonSchema(events, repository, outputPath, stdout = false) {
+  const schema = {
+    version: VERSION,
+    source: repository,
+    events,
+  };
+  const jsonFile = JSON.stringify(schema, null, 2);
+  if (stdout) {
+    process.stdout.write(jsonFile);
+  } else {
+    fs.writeFileSync(outputPath, jsonFile, 'utf8');
+    console.log(`Tracking schema JSON file generated: ${outputPath}`);
+  }
+}
+
+module.exports = { generateYamlSchema, generateJsonSchema };