@flisk/analyze-tracking 0.7.2 → 0.7.3

package/src/analyze/typescript/utils/type-resolver.js

@@ -0,0 +1,193 @@
+/**
+ * @fileoverview Utilities for resolving TypeScript types and identifiers
+ * @module analyze/typescript/utils/type-resolver
+ */
+
+const ts = require('typescript');
+
+/**
+ * Resolves an identifier to its initializer node
+ * @param {Object} checker - TypeScript type checker
+ * @param {Object} identifier - Identifier node to resolve
+ * @param {Object} sourceFile - Source file containing the identifier
+ * @returns {Object|null} Initializer node or null
+ */
+function resolveIdentifierToInitializer(checker, identifier, sourceFile) {
+  try {
+    const symbol = checker.getSymbolAtLocation(identifier);
+    if (!symbol || !symbol.valueDeclaration) {
+      return null;
+    }
+    
+    const declaration = symbol.valueDeclaration;
+    
+    // Handle variable declarations
+    if (ts.isVariableDeclaration(declaration) && declaration.initializer) {
+      return declaration.initializer;
+    }
+    
+    // Handle property assignments
+    if (ts.isPropertyAssignment(declaration) && declaration.initializer) {
+      return declaration.initializer;
+    }
+    
+    // Handle parameter with default value
+    if (ts.isParameter(declaration) && declaration.initializer) {
+      return declaration.initializer;
+    }
+    
+    return null;
+  } catch (error) {
+    return null;
+  }
+}
+
+/**
+ * Gets the type string for a node
+ * @param {Object} checker - TypeScript type checker
+ * @param {Object} node - AST node
+ * @returns {string} Type string
+ */
+function getTypeOfNode(checker, node) {
+  try {
+    const type = checker.getTypeAtLocation(node);
+    return checker.typeToString(type);
+  } catch (error) {
+    return 'any';
+  }
+}
+
+/**
+ * Resolves a type string to its properties structure
+ * @param {Object} checker - TypeScript type checker
+ * @param {string} typeString - Type string to resolve
+ * @param {Set} [visitedTypes] - Set of visited types to prevent cycles
+ * @returns {Object} Resolved type structure
+ */
+function resolveTypeToProperties(checker, typeString, visitedTypes = new Set()) {
+  // Prevent infinite recursion for circular references
+  if (visitedTypes.has(typeString)) {
+    return { type: 'object' };
+  }
+  
+  // Handle primitive types
+  if (['string', 'number', 'boolean', 'any', 'unknown', 'null', 'undefined', 'void', 'never'].includes(typeString)) {
+    return { type: typeString };
+  }
+  
+  // Handle array types: T[] or Array<T>
+  const arrayMatch = typeString.match(/^(.+)\[\]$/) || typeString.match(/^Array<(.+)>$/);
+  if (arrayMatch) {
+    const elementType = arrayMatch[1].trim();
+    visitedTypes.add(typeString);
+    const elementProps = resolveTypeToProperties(checker, elementType, visitedTypes);
+    return {
+      type: 'array',
+      items: elementProps
+    };
+  }
+  
+  // Handle readonly array types: readonly T[] or ReadonlyArray<T>
+  const readonlyArrayMatch = typeString.match(/^readonly (.+)\[\]$/) || typeString.match(/^ReadonlyArray<(.+)>$/);
+  if (readonlyArrayMatch) {
+    const elementType = readonlyArrayMatch[1].trim();
+    visitedTypes.add(typeString);
+    const elementProps = resolveTypeToProperties(checker, elementType, visitedTypes);
+    return {
+      type: 'array',
+      items: elementProps
+    };
+  }
+  
+  // Handle union types - preserve them as-is
+  if (typeString.includes('|')) {
+    return { type: typeString };
+  }
+  
+  // Handle intersection types
+  if (typeString.includes('&')) {
+    // For simplicity, mark intersection types as 'object'
+    return { type: 'object' };
+  }
+  
+  // Check if it looks like a custom type/interface
+  if (isCustomType(typeString)) {
+    return {
+      type: 'object',
+      __unresolved: typeString
+    };
+  }
+  
+  // Default case - preserve the type string as-is
+  return { type: typeString };
+}
+
+/**
+ * Checks if a type string represents a custom type or interface
+ * @param {string} typeString - Type string to check
+ * @returns {boolean}
+ */
+function isCustomType(typeString) {
+  // Custom types typically start with uppercase and don't contain certain characters
+  return typeString[0] === typeString[0].toUpperCase() && 
+         !typeString.includes('<') && 
+         !typeString.includes('|') && 
+         !typeString.includes('&') &&
+         !typeString.includes('(') &&
+         !typeString.includes('[');
+}
+
+/**
+ * Gets the basic type of an array element
+ * @param {Object} checker - TypeScript type checker
+ * @param {Object} element - Array element node
+ * @returns {string} Basic type string
+ */
+function getBasicTypeOfArrayElement(checker, element) {
+  if (!element) return 'any';
+  
+  // Check for literal values first
+  if (ts.isStringLiteral(element)) {
+    return 'string';
+  } else if (ts.isNumericLiteral(element)) {
+    return 'number';
+  } else if (element.kind === ts.SyntaxKind.TrueKeyword || element.kind === ts.SyntaxKind.FalseKeyword) {
+    return 'boolean';
+  } else if (ts.isObjectLiteralExpression(element)) {
+    return 'object';
+  } else if (ts.isArrayLiteralExpression(element)) {
+    return 'array';
+  } else if (element.kind === ts.SyntaxKind.NullKeyword) {
+    return 'null';
+  } else if (element.kind === ts.SyntaxKind.UndefinedKeyword) {
+    return 'undefined';
+  }
+  
+  // For identifiers and other expressions, try to get the type
+  const typeString = getTypeOfNode(checker, element);
+  
+  // Extract basic type from TypeScript type string
+  if (typeString.startsWith('"') || typeString.startsWith("'")) {
+    return 'string'; // String literal type
+  } else if (!isNaN(Number(typeString))) {
+    return 'number'; // Numeric literal type
+  } else if (typeString === 'true' || typeString === 'false') {
+    return 'boolean'; // Boolean literal type
+  } else if (typeString.includes('[]') || typeString.startsWith('Array<')) {
+    return 'array';
+  } else if (['string', 'number', 'boolean', 'object', 'null', 'undefined'].includes(typeString)) {
+    return typeString;
+  } else if (isCustomType(typeString)) {
+    return 'object';
+  }
+  
+  return 'any';
+}
+
+module.exports = {
+  resolveIdentifierToInitializer,
+  getTypeOfNode,
+  resolveTypeToProperties,
+  isCustomType,
+  getBasicTypeOfArrayElement
+};

package/src/generateDescriptions/index.js

@@ -0,0 +1,81 @@
+/**
+ * @fileoverview AI-powered description generator for analytics events
+ * @module analyze-tracking/generateDescriptions
+ */
+
+const { createPrompt } = require('./promptUtils');
+const { createEventDescriptionSchema } = require('./schemaUtils');
+const { sendPromptToLLM } = require('./llmUtils');
+
+async function generateEventDescription(eventName, event, codebaseDir, model) {
+  const properties = event.properties || {};
+  const implementations = event.implementations || [];
+
+  // Create prompt for the LLM
+  const prompt = createPrompt(eventName, properties, implementations, codebaseDir);
+
+  // Define the output schema using Zod
+  const eventDescriptionSchema = createEventDescriptionSchema(properties);
+
+  // Send prompt to the LLM and get the structured response
+  const result = await sendPromptToLLM(prompt, eventDescriptionSchema, model);
+
+  return { eventName, descriptions: result?.descriptions || null };
+}
+
+async function generateDescriptions(events, codebaseDir, model) {
+  const eventPromises = Object.entries(events).map(([eventName, event]) =>
+    generateEventDescription(eventName, event, codebaseDir, model)
+  );
+
+  console.info(`Running ${eventPromises.length} prompts in parallel...`);
+
+  const results = await Promise.all(eventPromises);
+
+  // Process results and update the events object
+  results.forEach(({ eventName, descriptions }) => {
+    if (descriptions) {
+      const event = events[eventName];
+      event.description = descriptions.eventDescription;
+
+      // Update property descriptions recursively
+      function updatePropertyDescriptions(eventProperties, descriptionProperties) {
+        for (const propName in descriptionProperties) {
+          if (eventProperties[propName]) {
+            eventProperties[propName].description = descriptionProperties[propName].description;
+            if (eventProperties[propName].properties && descriptionProperties[propName].properties) {
+              updatePropertyDescriptions(
+                eventProperties[propName].properties,
+                descriptionProperties[propName].properties
+              );
+            }
+          }
+        }
+      }
+
+      updatePropertyDescriptions(event.properties, descriptions.properties);
+
+      // Update implementations with descriptions
+      for (let i = 0; i < descriptions.implementations.length; i++) {
+        if (event.implementations[i]) {
+          if (
+            event.implementations[i].path === descriptions.implementations[i].path &&
+            event.implementations[i].line === descriptions.implementations[i].line
+          ) {
+            event.implementations[i].description = descriptions.implementations[i].description;
+          } else {
+            console.error(`Returned implementation description does not match path or line for event: ${eventName}`);
+          }
+        }
+      }
+    } else {
+      console.error(`Failed to get description for event: ${eventName}`);
+    }
+  });
+
+  return events;
+}
+
+module.exports = {
+  generateDescriptions
+};

package/src/generateDescriptions/llmUtils.js

@@ -0,0 +1,33 @@
+/**
+ * @fileoverview LLM integration utilities for generating structured responses
+ * @module analyze-tracking/generateDescriptions/llmUtils
+ */
+
+const { PromptTemplate } = require('@langchain/core/prompts');
+
+async function sendPromptToLLM(prompt, schema, model) {
+  try {
+    const promptTemplate = new PromptTemplate({
+      template: `You are an expert at structured data extraction. Generate detailed descriptions for the following analytics event, its properties, and implementations.\n{input}`,
+      inputVariables: ['input'],
+    });
+
+    const formattedPrompt = await promptTemplate.format({
+      input: prompt,
+    });
+
+    const structuredModel = model.withStructuredOutput(schema);
+    const response = await structuredModel.invoke(formattedPrompt);
+
+    return {
+      descriptions: response,
+    };
+  } catch (error) {
+    console.error('Error during LLM response parsing:', error.message);
+    return null;
+  }
+}
+
+module.exports = {
+  sendPromptToLLM
+};

package/src/generateDescriptions/promptUtils.js

@@ -0,0 +1,62 @@
+/**
+ * @fileoverview Utilities for creating LLM prompts with code snippets
+ * @module analyze-tracking/generateDescriptions/promptUtils
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+function createPrompt(eventName, properties, implementations, codebaseDir) {
+  let prompt = `Event Name: "${eventName}"\n\n`;
+  prompt += `Properties:\n`;
+
+  function appendPropertiesToPrompt(properties, indent = '') {
+    for (const propName in properties) {
+      const prop = properties[propName];
+      prompt += `${indent}- "${propName}" (type: ${prop.type})\n`;
+      if (prop.properties) {
+        prompt += `${indent}  Sub-properties:\n`;
+        appendPropertiesToPrompt(prop.properties, indent + '    ');
+      }
+    }
+  }
+
+  appendPropertiesToPrompt(properties);
+
+  // Add implementations with code snippets
+  prompt += `\nImplementations:\n`;
+  for (const impl of implementations) {
+    const codeSnippet = getCodeSnippet(path.join(codebaseDir, impl.path), impl.line);
+    prompt += `- Path: "${impl.path}", Line: ${impl.line}, Function: "${impl.function}", Destination: "${impl.destination}"\n`;
+    prompt += `Code Snippet:\n`;
+    prompt += '```\n';
+    prompt += codeSnippet + '\n';
+    prompt += '```\n';
+  }
+
+  return prompt;
+}
+
+function getCodeSnippet(filePath, lineNumber, contextLines = 5) {
+  // Extract a code snippet from the file around the specified line
+  try {
+    const fileContent = fs.readFileSync(filePath, 'utf8');
+    const lines = fileContent.split('\n');
+    const startLine = Math.max(0, lineNumber - contextLines - 1);
+    const endLine = Math.min(lines.length, lineNumber + contextLines);
+
+    const snippetLines = lines.slice(startLine, endLine);
+    return snippetLines.join('\n');
+  } catch (e) {
+    // Only log errors if not in test mode
+    if (process.env.NODE_ENV !== 'test' && !process.env.NODE_TEST_CONTEXT) {
+      console.error(`Failed to read file ${filePath}:`, e);
+    }
+    return '';
+  }
+}
+
+module.exports = {
+  createPrompt,
+  getCodeSnippet
+};

package/src/generateDescriptions/schemaUtils.js

@@ -0,0 +1,61 @@
+/**
+ * @fileoverview Zod schema utilities for structuring event descriptions
+ * @module analyze-tracking/generateDescriptions/schemaUtils
+ */
+
+const { z } = require('zod');
+
+function createEventDescriptionSchema(properties) {
+  function buildPropertySchema(prop) {
+    if (prop.properties) {
+      const subPropertiesSchema = {};
+      for (const subPropName in prop.properties) {
+        subPropertiesSchema[subPropName] = buildPropertySchema(prop.properties[subPropName]);
+      }
+      return z.object({
+        description: z
+          .string()
+          .describe('A maximum of 10 words describing the property and what it means'),
+        properties: z.object(subPropertiesSchema),
+      });
+    } else {
+      return z.object({
+        description: z
+          .string()
+          .describe('A maximum of 10 words describing the property and what it means'),
+      });
+    }
+  }
+
+  // Define the schema for properties
+  const propertiesSchema = {};
+  for (const propName in properties) {
+    propertiesSchema[propName] = buildPropertySchema(properties[propName]);
+  }
+
+  // Define the schema for implementations
+  const implementationsSchema = z.array(
+    z.object({
+      description: z
+        .string()
+        .describe('A maximum of 10 words describing how this event is triggered without using the word "triggered"'),
+      path: z.string(),
+      line: z.number(),
+    })
+  );
+
+  // Construct the full schema
+  const eventDescriptionSchema = z.object({
+    eventDescription: z
+      .string()
+      .describe('A maximum of 10 words describing the event and what it tracks without using the word "tracks"'),
+    properties: z.object(propertiesSchema),
+    implementations: implementationsSchema,
+  });
+
+  return eventDescriptionSchema;
+}
+
+module.exports = {
+  createEventDescriptionSchema
+};

package/src/index.js

@@ -1,6 +1,11 @@
+/**
+ * @fileoverview Main entry point for the analytics tracking analyzer
+ * @module analyze-tracking
+ */
+
 const { analyzeDirectory } = require('./analyze');
-const { getRepoDetails } = require('./repoDetails');
-const { generateYamlSchema } = require('./yamlGenerator');
+const { getRepoDetails } = require('./utils/repoDetails');
+const { generateYamlSchema } = require('./utils/yamlGenerator');
 const { generateDescriptions } = require('./generateDescriptions');
 
 const { ChatOpenAI } = require('@langchain/openai');

package/src/{fileProcessor.js → utils/fileProcessor.js}

@@ -1,3 +1,8 @@
+/**
+ * @fileoverview File system utilities for recursively reading directories
+ * @module analyze-tracking/utils/fileProcessor
+ */
+
 const fs = require('fs');
 const path = require('path');
 

package/src/{repoDetails.js → utils/repoDetails.js}

@@ -1,3 +1,8 @@
+/**
+ * @fileoverview Git repository utilities for extracting commit and repository information
+ * @module analyze-tracking/utils/repoDetails
+ */
+
 const fs = require('fs');
 const git = require('isomorphic-git');
 const { execSync } = require('child_process');

package/src/{yamlGenerator.js → utils/yamlGenerator.js}

@@ -1,3 +1,8 @@
+/**
+ * @fileoverview YAML schema generator for analytics tracking events
+ * @module analyze-tracking/utils/yamlGenerator
+ */
+
 const fs = require('fs');
 const yaml = require('js-yaml');