@flisk/analyze-tracking 0.3.1 → 0.4.1

package/README.md

@@ -205,5 +205,12 @@ See [schema.json](schema.json) for a JSON Schema of the output.
 </details>
 
 
+## Supported languages
+
+- JavaScript
+- TypeScript
+- Ruby (Experimental - only supports Segment for now)
+
+
 ## Contribute
 We’re actively improving this package. Found a bug? Want to request a feature? Open an issue or contribute directly!

package/bin/cli.js

@@ -1,4 +1,4 @@
-#!/usr/bin/env node
+#!/usr/bin/env node --no-warnings=ExperimentalWarning
 
 const path = require('path');
 const commandLineArgs = require('command-line-args');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flisk/analyze-tracking",
-  "version": "0.3.1",
+  "version": "0.4.1",
   "description": "Analyzes tracking code in a project and generates data schemas",
   "main": "src/index.js",
   "bin": {
@@ -20,6 +20,7 @@
   },
   "homepage": "https://github.com/fliskdata/analyze-tracking#readme",
   "dependencies": {
+    "@ruby/prism": "^1.4.0",
     "@typescript-eslint/parser": "^8.1.0",
     "acorn": "^8.12.1",
     "acorn-jsx": "^5.3.2",

package/src/analyze/analyzeRubyFile.js

@@ -0,0 +1,290 @@
+const fs = require('fs');
+
+let parse = null;
+
+// Create a visitor to traverse the AST
+class TrackingVisitor {
+  constructor(code, filePath) {
+    this.code = code;
+    this.lines = code.split('\n');
+    this.ancestors = [];
+    this.events = [];
+    this.filePath = filePath;
+  }
+
+  getLineNumber(location) {
+    // Count the number of newlines before the start offset
+    const beforeStart = this.code.slice(0, location.startOffset);
+    return beforeStart.split('\n').length;
+  }
+
+  async findWrappingFunction(node, ancestors) {
+    const { DefNode, BlockNode, LambdaNode } = await import('@ruby/prism');
+  
+    for (let i = ancestors.length - 1; i >= 0; i--) {
+      const current = ancestors[i];
+  
+      // Handle method definitions
+      if (current instanceof DefNode) {
+        return current.name;
+      }
+  
+      // Handle blocks and lambdas
+      if (current instanceof BlockNode || current instanceof LambdaNode) {
+        return 'block';
+      }
+    }
+    return 'global';
+  }
+
+  detectSource(node) {
+    if (!node) return null;
+  
+    // Check for other analytics libraries
+    if (node.receiver) {
+      const objectName = node.receiver.name;
+      const methodName = node.name;
+
+      if (objectName === 'Analytics' && methodName === 'track') return 'segment';
+    }
+  
+    return null;
+  }
+
+  extractEventName(node, source) {
+    if (source === 'segment') {
+      const params = node.arguments_.arguments_[0].elements;
+      const eventProperty = params.find(param => param?.key?.unescaped?.value === 'event');
+      return eventProperty?.value?.unescaped?.value || null;
+    }
+
+    return null;
+  }
+
+  async extractProperties(node, source) {
+    const { HashNode, ArrayNode } = await import('@ruby/prism');
+
+    if (source === 'segment') {
+      const params = node.arguments_.arguments_[0].elements;
+      const properties = {};
+
+      // Process all top-level fields except 'event'
+      for (const param of params) {
+        const key = param?.key?.unescaped?.value;
+        
+        if (key && key !== 'event') {
+          const value = param?.value;
+
+          if (key === 'properties' && value instanceof HashNode) {
+            // Merge properties from the 'properties' hash into the top level
+            const nestedProperties = await this.extractHashProperties(value);
+            Object.assign(properties, nestedProperties);
+          } else if (value instanceof HashNode) {
+            // Handle other nested hash objects
+            const hashProperties = await this.extractHashProperties(value);
+            properties[key] = {
+              type: 'object',
+              properties: hashProperties
+            };
+          } else if (value instanceof ArrayNode) {
+            // Handle arrays
+            const arrayItems = await this.extractArrayItemProperties(value);
+            properties[key] = {
+              type: 'array',
+              items: arrayItems
+            };
+          } else {
+            // Handle primitive values
+            const valueType = await this.getValueType(value);
+            properties[key] = {
+              type: valueType
+            };
+          }
+        }
+      }
+
+      return properties;
+    }
+
+    return null;
+  }
+
+  async extractHashProperties(hashNode) {
+    const { AssocNode, HashNode, ArrayNode } = await import('@ruby/prism');
+    const properties = {};
+    
+    for (const element of hashNode.elements) {
+      if (element instanceof AssocNode) {
+        const key = element.key.unescaped?.value;
+        const value = element.value;
+
+        if (key) {
+          if (value instanceof HashNode) {
+            // Handle nested hash objects
+            const nestedProperties = await this.extractHashProperties(value);
+            properties[key] = {
+              type: 'object',
+              properties: nestedProperties
+            };
+          } else if (value instanceof ArrayNode) {
+            // Handle arrays
+            const items = await this.extractArrayItemProperties(value);
+            properties[key] = {
+              type: 'array',
+              items
+            };
+          } else {
+            // Handle primitive values
+            const valueType = await this.getValueType(value);
+            properties[key] = {
+              type: valueType
+            };
+          }
+        }
+      }
+    }
+
+    return properties;
+  }
+
+  async extractArrayItemProperties(arrayNode) {
+    const { HashNode } = await import('@ruby/prism');
+
+    if (arrayNode.elements.length === 0) {
+      return { type: 'any' };
+    }
+
+    const firstItem = arrayNode.elements[0];
+    if (firstItem instanceof HashNode) {
+      return {
+        type: 'object',
+        properties: this.extractHashProperties(firstItem)
+      };
+    } else {
+      const valueType = await this.getValueType(firstItem);
+      return {
+        type: valueType
+      };
+    }
+  }
+
+  async getValueType(node) {
+    const { StringNode, IntegerNode, FloatNode, TrueNode, FalseNode, NilNode, SymbolNode, CallNode } = await import('@ruby/prism');
+
+    if (node instanceof StringNode) return 'string';
+    if (node instanceof IntegerNode || node instanceof FloatNode) return 'number';
+    if (node instanceof TrueNode || node instanceof FalseNode) return 'boolean';
+    if (node instanceof NilNode) return 'null';
+    if (node instanceof SymbolNode) return 'string';
+    if (node instanceof CallNode) return 'any'; // Dynamic values
+    return 'any'; // Default type
+  }
+
+  async visit(node) {
+    const { CallNode, ProgramNode, StatementsNode, DefNode, IfNode, BlockNode, ArgumentsNode, HashNode, AssocNode, ClassNode } = await import('@ruby/prism');
+    if (!node) return;
+
+    this.ancestors.push(node);
+
+    // Check if this is a tracking call
+    if (node instanceof CallNode) {
+      try {
+        const source = this.detectSource(node);
+        const eventName = this.extractEventName(node, source);
+
+        if (!source || !eventName) {
+          this.ancestors.pop();
+          return;
+        }
+
+        const line = this.getLineNumber(node.location);
+        const functionName = await this.findWrappingFunction(node, this.ancestors);
+        const properties = await this.extractProperties(node, source);
+
+        this.events.push({
+          eventName,
+          source,
+          properties,
+          filePath: this.filePath,
+          line,
+          functionName
+        });
+      } catch (nodeError) {
+        console.error(`Error processing node in ${this.filePath}`);
+      }
+    }
+
+    // Visit all child nodes
+    if (node instanceof ProgramNode) {
+      await this.visit(node.statements);
+    } else if (node instanceof StatementsNode) {
+      for (const child of node.body) {
+        await this.visit(child);
+      }
+    } else if (node instanceof ClassNode) {
+      if (node.body) {
+        await this.visit(node.body);
+      }
+    } else if (node instanceof DefNode) {
+      if (node.body) {
+        await this.visit(node.body);
+      }
+    } else if (node instanceof IfNode) {
+      if (node.statements) {
+        await this.visit(node.statements);
+      }
+      if (node.subsequent) {
+        await this.visit(node.subsequent);
+      }
+    } else if (node instanceof BlockNode) {
+      if (node.body) {
+        await this.visit(node.body);
+      }
+    } else if (node instanceof ArgumentsNode) {
+      for (const arg of node.arguments) {
+        await this.visit(arg);
+      }
+    } else if (node instanceof HashNode) {
+      for (const element of node.elements) {
+        await this.visit(element);
+      }
+    } else if (node instanceof AssocNode) {
+      await this.visit(node.key);
+      await this.visit(node.value);
+    }
+
+    this.ancestors.pop();
+  }
+}
+
+async function analyzeRubyFile(filePath) {
+  // Lazy load the ruby prism parser
+  if (!parse) {
+    const { loadPrism } = await import('@ruby/prism');
+    parse = await loadPrism();
+  }
+
+  try {
+    const code = fs.readFileSync(filePath, 'utf8');
+    let ast;
+    try {
+      ast = await parse(code);
+    } catch (parseError) {
+      console.error(`Error parsing file ${filePath}`);
+      return []; // Return empty events array if parsing fails
+    }
+
+    // Traverse the AST starting from the program node
+    const visitor = new TrackingVisitor(code, filePath);
+    await visitor.visit(ast.value);
+
+    return visitor.events;
+
+  } catch (fileError) {
+    console.error(`Error reading or processing file ${filePath}`);
+  }
+
+  return [];
+}
+
+module.exports = { analyzeRubyFile };

package/src/analyze/index.js

@@ -1,25 +1,40 @@
+const path = require('path');
+const ts = require('typescript');
+const { getAllFiles } = require('../fileProcessor');
 const { analyzeJsFile } = require('./analyzeJsFile');
 const { analyzeTsFile } = require('./analyzeTsFile');
-const { getAllFiles } = require('../fileProcessor');
-const ts = require('typescript');
-const path = require('path');
+const { analyzeRubyFile } = require('./analyzeRubyFile');
 
-function analyzeDirectory(dirPath, customFunction) {
-  const files = getAllFiles(dirPath);
+async function analyzeDirectory(dirPath, customFunction) {
   const allEvents = {};
 
+  const files = getAllFiles(dirPath);
   const tsFiles = files.filter(file => /\.(tsx?)$/.test(file));
-  const program = ts.createProgram(tsFiles, {
+  const tsProgram = ts.createProgram(tsFiles, {
     target: ts.ScriptTarget.ESNext,
     module: ts.ModuleKind.CommonJS,
   });
 
-  files.forEach((file) => {
+  for (const file of files) {
+    let events = [];
+
+    const isJsFile = /\.(jsx?)$/.test(file);
     const isTsFile = /\.(tsx?)$/.test(file);
-    const events = isTsFile ? analyzeTsFile(file, program, customFunction) : analyzeJsFile(file, customFunction);
+    const isRubyFile = /\.(rb)$/.test(file);
+
+    if (isJsFile) {
+      events = analyzeJsFile(file, customFunction);
+    } else if (isTsFile) {
+      events = analyzeTsFile(file, tsProgram, customFunction);
+    } else if (isRubyFile) {
+      events = await analyzeRubyFile(file);
+    } else {
+      console.info(`Skipping file ${file} because it is not a supported file type`);
+      continue;
+    }
 
     events.forEach((event) => {
-      const relativeFilePath = path.relative(dirPath, event.filePath); // Calculate relative path
+      const relativeFilePath = path.relative(dirPath, event.filePath);
 
       if (!allEvents[event.eventName]) {
         allEvents[event.eventName] = {
@@ -45,7 +60,7 @@ function analyzeDirectory(dirPath, customFunction) {
         };
       }
     });
-  });
+  }
 
   return allEvents;
 }

package/src/fileProcessor.js

@@ -18,12 +18,19 @@ function getAllFiles(dirPath, arrayOfFiles = []) {
       }
     }
 
+    // Skip hidden files and directories
+    if (file.startsWith('.')) return
+
+    // Skip common directories we don't want to analyze
+    if (file === 'node_modules') return
+    if (file === 'coverage') return
+    if (file === 'temp') return
+    if (file === 'tmp') return
+    if (file === 'log') return
+
     if (stats.isDirectory()) {
-      if (file === 'node_modules') {
-        return; // Ignore the node_modules directory
-      }
       arrayOfFiles = getAllFiles(fullPath, arrayOfFiles);
-    } else if (/\.((j|t)sx?)$/.test(file)) {
+    } else {
       arrayOfFiles.push(fullPath);
     }
   });

package/src/generateDescriptions.js

@@ -5,7 +5,7 @@ const { z } = require('zod');
 const { zodResponseFormat } = require('openai/helpers/zod');
 
 const openai = new OpenAI({
-  apiKey: process.env.OPENAI_API_KEY,
+  apiKey: process.env.OPENAI_API_KEY || 'undefined',
 });
 const model = 'gpt-4o-mini';
 

package/src/index.js

@@ -4,7 +4,7 @@ const { generateYamlSchema } = require('./yamlGenerator');
 const { generateDescriptions } = require('./generateDescriptions');
 
 async function run(targetDir, outputPath, customFunction, customSourceDetails, generateDescription) {
-  let events = analyzeDirectory(targetDir, customFunction);
+  let events = await analyzeDirectory(targetDir, customFunction);
   if (generateDescription) {
     events = await generateDescriptions(events, targetDir);
   }