@flisk/analyze-tracking 0.7.1 → 0.7.3

package/README.md

@@ -1,8 +1,8 @@
 # @flisk/analyze-tracking
 
-Automatically document your analytics setup by analyzing tracking code and generating data schemas from tools like Segment, Amplitude, Mixpanel, and more 🚀.
+Automatically document your analytics setup by analyzing tracking code and generating data schemas from tools like Segment, Amplitude, Mixpanel, and more 🚀
 
-[![NPM version](https://img.shields.io/npm/v/@flisk/analyze-tracking.svg)](https://www.npmjs.com/package/@flisk/analyze-tracking)
+[![NPM version](https://img.shields.io/npm/v/@flisk/analyze-tracking.svg)](https://www.npmjs.com/package/@flisk/analyze-tracking) [![Tests](https://github.com/fliskdata/analyze-tracking/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/fliskdata/analyze-tracking/actions/workflows/tests.yml)
 
 
 ## Why Use @flisk/analyze-tracking?
@@ -30,7 +30,7 @@ npx @flisk/analyze-tracking /path/to/project [options]
 - `-o, --output <output_file>`: Name of the output file (default: `tracking-schema.yaml`)
 - `-c, --customFunction <function_name>`: Specify a custom tracking function
 
-🔑&nbsp; **Important:** If you are using `generateDescription`, you must set the appropriate credentials for the provider you are using as an environment variable. OpenAI uses `OPENAI_API_KEY` and Google Vertex AI uses `GOOGLE_APPLICATION_CREDENTIALS`.
+🔑&nbsp; **Important:** If you are using `generateDescription`, you must set the appropriate credentials for the LLM provider you are using as an environment variable. OpenAI uses `OPENAI_API_KEY` and Google Vertex AI uses `GOOGLE_APPLICATION_CREDENTIALS`.
 
 <details>
   <summary>Note on Custom Functions 💡</summary>
@@ -96,10 +96,10 @@ See [schema.json](schema.json) for a JSON Schema of the output.
 | Mixpanel          | ✅ | ✅ | ✅ | ✅ |
 | Amplitude         | ✅ | ✅ | ❌ | ✅ |
 | Rudderstack       | ✅ | ✅ | ✳️ | ✳️ |
-| mParticle         | ✅ | ✅ | ❌ | ❌ |
+| mParticle         | ✅ | ❌ | ❌ | ❌ |
 | PostHog           | ✅ | ✅ | ✅ | ✅ |
-| Pendo             | ✅ | ✅ | ❌ | ❌ |
-| Heap              | ✅ | ✅ | ❌ | ❌ |
+| Pendo             | ✅ | ❌ | ❌ | ❌ |
+| Heap              | ✅ | ❌ | ❌ | ❌ |
 | Snowplow          | ✅ | ✅ | ✅ | ✅ |
 | Custom Function   | ✅ | ✅ | ✅ | ✅ |
 
@@ -198,16 +198,22 @@ See [schema.json](schema.json) for a JSON Schema of the output.
 
   **JavaScript/TypeScript**
   ```js
-  amplitude.logEvent('<event_name>', {
+  amplitude.track('<event_name>', {
     <event_parameters>
   });
   ```
 
   **Python**
   ```python
-  amplitude.track('<event_name>', {
-    '<property_name>': '<property_value>'
-  })
+  client.track(
+    BaseEvent(
+      event_type="<event_name>",
+      user_id="<user_id>",
+      event_properties={
+        "<property_name>": "<property_value>",
+      },
+    )
+  )
   ```
 
   **Go**
@@ -266,19 +272,10 @@ See [schema.json](schema.json) for a JSON Schema of the output.
 
   **JavaScript/TypeScript**
   ```js
-  mParticle.logEvent('<event_name>', {
+  mParticle.logEvent('<event_name>', mParticle.EventType.<event_type>, {
     '<property_name>': '<property_value>'
   });
   ```
-
-  **Python**
-  ```python
-  mParticle.logEvent('<event_name>', {
-    '<property_name>': '<property_value>'
-  })
-  ```
-
-
 </details>
 
 <details>
@@ -293,13 +290,9 @@ See [schema.json](schema.json) for a JSON Schema of the output.
 
   **Python**
   ```python
-  posthog.capture(
-    'distinct_id',
-    '<event_name>',
-    {
-      '<property_name>': '<property_value>'
-    }
-  )
+  posthog.capture('distinct_id', '<event_name>', {
+    '<property_name>': '<property_value>'
+  })
   # Or
   posthog.capture(
     'distinct_id',
@@ -377,61 +370,42 @@ See [schema.json](schema.json) for a JSON Schema of the output.
 
   **JavaScript/TypeScript**
   ```js
-  snowplow('trackStructEvent', {
+  tracker.track(buildStructEvent({
+    action: '<event_name>',
     category: '<category>',
-    action: '<action>',
     label: '<label>',
     property: '<property>',
-    value: '<value>'
-  });
+    value: <value>
+  }));
   ```
 
   **Python**
   ```python
-  # Direct tracking
-  trackStructEvent({
-    'category': '<category>',
-    'action': '<action>',
-    'label': '<label>',
-    'property': '<property>',
-    'value': '<value>'
-  })
-  
-  # Builder pattern
-  buildStructEvent({
-    'category': '<category>',
-    'action': '<action>',
-    'label': '<label>',
-    'property': '<property>',
-    'value': '<value>'
-  })
-  
-  # Function call pattern
-  snowplow('trackStructEvent', {
-    'category': '<category>',
-    'action': '<action>',
-    'label': '<label>',
-    'property': '<property>',
-    'value': '<value>'
-  })
+  tracker.track(StructuredEvent(
+    action="<event_name>",
+    category="<category>",
+    label="<label>",
+    property_="<property>",
+    value=<value>,
+  ))
   ```
 
   **Ruby**
   ```ruby
   tracker.track_struct_event(
+    action: '<event_name>',
     category: '<category>',
-    action: '<action>',
     label: '<label>',
     property: '<property>',
-    value: '<value>'
+    value: <value>
   )
   ```
 
   **Go**
   ```go
   tracker.TrackStructEvent(sp.StructuredEvent{
+		Action:   sp.NewString("<event_name>"),
 		Category: sp.NewString("<category>"),
-		Action:   sp.NewString("<action>"),
 		Label:    sp.NewString("<label>"),
 		Property: sp.NewString("<property>"),
 		Value:    sp.NewFloat64(<value>),
@@ -441,4 +415,4 @@ See [schema.json](schema.json) for a JSON Schema of the output.
 
 
 ## Contribute
-We're actively improving this package. Found a bug? Want to request a feature? Open an issue or contribute directly!
+We're actively improving this package. Found a bug? Have a feature request? Open an issue or submit a pull request!

package/bin/cli.js

@@ -31,7 +31,7 @@ const optionDefinitions = [
     name: 'model',
     alias: 'm',
     type: String,
-    defaultValue: 'gpt-4o-mini',
+    defaultValue: 'gpt-4.1-nano',
   },
   {
     name: 'output',

package/package.json

@@ -1,14 +1,28 @@
 {
   "name": "@flisk/analyze-tracking",
-  "version": "0.7.1",
+  "version": "0.7.3",
   "description": "Analyzes tracking code in a project and generates data schemas",
   "main": "src/index.js",
   "bin": {
     "analyze-tracking": "bin/cli.js"
   },
   "scripts": {
-    "test": "jest"
+    "test": "node tests",
+    "test:js": "node --test tests/analyzeJavaScript.test.js",
+    "test:ts": "node --test tests/analyzeTypeScript.test.js",
+    "test:python": "node --experimental-vm-modules --test tests/analyzePython.test.js",
+    "test:ruby": "node --experimental-vm-modules --test tests/analyzeRuby.test.js",
+    "test:go": "node --test tests/analyzeGo.test.js",
+    "test:cli": "node --test tests/cli.test.js",
+    "test:schema": "node --test tests/schema.test.js",
+    "test:generateDescriptions": "node --test tests/generateDescriptions.test.js",
+    "test:utils": "node --test tests/utils.test.js"
   },
+  "files": [
+    "bin",
+    "src",
+    "schema.json"
+  ],
   "repository": {
     "type": "git",
     "url": "git+https://github.com/fliskdata/analyze-tracking.git"
@@ -38,6 +52,7 @@
     "zod": "^3.24.4"
   },
   "devDependencies": {
-    "jest": "^29.7.0"
+    "ajv": "^8.17.1",
+    "lodash": "^4.17.21"
   }
 }

package/src/analyze/go/astTraversal.js

@@ -0,0 +1,121 @@
+/**
+ * @fileoverview AST traversal utilities for Go code analysis
+ * @module analyze/go/astTraversal
+ */
+
+const { MAX_RECURSION_DEPTH } = require('./constants');
+const { extractTrackingEvent } = require('./trackingExtractor');
+
+/**
+ * Extract events from a body of statements
+ * @param {Array<Object>} body - Array of AST statement nodes to process
+ * @param {Array<Object>} events - Array to collect found tracking events (modified in place)
+ * @param {string} filePath - Path to the file being analyzed
+ * @param {string} functionName - Name of the current function being processed
+ * @param {string|null} customFunction - Name of custom tracking function to detect
+ * @param {Object} typeContext - Type information context for variable resolution
+ * @param {string} currentFunction - Current function context for type lookups
+ */
+function extractEventsFromBody(body, events, filePath, functionName, customFunction, typeContext, currentFunction) {
+  for (const stmt of body) {
+    if (stmt.tag === 'exec' && stmt.expr) {
+      processExpression(stmt.expr, events, filePath, functionName, customFunction, typeContext, currentFunction);
+    } else if (stmt.tag === 'declare' && stmt.value) {
+      // Handle variable declarations with tracking calls
+      processExpression(stmt.value, events, filePath, functionName, customFunction, typeContext, currentFunction);
+    } else if (stmt.tag === 'assign' && stmt.rhs) {
+      // Handle assignments with tracking calls
+      processExpression(stmt.rhs, events, filePath, functionName, customFunction, typeContext, currentFunction);
+    } else if (stmt.tag === 'if' && stmt.body) {
+      extractEventsFromBody(stmt.body, events, filePath, functionName, customFunction, typeContext, currentFunction);
+    } else if (stmt.tag === 'elseif' && stmt.body) {
+      extractEventsFromBody(stmt.body, events, filePath, functionName, customFunction, typeContext, currentFunction);
+    } else if (stmt.tag === 'else' && stmt.body) {
+      extractEventsFromBody(stmt.body, events, filePath, functionName, customFunction, typeContext, currentFunction);
+    } else if (stmt.tag === 'for' && stmt.body) {
+      extractEventsFromBody(stmt.body, events, filePath, functionName, customFunction, typeContext, currentFunction);
+    } else if (stmt.tag === 'foreach' && stmt.body) {
+      extractEventsFromBody(stmt.body, events, filePath, functionName, customFunction, typeContext, currentFunction);
+    } else if (stmt.tag === 'switch' && stmt.cases) {
+      for (const caseNode of stmt.cases) {
+        if (caseNode.body) {
+          extractEventsFromBody(caseNode.body, events, filePath, functionName, customFunction, typeContext, currentFunction);
+        }
+      }
+    }
+  }
+}
+
+/**
+ * Process an expression to find tracking calls
+ * @param {Object} expr - AST expression node to process
+ * @param {Array<Object>} events - Array to collect found tracking events (modified in place)
+ * @param {string} filePath - Path to the file being analyzed
+ * @param {string} functionName - Name of the current function being processed
+ * @param {string|null} customFunction - Name of custom tracking function to detect
+ * @param {Object} typeContext - Type information context for variable resolution
+ * @param {string} currentFunction - Current function context for type lookups
+ * @param {number} [depth=0] - Current recursion depth (used to prevent infinite recursion)
+ */
+function processExpression(expr, events, filePath, functionName, customFunction, typeContext, currentFunction, depth = 0) {
+  if (!expr || depth > MAX_RECURSION_DEPTH) return; // Prevent infinite recursion with depth limit
+  
+  // Handle array of expressions
+  if (Array.isArray(expr)) {
+    for (const item of expr) {
+      processExpression(item, events, filePath, functionName, customFunction, typeContext, currentFunction, depth + 1);
+    }
+    return;
+  }
+  
+  // Handle single expression with body
+  if (expr.body) {
+    for (const item of expr.body) {
+      processExpression(item, events, filePath, functionName, customFunction, typeContext, currentFunction, depth + 1);
+    }
+    return;
+  }
+  
+  // Handle specific node types
+  if (expr.tag === 'call') {
+    const trackingCall = extractTrackingEvent(expr, filePath, functionName, customFunction, typeContext, currentFunction);
+    if (trackingCall) {
+      events.push(trackingCall);
+    }
+    
+    // Also process call arguments
+    if (expr.args) {
+      processExpression(expr.args, events, filePath, functionName, customFunction, typeContext, currentFunction, depth + 1);
+    }
+  } else if (expr.tag === 'structlit') {
+    // Check if this struct literal is a tracking event
+    const trackingCall = extractTrackingEvent(expr, filePath, functionName, customFunction, typeContext, currentFunction);
+    if (trackingCall) {
+      events.push(trackingCall);
+    }
+    
+    // Process fields (but don't recurse into field values for tracking structs)
+    if (!trackingCall && expr.fields) {
+      for (const field of expr.fields) {
+        if (field.value) {
+          processExpression(field.value, events, filePath, functionName, customFunction, typeContext, currentFunction, depth + 1);
+        }
+      }
+    }
+  }
+  
+  // Process other common properties that might contain expressions
+  if (expr.value && expr.tag !== 'structlit') {
+    processExpression(expr.value, events, filePath, functionName, customFunction, typeContext, currentFunction, depth + 1);
+  }
+  if (expr.lhs) {
+    processExpression(expr.lhs, events, filePath, functionName, customFunction, typeContext, currentFunction, depth + 1);
+  }
+  if (expr.rhs) {
+    processExpression(expr.rhs, events, filePath, functionName, customFunction, typeContext, currentFunction, depth + 1);
+  }
+}
+
+module.exports = {
+  extractEventsFromBody
+};

package/src/analyze/go/constants.js

@@ -0,0 +1,20 @@
+/**
+ * @fileoverview Constants for Go analytics tracking analysis
+ * @module analyze/go/constants
+ */
+
+const ANALYTICS_SOURCES = {
+  SEGMENT: 'segment',
+  POSTHOG: 'posthog',
+  AMPLITUDE: 'amplitude',
+  MIXPANEL: 'mixpanel',
+  SNOWPLOW: 'snowplow',
+  CUSTOM: 'custom'
+};
+
+const MAX_RECURSION_DEPTH = 20;
+
+module.exports = {
+  ANALYTICS_SOURCES,
+  MAX_RECURSION_DEPTH
+};

package/src/analyze/go/eventDeduplicator.js

@@ -0,0 +1,47 @@
+/**
+ * @fileoverview Event deduplication utilities for Go analytics tracking
+ * @module analyze/go/eventDeduplicator
+ */
+
+const { ANALYTICS_SOURCES } = require('./constants');
+
+/**
+ * Deduplicate events based on eventName, source, and function
+ * For Amplitude, prefer struct literal line numbers over function call line numbers
+ * @param {Array<Object>} events - Array of tracking events to deduplicate
+ * @returns {Array<Object>} Array of unique tracking events
+ */
+function deduplicateEvents(events) {
+  const uniqueEvents = [];
+  const seen = new Set();
+  
+  for (const event of events) {
+    // For Amplitude, we want to keep the line number from the struct literal
+    // For other sources, we can use any line number since they don't have this issue
+    const key = `${event.eventName}:${event.source}:${event.functionName}`;
+    if (!seen.has(key)) {
+      seen.add(key);
+      uniqueEvents.push(event);
+    } else {
+      // If we've seen this event before and it's Amplitude, check if this is the struct literal version
+      const existingEvent = uniqueEvents.find(e => 
+        e.eventName === event.eventName && 
+        e.source === event.source && 
+        e.functionName === event.functionName
+      );
+      
+      // If this is Amplitude and the existing event is from the function call (higher line number),
+      // replace it with this one (from the struct literal)
+      if (event.source === ANALYTICS_SOURCES.AMPLITUDE && existingEvent && existingEvent.line > event.line) {
+        const index = uniqueEvents.indexOf(existingEvent);
+        uniqueEvents[index] = event;
+      }
+    }
+  }
+  
+  return uniqueEvents;
+}
+
+module.exports = {
+  deduplicateEvents
+};

package/src/analyze/go/eventExtractor.js

@@ -0,0 +1,156 @@
+/**
+ * @fileoverview Event extraction logic for Go analytics tracking
+ * @module analyze/go/eventExtractor
+ */
+
+const { ANALYTICS_SOURCES } = require('./constants');
+const { extractStringValue, findStructLiteral, findStructField, extractSnowplowValue } = require('./utils');
+
+/**
+ * Extract event name from a tracking call based on the source
+ * @param {Object} callNode - AST node representing a function call or struct literal
+ * @param {string} source - Analytics source (e.g., 'segment', 'amplitude')
+ * @returns {string|null} Event name or null if not found
+ */
+function extractEventName(callNode, source) {
+  if (!callNode.args || callNode.args.length === 0) {
+    // For struct literals, we need to check fields instead of args
+    if (!callNode.fields || callNode.fields.length === 0) {
+      return null;
+    }
+  }
+  
+  switch (source) {
+    case ANALYTICS_SOURCES.MIXPANEL:
+      return extractMixpanelEventName(callNode);
+      
+    case ANALYTICS_SOURCES.SEGMENT:
+    case ANALYTICS_SOURCES.POSTHOG:
+      return extractSegmentPosthogEventName(callNode);
+      
+    case ANALYTICS_SOURCES.AMPLITUDE:
+      return extractAmplitudeEventName(callNode);
+      
+    case ANALYTICS_SOURCES.SNOWPLOW:
+      return extractSnowplowEventName(callNode);
+      
+    case ANALYTICS_SOURCES.CUSTOM:
+      return extractCustomEventName(callNode);
+  }
+  
+  return null;
+}
+
+/**
+ * Extract Mixpanel event name
+ * Pattern: mp.Track(ctx, []*mixpanel.Event{mp.NewEvent("event_name", "", props)})
+ * @param {Object} callNode - AST node for Mixpanel tracking call
+ * @returns {string|null} Event name or null if not found
+ */
+function extractMixpanelEventName(callNode) {
+  if (callNode.args && callNode.args.length > 1) {
+    const arrayArg = callNode.args[1];
+    if (arrayArg.tag === 'expr' && arrayArg.body) {
+      const arrayLit = arrayArg.body.find(item => item.tag === 'arraylit');
+      if (arrayLit && arrayLit.items && arrayLit.items.length > 0) {
+        // Each item is an array of tokens that needs to be parsed
+        const firstItem = arrayLit.items[0];
+        if (Array.isArray(firstItem)) {
+          // Look for pattern: mp.NewEvent("event_name", ...)
+          for (let i = 0; i < firstItem.length - 4; i++) {
+            if (firstItem[i].tag === 'ident' && firstItem[i].value === 'mp' &&
+                firstItem[i+1].tag === 'sigil' && firstItem[i+1].value === '.' &&
+                firstItem[i+2].tag === 'ident' && firstItem[i+2].value === 'NewEvent' &&
+                firstItem[i+3].tag === 'sigil' && firstItem[i+3].value === '(') {
+              // Found mp.NewEvent( - next token should be the event name
+              if (firstItem[i+4] && firstItem[i+4].tag === 'string') {
+                return firstItem[i+4].value.slice(1, -1); // Remove quotes
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+  return null;
+}
+
+/**
+ * Extract Segment/PostHog event name
+ * Pattern: analytics.Track{Event: "event_name", ...} or posthog.Capture{Event: "event_name", ...}
+ * @param {Object} callNode - AST node for Segment/PostHog struct literal
+ * @returns {string|null} Event name or null if not found
+ */
+function extractSegmentPosthogEventName(callNode) {
+  if (callNode.fields) {
+    const eventField = findStructField(callNode, 'Event');
+    if (eventField) {
+      return extractStringValue(eventField.value);
+    }
+  }
+  return null;
+}
+
+/**
+ * Extract Amplitude event name
+ * Pattern: amplitude.Event{EventType: "event_name", ...} or client.Track(amplitude.Event{EventType: "event_name", ...})
+ * @param {Object} callNode - AST node for Amplitude tracking call
+ * @returns {string|null} Event name or null if not found
+ */
+function extractAmplitudeEventName(callNode) {
+  // For struct literals: amplitude.Event{EventType: "event_name", ...}
+  if (callNode.tag === 'structlit' && callNode.fields) {
+    const eventTypeField = findStructField(callNode, 'EventType');
+    if (eventTypeField) {
+      return extractStringValue(eventTypeField.value);
+    }
+  }
+  // For function calls: client.Track(amplitude.Event{EventType: "event_name", ...})
+  else if (callNode.args && callNode.args.length > 0) {
+    const eventStruct = findStructLiteral(callNode.args[0]);
+    if (eventStruct && eventStruct.fields) {
+      const eventTypeField = findStructField(eventStruct, 'EventType');
+      if (eventTypeField) {
+        return extractStringValue(eventTypeField.value);
+      }
+    }
+  }
+  return null;
+}
+
+/**
+ * Extract Snowplow event name
+ * Pattern: tracker.TrackStructEvent(sp.StructuredEvent{Action: sphelp.NewString("event_name"), ...})
+ * @param {Object} callNode - AST node for Snowplow tracking call
+ * @returns {string|null} Event name or null if not found
+ */
+function extractSnowplowEventName(callNode) {
+  if (callNode.args && callNode.args.length > 0) {
+    const structEvent = findStructLiteral(callNode.args[0]);
+    if (structEvent && structEvent.fields) {
+      const actionField = findStructField(structEvent, 'Action');
+      if (actionField) {
+        // Snowplow uses sphelp.NewString("value")
+        return extractSnowplowValue(actionField.value);
+      }
+    }
+  }
+  return null;
+}
+
+/**
+ * Extract custom event name
+ * Pattern: customFunction("event_name", props)
+ * @param {Object} callNode - AST node for custom tracking function call
+ * @returns {string|null} Event name or null if not found
+ */
+function extractCustomEventName(callNode) {
+  if (callNode.args && callNode.args.length > 0) {
+    return extractStringValue(callNode.args[0]);
+  }
+  return null;
+}
+
+module.exports = {
+  extractEventName
+};

package/src/analyze/go/goAstParser/constants.js

@@ -0,0 +1,39 @@
+/**
+ * Constants used by the Go AST parser
+ */
+
+// Token sigils (operators and delimiters)
+const SIGIL = [
+  /*TRIPLE*/ "<<=", ">>=",
+  /*DOUBLE*/ "+=", "-=", "*=", "/=", "%=", "++", "--", ":=", "==", "&&", "||", ">=", "<=", "<<", ">>", "&=", "^=", "|=", "!=", "<-",
+  /*SINGLE*/ "=", "+", "-", "*", "/", "%", "{", "}", "[", "]", "(", ")", ",", "&", "|", "!", "<", ">", "^", ";", ":"
+];
+
+// Operators (sigils excluding delimiters)
+const OPERATOR = SIGIL.filter(x => !["{", "}", "[", "]", ";", ":=", "="].includes(x));
+
+// Character classes
+const DOT = ".";
+const WHITESPACE = " \t";
+const NEWLINE = "\n\r";
+const NUMBER = "01234567890";
+const QUOTE = "\"'`";
+
+// Primitive Go types
+const PRIMTYPES = [
+  "int", "byte", "bool", "float32", "float64", 
+  "int8", "int32", "int16", "int64", 
+  "uint8", "uint32", "uint16", "uint64", 
+  "rune", "string"
+];
+
+module.exports = {
+  SIGIL,
+  OPERATOR,
+  DOT,
+  WHITESPACE,
+  NEWLINE,
+  NUMBER,
+  QUOTE,
+  PRIMTYPES
+};