@flisk/analyze-tracking 0.7.1 → 0.7.3

package/src/analyze/go/goAstParser/tokenizer.js

@@ -0,0 +1,196 @@
+const { SIGIL, DOT, WHITESPACE, NEWLINE, NUMBER, QUOTE } = require('./constants');
+
+/**
+ * Convert Go source code to tokens
+ * @param {string} src - Go source code
+ * @returns {Array<Object>} Array of token objects with tag, value, line, and col properties
+ */
+function tokenize(src) {
+  let ident = "";
+  let isNum = false;
+  let tokens = [];
+  let i = 0;
+  let line = 1;
+  let col = 1;
+  let sigilset = Array.from(new Set(SIGIL.join("")));
+
+  /**
+   * Add a newline token if the last token is not already a newline
+   */
+  function newlineMaybe() {
+    if (tokens.length && tokens[tokens.length - 1].tag != "newline") {
+      tokens.push({ tag: "newline", value: "\n", line: line, col: col });
+    }
+  }
+
+  /**
+   * Push the current identifier/number to tokens
+   */
+  function pushIdent() {
+    if (ident.length) {
+      tokens.push({ tag: isNum ? "number" : "ident", value: ident, line: line, col: col - ident.length });
+      ident = "";
+      isNum = false;
+    }
+  }
+
+  /**
+   * Update line and column position based on character
+   * @param {string} char - Character being processed
+   */
+  function advancePosition(char) {
+    if (NEWLINE.includes(char)) {
+      line++;
+      col = 1;
+    } else {
+      col++;
+    }
+  }
+
+  while (i < src.length) {
+    if (WHITESPACE.includes(src[i])) {
+      pushIdent();
+      advancePosition(src[i]);
+      i++;
+    } else if (NEWLINE.includes(src[i])) {
+      pushIdent();
+      newlineMaybe();
+      advancePosition(src[i]);
+      i++;
+    } else if (src[i] == "/" && src[i + 1] == "/") {
+      // Single-line comment
+      var cmt = "";
+      while (src[i] != "\n" && i < src.length) {
+        cmt += src[i];
+        advancePosition(src[i]);
+        i++;
+      }
+      // tokens.push({tag:"comment",value:cmt});
+      newlineMaybe();
+      if (i < src.length) {
+        advancePosition(src[i]);
+        i++;
+      }
+    } else if (src[i] == "/" && src[i + 1] == "*") {
+      // Multi-line comment
+      advancePosition(src[i]);
+      advancePosition(src[i + 1]);
+      i += 2;
+      let lvl = 0;
+      while (true) {
+        if (i > src.length * 2) {
+          throw "Unexpected EOF";
+        }
+        if (src[i - 1] == "/" && src[i] == "*") {
+          lvl++;
+        }
+        if (src[i - 1] == "*" && src[i] == "/") {
+          if (!lvl) {
+            advancePosition(src[i]);
+            i++;
+            break;
+          }
+          lvl--;
+        }
+        advancePosition(src[i]);
+        i++;
+      }
+    } else if (QUOTE.includes(src[i])) {
+      // String/char literal
+      let startLine = line;
+      let startCol = col;
+      let j = i + 1;
+      advancePosition(src[i]); // advance for opening quote
+      while (true) {
+        if (src[j] == "\\") {
+          advancePosition(src[j]);
+          j++;
+          if (j < src.length) {
+            advancePosition(src[j]);
+            j++;
+          }
+        } else if (src[j] == src[i]) {
+          advancePosition(src[j]); // advance for closing quote
+          break;
+        } else {
+          advancePosition(src[j]);
+          j++;
+        }
+      }
+      j++;
+      tokens.push({ tag: src[i] == "'" ? "char" : "string", value: src.slice(i, j), line: startLine, col: startCol });
+      i = j;
+    } else if (src[i] == "." && src[i + 1] == "." && src[i + 2] == ".") {
+      // Ellipsis
+      pushIdent();
+      tokens.push({ tag: "sigil", value: "...", line: line, col: col });
+      advancePosition(src[i]);
+      advancePosition(src[i + 1]);
+      advancePosition(src[i + 2]);
+      i += 3;
+    } else if (sigilset.includes(src[i])) {
+      // Handle sigils
+      if (src[i] == "-" || src[i] == "+") { // e.g. 1e+8 1E-9
+        if (isNum && ident[ident.length - 1] == "e" || ident[ident.length - 1] == "E") {
+          ident += src[i];
+          advancePosition(src[i]);
+          i++;
+          continue;
+        }
+      }
+      pushIdent();
+      let done = false;
+      for (var j = 0; j < SIGIL.length; j++) {
+        let l = SIGIL[j].length;
+        let ok = true;
+        for (var k = 0; k < l; k++) {
+          if (src[i + k] != SIGIL[j][k]) {
+            ok = false;
+            break;
+          }
+        }
+        if (ok) {
+          tokens.push({ tag: "sigil", value: SIGIL[j], line: line, col: col });
+          for (let k = 0; k < l; k++) {
+            advancePosition(src[i + k]);
+          }
+          i += l;
+          done = true;
+          break;
+        }
+      }
+    } else if (DOT.includes(src[i])) {
+      // Handle dot
+      if (isNum) {
+        ident += src[i];
+        advancePosition(src[i]);
+        i++;
+      } else {
+        pushIdent();
+        tokens.push({ tag: "sigil", value: DOT, line: line, col: col });
+        advancePosition(src[i]);
+        i++;
+      }
+    } else if (NUMBER.includes(src[i])) {
+      // Handle numbers
+      if (ident.length == 0) {
+        isNum = true;
+      }
+      ident += src[i];
+      advancePosition(src[i]);
+      i++;
+    } else {
+      // Handle identifiers
+      ident += src[i];
+      advancePosition(src[i]);
+      i++;
+    }
+  }
+  pushIdent();
+  newlineMaybe();
+  return tokens;
+}
+
+module.exports = {
+  tokenize
+};

package/src/analyze/go/goAstParser/typeParser.js

@@ -0,0 +1,202 @@
+const { tillNestEndImpl, splitTokensBy } = require('./utils');
+
+/**
+ * Parse a type from tokens
+ * @param {Array<Object>} toks - Tokens representing a type
+ * @returns {Object} AST node representing the type
+ */
+function parseType(toks) {
+  if (toks.length == 1) {
+    return { tag: toks[0].value };
+  }
+  
+  let i = 0;
+  while (i < toks.length) {
+    function tillNestEnd(l, r) {
+      let [j, tk] = tillNestEndImpl(toks, i, l, r);
+      i = j;
+      return tk;
+    }
+
+    if (toks[i].value == "[") {
+      // Array type
+      let typ = { tag: "array", size: null, item: null };
+      i++;
+      typ.size = parseExpr(tillNestEnd("[", "]"));
+      i++;
+      typ.item = parseType(toks.slice(i));
+      return typ;
+    } else if (toks[i].value == "...") {
+      // Variadic type
+      let typ = { tag: "rest", item: null };
+      i++;
+      typ.item = parseType(toks.slice(i));
+      return typ;
+    } else if (toks[i].value == "*") {
+      // Pointer type
+      return { tag: "ptr", item: parseType(toks.slice(i + 1)) };
+    } else if (toks[i].value == "map") {
+      // Map type
+      let typ = { tag: "map", key: null, value: null };
+      i += 2;
+      let te = tillNestEnd("[", "]");
+      typ.key = parseType(te);
+      i++;
+      typ.value = parseType(toks.slice(i));
+      return typ;
+    } else if (toks[i].value == "func") {
+      // Function type
+      return { tag: "lambda", ...parseFuncSig(toks.slice(i + 1)) };
+    } else if (toks[i].value == "interface") {
+      // Interface type
+      return { tag: "interface" };
+    } else if (toks[i].value == "<-" && toks[i + 1].value == "chan") {
+      // Receive-only channel
+      return { tag: "chan", item: parseType(toks.slice(i + 2)), mode: 'i' };
+    } else if (toks[i].value == "chan" && toks[i + 1].value == "<-") {
+      // Send-only channel
+      return { tag: "chan", item: parseType(toks.slice(i + 2)), mode: 'o' };
+    } else if (toks[i].value == "chan") {
+      // Bidirectional channel
+      return { tag: "chan", item: parseType(toks.slice(i + 1)), mode: 'io' };
+    } else if (toks[i + 1] && toks[i + 1].value == ".") {
+      // Namespaced type
+      return { tag: "namespaced", namespace: toks[i].value, item: parseType(toks.slice(i + 2)) };
+    }
+  }
+}
+
+/**
+ * Parse return types from tokens
+ * @param {Array<Object>} toks - Tokens containing return types
+ * @returns {Array<Object>} Array of return type objects with name and type properties
+ */
+function parseRetTypes(toks) {
+  let items = splitTokensBy(toks, ",");
+  let simple = true;
+  
+  for (let j = 0; j < items.length; j++) {
+    if (items[j].length != 1) {
+      if (items[j][0].value != "map" && items[j][0].value != "[" && items[j][0].value != "*") {
+        simple = false;
+      }
+      break;
+    }
+  }
+  
+  if (simple) {
+    return items.map(x => ({ name: null, type: parseType(x) }));
+  }
+  
+  let ret = items.map(x => ({}));
+  for (let j = items.length - 1; j >= 0; j--) {
+    let name = items[j][0].value;
+    let type = items[j].slice(1);
+    if (!type.length) {
+      type = ret[j + 1].type;
+    } else {
+      type = parseType(type);
+    }
+    ret[j].name = name;
+    ret[j].type = type;
+  }
+  return ret;
+}
+
+/**
+ * Parse function arguments
+ * @param {Array<Object>} toks - Tokens containing arguments
+ * @returns {Array<Object>} Array of argument objects with name and type
+ */
+function parseArgs(toks) {
+  let args = [];
+  let i = 0;
+  let lvl = 0;
+  
+  while (i < toks.length) {
+    let arg = {};
+    arg.name = toks[i].value;
+    i++;
+    let typ = [];
+    let lvl = 0;
+    while (i < toks.length) {
+      if (toks[i].value == "(") {
+        lvl++;
+      } else if (toks[i].value == ")") {
+        lvl--;
+      } else if (toks[i].value == ",") {
+        if (lvl == 0) {
+          break;
+        }
+      }
+      typ.push(toks[i]);
+      i++;
+    }
+    arg.type = parseType(typ);
+    i++;
+    args.push(arg);
+  }
+
+  // Infer types for arguments without explicit types
+  for (i = args.length - 1; i >= 0; i--) {
+    if (args[i].type == undefined) {
+      args[i].type = args[i + 1].type;
+    }
+  }
+  return args;
+}
+
+/**
+ * Parse function signature
+ * @param {Array<Object>} toks - Tokens containing function signature
+ * @returns {Object} Object with args and returns properties
+ */
+function parseFuncSig(toks) {
+  let lvl = 0;
+  let k;
+  
+  for (k = 1; k < toks.length; k++) {
+    if (toks[k].value == "(") {
+      lvl++;
+    } else if (toks[k].value == ")") {
+      if (lvl == 0) {
+        break;
+      }
+      lvl--;
+    }
+  }
+
+  let args = toks.slice(1, k);
+  args = parseRetTypes(args);
+  
+  let rets = toks.slice(k + 1);
+  if (rets.length) {
+    while (rets[0].value == "(") {
+      rets = rets.slice(1, -1);
+    }
+    rets = parseRetTypes(rets);
+  } else {
+    rets = [];
+  }
+  
+  return { args: args, returns: rets };
+}
+
+// These need to be injected from the expression parser to avoid circular dependencies
+let parseExpr = null;
+
+/**
+ * Set the expression parser function (to avoid circular dependencies)
+ * @param {Function} exprParser - Expression parser function
+ */
+function setExpressionParser(exprParser) {
+  parseExpr = exprParser;
+}
+
+module.exports = {
+  parseType,
+  parseRetTypes,
+  parseArgs,
+  parseFuncSig,
+  setExpressionParser
+};

package/src/analyze/go/goAstParser/utils.js

@@ -0,0 +1,99 @@
+/**
+ * Parser utility functions
+ */
+
+/**
+ * Get token at index or empty object
+ * @param {Array<Object>} tokens - Token array
+ * @param {number} i - Token index
+ * @returns {Object} Token object or empty object
+ */
+function getToken(tokens, i) {
+  return tokens[i] || {};
+}
+
+/**
+ * Check if tokens array contains a specific value
+ * @param {Array<Object>} toks - Array of tokens
+ * @param {string} val - Value to search for
+ * @returns {boolean} True if value is found
+ */
+function tokensHasValue(toks, val) {
+  for (var i = 0; i < toks.length; i++) {
+    if (toks[i].value == val) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Extract tokens until a matching closing delimiter
+ * @param {Array<Object>} toks - Token array
+ * @param {number} i - Starting index
+ * @param {string} l - Opening delimiter
+ * @param {string} r - Closing delimiter
+ * @returns {Array} [endIndex, extractedTokens]
+ */
+function tillNestEndImpl(toks, i, l, r) {
+  let tk = [];
+  let lvl = 0;
+
+  while (true) {
+    if (i >= toks.length) {
+      return [i, tk];
+    }
+    if (toks[i].value == l) {
+      lvl++;
+    } else if (toks[i].value == r) {
+      if (lvl == 0) {
+        return [i, tk];
+      }
+      lvl--;
+    }
+    tk.push(toks[i]);
+    i++;
+  }
+  return [i, tk];
+}
+
+/**
+ * Split tokens by delimiter(s)
+ * @param {Array<Object>} toks - Tokens to split
+ * @param {string} delim - Primary delimiter
+ * @param {string} [delim2] - Optional secondary delimiter
+ * @returns {Array<Array<Object>>} Array of token groups
+ */
+function splitTokensBy(toks, delim, delim2) {
+  let groups = [];
+  let gp = [];
+  let lvl = 0;
+  for (let i = 0; i < toks.length; i++) {
+    if (toks[i].value == "{" || toks[i].value == "(" || toks[i].value == "[") {
+      lvl++;
+      gp.push(toks[i]);
+    } else if (toks[i].value == "}" || toks[i].value == ")" || toks[i].value == "]") {
+      lvl--;
+      gp.push(toks[i]);
+    } else if (toks[i].value == delim && lvl == 0) {
+      groups.push(gp);
+      gp = [];
+    } else if (delim2 != undefined && toks[i].value == delim2 && lvl == 0) {
+      groups.push(gp);
+      gp = [];
+    } else {
+      gp.push(toks[i]);
+    }
+  }
+  if (groups.length || gp.length) {
+    groups.push(gp);
+  }
+  return groups;
+}
+
+module.exports = {
+  getToken,
+  tokensHasValue,
+  tillNestEndImpl,
+  splitTokensBy
+};

package/src/analyze/go/index.js

@@ -0,0 +1,55 @@
+/**
+ * @fileoverview Go analytics tracking analyzer - main entry point
+ * @module analyze/go
+ */
+
+const fs = require('fs');
+const { extractGoAST } = require('./goAstParser');
+const { buildTypeContext } = require('./typeContext');
+const { deduplicateEvents } = require('./eventDeduplicator');
+const { extractEventsFromBody } = require('./astTraversal');
+
+/**
+ * Analyze a Go file and extract tracking events
+ * @param {string} filePath - Path to the Go file to analyze
+ * @param {string|null} customFunction - Name of custom tracking function to detect (optional)
+ * @returns {Promise<Array>} Array of tracking events found in the file
+ * @throws {Error} If the file cannot be read or parsed
+ */
+async function analyzeGoFile(filePath, customFunction) {
+  try {
+    // Read the Go file
+    const source = fs.readFileSync(filePath, 'utf8');
+    
+    // Parse the Go file using goAstParser
+    const ast = extractGoAST(source);
+    
+    // First pass: build type information for functions and variables
+    const typeContext = buildTypeContext(ast);
+    
+    // Extract tracking events from the AST
+    const events = [];
+    let currentFunction = 'global';
+    
+    // Walk through the AST
+    for (const node of ast) {
+      if (node.tag === 'func') {
+        currentFunction = node.name;
+        // Process the function body
+        if (node.body) {
+          extractEventsFromBody(node.body, events, filePath, currentFunction, customFunction, typeContext, currentFunction);
+        }
+      }
+    }
+    
+    // Deduplicate events based on eventName, source, and function
+    const uniqueEvents = deduplicateEvents(events);
+    
+    return uniqueEvents;
+  } catch (error) {
+    console.error(`Error analyzing Go file ${filePath}:`, error.message);
+    return [];
+  }
+}
+
+module.exports = { analyzeGoFile };