@flisk/analyze-tracking 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Flisk
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,119 @@
+# @flisk/analyze-tracking
+Analyzes tracking code in a project and generates data schemas
+
+## Usage
+```sh
+npx @flisk/analyze-tracking /path/to/project
+```
+
+Optional arguments:
+- `--repository <repository_url>`: URL of the repository where the code is hosted (defaults to git remote origin in project directory)
+- `--output <output_file>`: Name of the output file (default: `tracking-schema.yaml`)
+
+
+## Output Schema
+A YAML file with the following structure is generated:
+
+```yaml
+version: 1.0
+events:
+  <event_name>:
+    sources:
+      - repository: <repository_name>
+        path: <path_to_file>
+        line: <line_number>
+        function: <function_name>
+    destinations:
+      - <destination_name>
+    properties:
+      <property_name>:
+        type: <property_type>
+        required: <property_required>
+        description: <property_description>
+```
+
+See [schema.json](schema.json) for the output schema.
+
+
+## Supported tracking libraries
+
+#### Google Analytics
+```js
+gtag('event', '<event_name>', {
+  <event_parameters>
+});
+```
+
+
+#### Segment
+```js
+analytics.track('<event_name>', {
+  <event_parameters>
+});
+```
+
+
+#### Mixpanel
+```js
+mixpanel.track('<event_name>', {
+  <event_parameters>
+});
+```
+
+
+#### Amplitude
+```js
+amplitude.logEvent('<event_name>', {
+  <event_parameters>
+});
+```
+
+
+#### Rudderstack
+```js
+rudderanalytics.track('<event_name>', {
+  <event_parameters>
+});
+```
+
+
+#### mParticle
+```js
+mParticle.logEvent('<event_name>', {
+  <event_parameters>
+});
+```
+
+
+#### Snowplow
+```js
+snowplow('trackStructEvent', {
+  category: '<category>',
+  action: '<action>',
+  label: '<label>',
+  property: '<property>',
+  value: '<value> '
+});
+```
+
+```js
+trackStructEvent({
+  category: '<category>',
+  action: '<action>',
+  label: '<label>',
+  property: '<property>',
+  value: '<value>'
+});
+```
+
+```js
+buildStructEvent({
+  category: '<category>',
+  action: '<action>',
+  label: '<label>',
+  property: '<property>',
+  value: '<value>'
+});
+```
+
+_Note: Snowplow Self Describing Events are not supported yet._

package/bin/analyze-tracking.js

@@ -0,0 +1,33 @@
+#!/usr/bin/env node
+
+const path = require('path');
+const { execSync } = require('child_process');
+const { run } = require('../src/index');
+
+// Parse command-line arguments
+const targetDir = process.argv[2];
+const repositoryArgIndex = process.argv.indexOf('--repository');
+const repositoryUrl = repositoryArgIndex !== -1 ? process.argv[repositoryArgIndex + 1] : null;
+const outputArgIndex = process.argv.indexOf('--output');
+const outputPath = outputArgIndex !== -1 ? process.argv[outputArgIndex + 1] : 'tracking-schema.yaml';
+
+if (!targetDir) {
+  console.error('Please provide the path to the repository.');
+  process.exit(1);
+}
+
+// Function to get the repository URL using Git
+function getRepositoryUrl() {
+  try {
+    const repoUrl = execSync('git config --get remote.origin.url', { cwd: targetDir, encoding: 'utf8' });
+    return repoUrl.trim();
+  } catch (error) {
+    console.warn('Could not retrieve repository URL. Using default value "unknown".');
+    return 'unknown';
+  }
+}
+
+// Determine the repository URL
+const repository = repositoryUrl || getRepositoryUrl();
+
+run(path.resolve(targetDir), repository, outputPath);

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@flisk/analyze-tracking",
+  "version": "0.1.0",
+  "description": "Analyzes tracking code in a project and generates data schemas",
+  "main": "src/index.js",
+  "bin": {
+    "analyze-tracking": "bin/analyze-tracking.js"
+  },
+  "scripts": {
+    "start": "node bin/analyze-tracking.js",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/fliskdata/analyze-tracking.git"
+  },
+  "author": "Sameen Karim",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/fliskdata/analyze-tracking/issues"
+  },
+  "homepage": "https://github.com/fliskdata/analyze-tracking#readme",
+  "dependencies": {
+    "@typescript-eslint/parser": "^8.1.0",
+    "acorn": "^8.12.1",
+    "acorn-walk": "^8.3.3",
+    "js-yaml": "^4.1.0",
+    "typescript": "^5.5.4"
+  }
+}

package/schema.json

@@ -0,0 +1,106 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "@flisk/analyze-tracking output schema",
+  "type": "object",
+  "properties": {
+    "version": {
+      "type": "number",
+      "enum": [
+        1.0
+      ],
+      "description": "Version of the schema"
+    },
+    "events": {
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9_-]+$": {
+          "type": "object",
+          "properties": {
+            "sources": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "repository": {
+                    "type": "string",
+                    "description": "Repository URL or name"
+                  },
+                  "path": {
+                    "type": "string",
+                    "description": "Relative path to the file where the event is tracked"
+                  },
+                  "line": {
+                    "type": "integer",
+                    "description": "Line number in the file where the event is tracked"
+                  },
+                  "function": {
+                    "type": "string",
+                    "description": "Name of the function where the event is tracked"
+                  }
+                },
+                "required": [
+                  "repository",
+                  "path",
+                  "line",
+                  "function"
+                ]
+              }
+            },
+            "destinations": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "enum": [
+                  "googleanalytics",
+                  "segment",
+                  "mixpanel",
+                  "amplitude",
+                  "rudderstack",
+                  "mparticle",
+                  "snowplow",
+                  "unknown"
+                ],
+                "description": "Destination analytics platform"
+              }
+            },
+            "properties": {
+              "type": "object",
+              "patternProperties": {
+                "^[a-zA-Z0-9_-]+$": {
+                  "type": "object",
+                  "properties": {
+                    "type": {
+                      "type": "string",
+                      "description": "Data type of the property (e.g., string, number)"
+                    },
+                    "required": {
+                      "type": "boolean",
+                      "description": "Whether this property is required"
+                    },
+                    "description": {
+                      "type": "string",
+                      "description": "Description of the property"
+                    }
+                  },
+                  "required": [
+                    "type"
+                  ]
+                }
+              }
+            }
+          },
+          "required": [
+            "sources",
+            "destinations",
+            "properties"
+          ]
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+  "required": [
+    "version",
+    "events"
+  ]
+}

package/src/analyze/analyzeJsFile.js

@@ -0,0 +1,52 @@
+const fs = require('fs');
+const acorn = require('acorn');
+const walk = require('acorn-walk');
+const { detectSourceJs, findWrappingFunctionJs, extractJsProperties } = require('./helpers');
+
+function analyzeJsFile(filePath) {
+  const code = fs.readFileSync(filePath, 'utf8');
+  const ast = acorn.parse(code, { ecmaVersion: 'latest', sourceType: 'module', locations: true });
+  const events = [];
+
+  walk.ancestor(ast, {
+    CallExpression(node, ancestors) {
+      const source = detectSourceJs(node);
+      if (source === 'unknown') return;
+
+      let eventName = null;
+      let propertiesNode = null;
+
+      if (source === 'googleanalytics' && node.arguments.length >= 3) {
+        eventName = node.arguments[1]?.value || null;
+        propertiesNode = node.arguments[2];
+      } else if (source === 'snowplow' && node.arguments.length >= 2) {
+        const actionProperty = node.arguments[1].properties.find(prop => prop.key.name === 'action');
+        eventName = actionProperty ? actionProperty.value.value : null;
+        propertiesNode = node.arguments[1];
+      } else if (node.arguments.length >= 2) {
+        eventName = node.arguments[0]?.value || null;
+        propertiesNode = node.arguments[1];
+      }
+
+      const line = node.loc.start.line;
+      const functionName = findWrappingFunctionJs(ancestors[ancestors.length - 2]);
+
+      if (eventName && propertiesNode && propertiesNode.type === 'ObjectExpression') {
+        const properties = extractJsProperties(propertiesNode);
+
+        events.push({
+          eventName,
+          source,
+          properties,
+          filePath,
+          line,
+          functionName
+        });
+      }
+    },
+  });
+
+  return events;
+}
+
+module.exports = { analyzeJsFile };

package/src/analyze/analyzeTsFile.js

@@ -0,0 +1,52 @@
+const ts = require('typescript');
+const { detectSourceTs, findWrappingFunctionTs, extractProperties } = require('./helpers');
+
+function analyzeTsFile(filePath, program) {
+  const sourceFile = program.getSourceFile(filePath);
+  const checker = program.getTypeChecker();
+  const events = [];
+
+  function visit(node) {
+    if (ts.isCallExpression(node)) {
+      const source = detectSourceTs(node);
+      if (source === 'unknown') return;
+
+      let eventName = null;
+      let propertiesNode = null;
+
+      if (source === 'googleanalytics' && node.arguments.length >= 3) {
+        eventName = node.arguments[1]?.text || null;
+        propertiesNode = node.arguments[2];
+      } else if (source === 'snowplow' && node.arguments.length >= 2) {
+        const actionProperty = node.arguments[1].properties.find(prop => prop.name.escapedText === 'action');
+        eventName = actionProperty ? actionProperty.initializer.text : null;
+        propertiesNode = node.arguments[1];
+      } else if (node.arguments.length >= 2) {
+        eventName = node.arguments[0]?.text || null;
+        propertiesNode = node.arguments[1];
+      }
+
+      const line = sourceFile.getLineAndCharacterOfPosition(node.getStart()).line + 1;
+      const functionName = findWrappingFunctionTs(node);
+
+      if (eventName && propertiesNode && ts.isObjectLiteralExpression(propertiesNode)) {
+        const properties = extractProperties(checker, propertiesNode);
+        events.push({
+          eventName,
+          source,
+          properties,
+          filePath,
+          line,
+          functionName
+        });
+      }
+    }
+    ts.forEachChild(node, visit);
+  }
+
+  ts.forEachChild(sourceFile, visit);
+
+  return events;
+}
+
+module.exports = { analyzeTsFile };

package/src/analyze/helpers.js

@@ -0,0 +1,139 @@
+const ts = require('typescript');
+
+function detectSourceJs(node) {
+  if (!node.callee) return 'unknown';
+
+  if (node.callee.type === 'Identifier' && node.callee.name === 'gtag') {
+    return 'googleanalytics';
+  } else if (node.callee.type === 'MemberExpression') {
+    const objectName = node.callee.object.name;
+    const methodName = node.callee.property.name;
+
+    if (objectName === 'analytics' && methodName === 'track') return 'segment';
+    if (objectName === 'mixpanel' && methodName === 'track') return 'mixpanel';
+    if (objectName === 'amplitude' && methodName === 'logEvent') return 'amplitude';
+    if (objectName === 'rudderanalytics' && methodName === 'track') return 'rudderstack';
+    if (objectName === 'mParticle' && methodName === 'logEvent') return 'mparticle';
+  } else if (node.callee.type === 'Identifier' && node.callee.name === 'snowplow') {
+    return 'snowplow';
+  }
+
+  return 'unknown';
+}
+
+function detectSourceTs(node) {
+  if (!node.expression) return 'unknown';
+
+  if (ts.isIdentifier(node.expression) && node.expression.escapedText === 'gtag') {
+    return 'googleanalytics';
+  } else if (ts.isPropertyAccessExpression(node.expression)) {
+    const objectName = node.expression.expression.escapedText;
+    const methodName = node.expression.name.escapedText;
+
+    if (objectName === 'analytics' && methodName === 'track') return 'segment';
+    if (objectName === 'mixpanel' && methodName === 'track') return 'mixpanel';
+    if (objectName === 'amplitude' && methodName === 'logEvent') return 'amplitude';
+    if (objectName === 'rudderanalytics' && methodName === 'track') return 'rudderstack';
+    if (objectName === 'mParticle' && methodName === 'logEvent') return 'mparticle';
+  } else if (ts.isIdentifier(node.expression) && node.expression.escapedText === 'snowplow') {
+    return 'snowplow';
+  }
+
+  return 'unknown';
+}
+
+function findWrappingFunctionTs(node) {
+  let current = node;
+  while (current) {
+    if (ts.isFunctionDeclaration(current) || ts.isMethodDeclaration(current) || ts.isArrowFunction(current)) {
+      return current.name ? current.name.escapedText : 'anonymous';
+    }
+    current = current.parent;
+  }
+  return 'global';
+}
+
+function findWrappingFunctionJs(node) {
+  let current = node;
+  while (current) {
+    if (current.type === 'FunctionDeclaration' || current.type === 'FunctionExpression' || current.type === 'ArrowFunctionExpression') {
+      return current.id ? current.id.name : 'anonymous';
+    }
+    current = current.parent;
+  }
+  return 'global';
+}
+
+function extractJsProperties(node) {
+  const properties = {};
+
+  node.properties.forEach((prop) => {
+    const key = prop.key?.name || prop.key?.value;
+    if (key) {
+      let valueType = typeof prop.value.value;
+      if (prop.value.type === 'ObjectExpression') {
+        properties[key] = {
+          type: 'object',
+          properties: extractJsProperties(prop.value),
+        };
+      } else {
+        if (valueType === 'undefined') {
+          valueType = 'any';
+        } else if (valueType === 'object') {
+          valueType = 'any';
+        }
+        properties[key] = { type: valueType };
+      }
+    }
+  });
+
+  return properties;
+}
+
+function extractProperties(checker, node) {
+  const properties = {};
+
+  node.properties.forEach((prop) => {
+    const key = prop.name ? prop.name.text : prop.key.text || prop.key.value;
+    let valueType = 'any';
+
+    if (prop.initializer) {
+      if (ts.isObjectLiteralExpression(prop.initializer)) {
+        properties[key] = {
+          type: 'object',
+          properties: extractProperties(checker, prop.initializer),
+        };
+      } else if (ts.isArrayLiteralExpression(prop.initializer)) {
+        properties[key] = {
+          type: 'array',
+          items: {
+            type: getTypeOfNode(checker, prop.initializer.elements[0]) || 'any',
+          },
+        };
+      } else {
+        valueType = getTypeOfNode(checker, prop.initializer) || 'any';
+        properties[key] = { type: valueType };
+      }
+    } else if (prop.type) {
+      valueType = checker.typeToString(checker.getTypeFromTypeNode(prop.type)) || 'any';
+      properties[key] = { type: valueType };
+    }
+  });
+
+  return properties;
+}
+
+function getTypeOfNode(checker, node) {
+  const type = checker.getTypeAtLocation(node);
+  return checker.typeToString(type);
+}
+
+module.exports = {
+  detectSourceJs,
+  detectSourceTs,
+  findWrappingFunctionTs,
+  findWrappingFunctionJs,
+  extractJsProperties,
+  extractProperties,
+  getTypeOfNode,
+};

package/src/analyze/index.js

@@ -0,0 +1,58 @@
+const { analyzeJsFile } = require('./analyzeJsFile');
+const { analyzeTsFile } = require('./analyzeTsFile');
+const { getAllFiles } = require('../fileProcessor');
+const ts = require('typescript');
+const path = require('path');
+
+function analyzeDirectory(dirPath, repository) {
+  const files = getAllFiles(dirPath);
+  const allEvents = {};
+
+  const tsFiles = files.filter(file => file.endsWith('.ts'));
+  const program = ts.createProgram(tsFiles, {
+    target: ts.ScriptTarget.ESNext,
+    module: ts.ModuleKind.CommonJS,
+  });
+
+  files.forEach((file) => {
+    const isTsFile = file.endsWith('.ts');
+    const events = isTsFile ? analyzeTsFile(file, program) : analyzeJsFile(file);
+
+    events.forEach((event) => {
+      const relativeFilePath = path.relative(dirPath, event.filePath); // Calculate relative path
+
+      if (!allEvents[event.eventName]) {
+        allEvents[event.eventName] = {
+          sources: [{
+            repository: repository,
+            path: relativeFilePath,
+            line: event.line,
+            function: event.functionName
+          }],
+          destinations: [event.source],
+          properties: event.properties,
+        };
+      } else {
+        allEvents[event.eventName].sources.push({
+          repository: repository,
+          path: relativeFilePath,
+          line: event.line,
+          function: event.functionName
+        });
+
+        if (!allEvents[event.eventName].destinations.includes(event.source)) {
+          allEvents[event.eventName].destinations.push(event.source);
+        }
+
+        allEvents[event.eventName].properties = {
+          ...allEvents[event.eventName].properties,
+          ...event.properties,
+        };
+      }
+    });
+  });
+
+  return allEvents;
+}
+
+module.exports = { analyzeDirectory };

package/src/cli.js

@@ -0,0 +1,11 @@
+#!/usr/bin/env node
+
+const { run } = require('./index');
+const targetDir = process.argv[2];
+
+if (!targetDir) {
+  console.error('Please provide the path to the repository.');
+  process.exit(1);
+}
+
+run(path.resolve(targetDir));

package/src/fileProcessor.js

@@ -0,0 +1,19 @@
+const fs = require('fs');
+const path = require('path');
+
+function getAllFiles(dirPath, arrayOfFiles = []) {
+  const files = fs.readdirSync(dirPath);
+
+  files.forEach((file) => {
+    const fullPath = path.join(dirPath, file);
+    if (fs.statSync(fullPath).isDirectory()) {
+      arrayOfFiles = getAllFiles(fullPath, arrayOfFiles);
+    } else if (file.endsWith('.js') || file.endsWith('.ts')) {
+      arrayOfFiles.push(fullPath);
+    }
+  });
+
+  return arrayOfFiles;
+}
+
+module.exports = { getAllFiles };

package/src/index.js

@@ -0,0 +1,9 @@
+const { analyzeDirectory } = require('./analyze');
+const { generateYamlSchema } = require('./yamlGenerator');
+
+function run(targetDir, repository, outputPath) {
+  const events = analyzeDirectory(targetDir, repository);
+  generateYamlSchema(events, outputPath);
+}
+
+module.exports = { run };

package/src/yamlGenerator.js

@@ -0,0 +1,15 @@
+const fs = require('fs');
+const yaml = require('js-yaml');
+
+function generateYamlSchema(events, outputPath) {
+  const schema = {
+    version: 1.0,
+    events,
+  };
+
+  const yamlOutput = yaml.dump(schema, { noRefs: true });
+  fs.writeFileSync(outputPath, yamlOutput, 'utf8');
+  console.log(`Tracking schema YAML file generated: ${outputPath}`);
+}
+
+module.exports = { generateYamlSchema };