@appthreat/atom-parsetools 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Team AppThreat
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,78 @@
+# Introduction
+
+This package hosts a collection of parsing tools that complement the `@appthreat/atom` project. These tools offer parsing and analysis-related functionalities such as generating AST and semantics information in JSON format. The full list of tools and bin commands exposed by this package is below:
+
+- astgen - Generates AST for JavaScript and TypeScript projects in JSON format
+- phpastgen - Generates AST for PHP projects using `php-parse` command from `nikic/php-parser`
+- rbastgen - Generates AST for Ruby projects using the AppThreat's `ruby_ast_gen` gem
+- scalasem - Generates a custom semantics slice for Scala Projects by utilising scalac command.
+
+## Command usages
+
+### astgen
+
+```text
+node astgen.js -h
+Options:
+  -i, --src      Source directory                                 [default: "."]
+  -o, --output   Output directory for generated AST JSON files
+                                                            [default: "ast_out"]
+  -t, --type     Project type. Default auto-detect
+  -r, --recurse  Recurse mode suitable for mono-repos  [boolean] [default: true]
+      --tsTypes  Generate type mappings using the TypeScript Compiler API
+                                                       [boolean] [default: true]
+      --version  Show version number                                   [boolean]
+  -h             Show help                                             [boolean]
+```
+
+### phpastgen
+
+```text
+node phpastgen.js --help
+
+Usage: phpastgen [operations] file1.php [file2.php ...]
+   or: phpastgen [operations] "<?php code"
+Turn PHP source code into an abstract syntax tree.
+
+Operations is a list of the following options (--dump by default):
+
+    -d, --dump              Dump nodes using NodeDumper
+    -p, --pretty-print      Pretty print file using PrettyPrinter\Standard
+    -j, --json-dump         Print json_encode() result
+        --var-dump          var_dump() nodes (for exact structure)
+    -N, --resolve-names     Resolve names using NodeVisitor\NameResolver
+    -c, --with-column-info  Show column numbers for errors (if available)
+    -P, --with-positions    Show positions in node dumps
+    -r, --with-recovery     Use parsing with error recovery
+    -h, --help              Display this page
+
+```
+
+### rbastgen
+
+```text
+node rbastgen.js --help
+Usage:
+  -i, --input      The input file or directory (required)
+  -o, --output     The output directory (default: '.ast')
+  -e, --exclude    The exclusion regex (default: '^(tests?|vendor|spec)')
+  -d, --debug      Enable debug logging
+      --version    Print the version
+      --help       Print usage
+```
+
+### scalasem
+
+```text
+scalasem <directory> <slices_file>
+```
+
+Example:
+
+```shell
+scalasem $(pwd) slices.json
+```
+
+## License
+
+MIT

package/astgen.js

@@ -0,0 +1,421 @@
+#!/usr/bin/env node
+
+import { join, dirname } from "path";
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { parse } from "@babel/parser";
+import tsc from "typescript";
+import {
+  readFileSync,
+  mkdirSync,
+  writeFileSync,
+  accessSync,
+  constants,
+  existsSync
+} from "fs";
+import { getAllFiles } from "@appthreat/atom-common";
+
+const ASTGEN_VERSION = "4.0.0";
+
+const babelParserOptions = {
+  sourceType: "unambiguous",
+  allowImportExportEverywhere: true,
+  allowAwaitOutsideFunction: true,
+  allowNewTargetOutsideFunction: true,
+  allowReturnOutsideFunction: true,
+  allowSuperOutsideMethod: true,
+  allowUndeclaredExports: true,
+  errorRecovery: true,
+  plugins: [
+    "optionalChaining",
+    "classProperties",
+    "decorators-legacy",
+    "exportDefaultFrom",
+    "doExpressions",
+    "numericSeparator",
+    "dynamicImport",
+    "jsx",
+    "typescript"
+  ]
+};
+
+const babelSafeParserOptions = {
+  sourceType: "module",
+  allowImportExportEverywhere: true,
+  allowAwaitOutsideFunction: true,
+  allowReturnOutsideFunction: true,
+  errorRecovery: true,
+  plugins: [
+    "optionalChaining",
+    "classProperties",
+    "decorators-legacy",
+    "exportDefaultFrom",
+    "doExpressions",
+    "numericSeparator",
+    "dynamicImport",
+    "typescript"
+  ]
+};
+
+/**
+ * Return paths to all (j|tsx?) files.
+ */
+const getAllSrcJSAndTSFiles = (src) =>
+  Promise.all([
+    getAllFiles(
+      src,
+      undefined,
+      undefined,
+      undefined,
+      new RegExp("\\.(js|jsx|cjs|mjs|ts|tsx|vue|svelte)$")
+    )
+  ]);
+
+/**
+ * Convert a single JS/TS file to AST
+ */
+const fileToJsAst = (file) => {
+  if (file.endsWith(".vue") || file.endsWith(".svelte")) {
+    return toVueAst(file);
+  }
+  return codeToJsAst(readFileSync(file, "utf-8"));
+};
+
+/**
+ * Convert a single JS/TS code snippet to AST
+ */
+const codeToJsAst = (code) => {
+  try {
+    return parse(code, babelParserOptions);
+  } catch {
+    return parse(code, babelSafeParserOptions);
+  }
+};
+
+const vueCleaningRegex = /<\/*script.*>|<style[\s\S]*style>|<\/*br>/gi;
+const vueTemplateRegex = /(<template.*>)([\s\S]*)(<\/template>)/gi;
+const vueCommentRegex = /<!--[\s\S]*?-->/gi;
+const vueBindRegex = /(:\[)([\s\S]*?)(\])/gi;
+const vuePropRegex = /\s([.:@])([a-zA-Z]*?=)/gi;
+const vueOpenImgTag = /(<img)((?!>)[\s\S]+?)( [^/]>)/gi;
+
+const TSC_FLAGS =
+  tsc.TypeFormatFlags.NoTruncation |
+  tsc.TypeFormatFlags.InTypeAlias |
+  tsc.TypeFormatFlags.WriteArrayAsGenericType |
+  tsc.TypeFormatFlags.GenerateNamesForShadowedTypeParams |
+  tsc.TypeFormatFlags.WriteTypeArgumentsOfSignature |
+  tsc.TypeFormatFlags.UseFullyQualifiedType |
+  tsc.TypeFormatFlags.NoTypeReduction;
+
+/**
+ * Convert a single vue file to AST
+ */
+const toVueAst = (file) => {
+  const code = readFileSync(file, "utf-8");
+  const cleanedCode = code
+    .replace(vueCommentRegex, function (match) {
+      return match.replaceAll(/\S/g, " ");
+    })
+    .replace(vueCleaningRegex, function (match) {
+      return match.replaceAll(/\S/g, " ").substring(1) + ";";
+    })
+    .replace(vueBindRegex, function (match, grA, grB, grC) {
+      return grA.replaceAll(/\S/g, " ") + grB + grC.replaceAll(/\S/g, " ");
+    })
+    .replace(vuePropRegex, function (match, grA, grB) {
+      return " " + grA.replace(/[.:@]/g, " ") + grB.replaceAll(".", "-");
+    })
+    .replace(vueOpenImgTag, function (match, grA, grB, grC) {
+      return grA + grB + grC.replace(" >", "/>");
+    })
+    .replace(vueTemplateRegex, function (match, grA, grB, grC) {
+      return grA + grB.replaceAll("{{", "{ ").replaceAll("}}", " }") + grC;
+    });
+  return codeToJsAst(cleanedCode);
+};
+
+function createTsc(srcFiles) {
+  try {
+    const program = tsc.createProgram(srcFiles, {
+      target: tsc.ScriptTarget.ES2020,
+      module: tsc.ModuleKind.CommonJS,
+      allowImportingTsExtensions: false,
+      allowArbitraryExtensions: false,
+      allowSyntheticDefaultImports: true,
+      allowUmdGlobalAccess: true,
+      allowJs: true,
+      allowUnreachableCode: true,
+      allowUnusedLabels: true,
+      alwaysStrict: false,
+      emitDecoratorMetadata: true,
+      exactOptionalPropertyTypes: true,
+      experimentalDecorators: false,
+      ignoreDeprecations: true,
+      noStrictGenericChecks: true,
+      noUncheckedIndexedAccess: false,
+      noPropertyAccessFromIndexSignature: false,
+      removeComments: true
+    });
+    const typeChecker = program.getTypeChecker();
+    const seenTypes = new Map();
+
+    const safeTypeToString = (node) => {
+      try {
+        return typeChecker.typeToString(node, TSC_FLAGS);
+      } catch (err) {
+        return "any";
+      }
+    };
+
+    const safeTypeWithContextToString = (node, context) => {
+      try {
+        return typeChecker.typeToString(node, context, TSC_FLAGS);
+      } catch (err) {
+        return "any";
+      }
+    };
+
+    const addType = (node) => {
+      let typeStr;
+      if (
+        tsc.isSetAccessor(node) ||
+        tsc.isGetAccessor(node) ||
+        tsc.isGetAccessorDeclaration(node) ||
+        tsc.isCallSignatureDeclaration(node) ||
+        tsc.isIndexSignatureDeclaration(node) ||
+        tsc.isClassStaticBlockDeclaration(node) ||
+        tsc.isConstructSignatureDeclaration(node) ||
+        tsc.isMethodDeclaration(node) ||
+        tsc.isFunctionDeclaration(node) ||
+        tsc.isConstructorDeclaration(node) ||
+        tsc.isTypeAliasDeclaration(node) ||
+        tsc.isEnumDeclaration(node) ||
+        tsc.isNamespaceExportDeclaration(node) ||
+        tsc.isImportEqualsDeclaration(node)
+      ) {
+        const signature = typeChecker.getSignatureFromDeclaration(node);
+        const returnType = typeChecker.getReturnTypeOfSignature(signature);
+        typeStr = safeTypeToString(returnType);
+      } else if (tsc.isFunctionLike(node)) {
+        const funcType = typeChecker.getTypeAtLocation(node);
+        const funcSignature = typeChecker.getSignaturesOfType(
+          funcType,
+          tsc.SignatureKind.Call
+        )[0];
+        if (funcSignature) {
+          typeStr = safeTypeToString(funcSignature.getReturnType());
+        } else {
+          typeStr = safeTypeWithContextToString(
+            typeChecker.getTypeAtLocation(node),
+            node
+          );
+        }
+      } else {
+        typeStr = safeTypeWithContextToString(
+          typeChecker.getTypeAtLocation(node),
+          node
+        );
+      }
+      if (!["any", "unknown", "any[]", "unknown[]"].includes(typeStr)) {
+        seenTypes.set(node.getStart(), typeStr);
+      }
+      tsc.forEachChild(node, addType);
+    };
+
+    return {
+      program: program,
+      typeChecker: typeChecker,
+      addType: addType,
+      seenTypes: seenTypes
+    };
+  } catch (err) {
+    console.warn("Retrieving types", err.message);
+    return undefined;
+  }
+}
+
+/**
+ * Generate AST for JavaScript or TypeScript
+ */
+const createJSAst = async (options) => {
+  try {
+    const promiseMap = await getAllSrcJSAndTSFiles(options.src);
+    const srcFiles = promiseMap.flatMap((d) => d);
+    let ts;
+    if (options.tsTypes) {
+      ts = createTsc(srcFiles);
+    }
+    for (const file of srcFiles) {
+      try {
+        const ast = fileToJsAst(file);
+        writeAstFile(file, ast, options);
+      } catch (err) {
+        console.error(file, err.message);
+      }
+      if (ts) {
+        try {
+          const tsAst = ts.program.getSourceFile(file);
+          tsc.forEachChild(tsAst, ts.addType);
+          writeTypesFile(file, ts.seenTypes, options);
+          ts.seenTypes.clear();
+        } catch (err) {
+          console.warn("Retrieving types", file, ":", err.message);
+        }
+      }
+    }
+  } catch (err) {
+    console.error(err);
+  }
+};
+
+/**
+ * Generate AST for .vue files
+ */
+const createVueAst = async (options) => {
+  const srcFiles = await getAllFiles(options.src, ".vue");
+  for (const file of srcFiles) {
+    try {
+      const ast = toVueAst(file);
+      if (ast) {
+        writeAstFile(file, ast, options);
+      }
+    } catch (err) {
+      console.error(file, err.message);
+    }
+  }
+};
+
+/**
+ * Deal with cyclic reference in json
+ */
+const getCircularReplacer = () => {
+  const seen = new WeakSet();
+  return (key, value) => {
+    if (typeof value === "object" && value !== null) {
+      if (seen.has(value)) {
+        return;
+      }
+      seen.add(value);
+    }
+    return value;
+  };
+};
+
+/**
+ * Write AST data to a json file
+ */
+const writeAstFile = (file, ast, options) => {
+  const relativePath = file.replace(new RegExp("^" + options.src + "/"), "");
+  const outAstFile = join(options.output, relativePath + ".json");
+  const data = {
+    fullName: file,
+    relativeName: relativePath,
+    ast: ast
+  };
+  mkdirSync(dirname(outAstFile), { recursive: true });
+  writeFileSync(outAstFile, JSON.stringify(data, getCircularReplacer(), "  "));
+  console.log("Converted AST for", relativePath, "to", outAstFile);
+};
+
+const writeTypesFile = (file, seenTypes, options) => {
+  const relativePath = file.replace(new RegExp("^" + options.src + "/"), "");
+  const outTypeFile = join(options.output, relativePath + ".typemap");
+  mkdirSync(dirname(outTypeFile), { recursive: true });
+  writeFileSync(
+    outTypeFile,
+    JSON.stringify(Object.fromEntries(seenTypes), undefined, "  ")
+  );
+  console.log("Converted types for", relativePath, "to", outTypeFile);
+};
+
+const createXAst = async (options) => {
+  const src_dir = options.src;
+  try {
+    accessSync(src_dir, constants.R_OK);
+  } catch (err) {
+    console.error(src_dir, "is invalid");
+    process.exit(1);
+  }
+  if (
+    existsSync(join(src_dir, "package.json")) ||
+    existsSync(join(src_dir, "rush.json"))
+  ) {
+    return await createJSAst(options);
+  }
+  console.error(src_dir, "unknown project type");
+  process.exit(1);
+};
+
+/**
+ * Method to start the ast generation process
+ *
+ * @args options CLI arguments
+ */
+const start = async (options) => {
+  let { type } = options;
+  if (!type) {
+    type = "";
+  }
+  type = type.toLowerCase();
+  switch (type) {
+    case "nodejs":
+    case "js":
+    case "javascript":
+    case "typescript":
+    case "ts":
+      return await createJSAst(options);
+    case "vue":
+      return await createVueAst(options);
+    default:
+      return await createXAst(options);
+  }
+};
+
+async function main(argvs) {
+  const args = yargs(hideBin(argvs))
+    .option("src", {
+      alias: "i",
+      default: ".",
+      description: "Source directory"
+    })
+    .option("output", {
+      alias: "o",
+      default: "ast_out",
+      description: "Output directory for generated AST json files"
+    })
+    .option("type", {
+      alias: "t",
+      description: "Project type. Default auto-detect"
+    })
+    .option("recurse", {
+      alias: "r",
+      default: true,
+      type: "boolean",
+      description: "Recurse mode suitable for mono-repos"
+    })
+    .option("tsTypes", {
+      default: true,
+      type: "boolean",
+      description: "Generate type mappings using the Typescript Compiler API"
+    })
+    .version(ASTGEN_VERSION)
+    .help("h").argv;
+
+  if (args.version) {
+    console.log(ASTGEN_VERSION);
+    process.exit(0);
+  }
+
+  try {
+    if (args.output === "ast_out") {
+      args.output = join(args.src, args.output);
+    }
+    await start(args);
+  } catch (e) {
+    console.error(e);
+    process.exit(1);
+  }
+}
+
+main(process.argv);

package/index.js

@@ -0,0 +1 @@
+console.log("atom-parsetools installed successfully.");

package/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@appthreat/atom-parsetools",
+  "version": "1.0.0",
+  "description": "Parsing tools that complement the @appthreat/atom project.",
+  "main": "./index.js",
+  "type": "module",
+  "scripts": {
+    "pretty": "prettier --write *.mjs *.js --trailing-comma=none",
+    "lint": "eslint *.mjs *.js"
+  },
+  "dependencies": {
+    "@appthreat/atom-common": "^1.0.0",
+    "@babel/parser": "^7.27.1",
+    "typescript": "^5.8.3",
+    "yargs": "^17.7.2"
+  },
+  "devDependencies": {
+    "eslint": "8.57.0"
+  },
+  "bin": {
+    "astgen": "astgen.js",
+    "phpastgen": "phpastgen.js",
+    "rbastgen": "rbastgen.js",
+    "scalasem": "scalasem.js"
+  },
+  "engines": {
+    "node": ">=16.0.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/AppThreat/atom.git"
+  },
+  "keywords": [
+    "atom",
+    "commands",
+    "parse",
+    "astgen",
+    "code",
+    "analysis"
+  ],
+  "author": "Team AppThreat <cloud@appthreat.com>",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/AppThreat/atom/issues"
+  },
+  "homepage": "https://github.com/AppThreat/atom#readme",
+  "files": [
+    "*.js",
+    "plugins/",
+    "LICENSE"
+  ],
+  "packageManager": "npm"
+}

package/phpastgen.js

@@ -0,0 +1,33 @@
+#!/usr/bin/env node
+
+import { dirname, join } from "node:path";
+import { spawnSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { detectPhp } from "@appthreat/atom-common";
+
+let url = import.meta.url;
+if (!url.startsWith("file://")) {
+  url = new URL(`file://${import.meta.url}`).toString();
+}
+const dirName = import.meta ? dirname(fileURLToPath(url)) : __dirname;
+export const PLUGINS_HOME = join(dirName, "plugins");
+const PHP_PARSER_BIN =
+  process.env.PHP_PARSER_BIN || join(PLUGINS_HOME, "bin", "php-parse");
+
+function main(argvs) {
+  if (!detectPhp()) {
+    console.warn("PHP is not installed!");
+    return false;
+  }
+  const cwd = process.env.ATOM_CWD || process.cwd();
+  argvs.splice(0, 1, PHP_PARSER_BIN);
+  spawnSync(process.env.PHP_CMD || "php", argvs, {
+    encoding: "utf-8",
+    cwd,
+    stdio: "inherit",
+    stderr: "inherit",
+    env: process.env,
+    timeout: process.env.ATOM_TIMEOUT || process.env.ASTGEN_TIMEOUT
+  });
+}
+main(process.argv.slice(2));

package/plugins/autoload.php

@@ -0,0 +1,22 @@
+<?php
+
+// autoload.php @generated by Composer
+
+if (PHP_VERSION_ID < 50600) {
+    if (!headers_sent()) {
+        header('HTTP/1.1 500 Internal Server Error');
+    }
+    $err = 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL;
+    if (!ini_get('display_errors')) {
+        if (PHP_SAPI === 'cli' || PHP_SAPI === 'phpdbg') {
+            fwrite(STDERR, $err);
+        } elseif (!headers_sent()) {
+            echo $err;
+        }
+    }
+    throw new RuntimeException($err);
+}
+
+require_once __DIR__ . '/composer/autoload_real.php';
+
+return ComposerAutoloaderInitf8ec96d581b5c88a936d13783175ff17::getLoader();

package/plugins/bin/bundler

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'bundler' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+$:.unshift File.expand_path "../rubyastgen/bundle", __dir__
+
+require "bundler/setup"
+load File.expand_path "../../../../../../../../../../../opt/hostedtoolcache/Ruby/3.4.3/x64/lib/ruby/gems/3.4.0/gems/bundler-2.6.7/exe/bundler", __dir__