@appthreat/atom 2.1.13 → 2.1.14

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@appthreat/atom",
-  "version": "2.1.13",
+  "version": "2.1.14",
   "description": "Create atom (⚛) representation for your application, packages and libraries",
   "exports": "./index.js",
   "type": "module",
@@ -20,7 +20,8 @@
     "atom": "index.js",
     "astgen": "astgen.js",
     "phpastgen": "phpastgen.js",
-    "rbastgen": "rbastgen.js"
+    "rbastgen": "rbastgen.js",
+    "scalasem": "scalasem.js"
   },
   "engines": {
     "node": ">=16.0.0"

package/plugins/bin/atom

@@ -360,7 +360,7 @@ declare -r lib_dir="$(realpath "${app_home}/../lib")"
 declare -a app_mainclass=('io.appthreat.atom.Atom')
 
 declare -r script_conf_file="${app_home}/../conf/application.ini"
-declare -r app_classpath="$lib_dir/io.appthreat.atom-2.1.13-classpath.jar"
+declare -r app_classpath="$lib_dir/io.appthreat.atom-2.1.14-classpath.jar"
 
 # java_cmd is overrode in process_args when -java-home is used
 declare java_cmd=$(get_java_cmd)

package/plugins/bin/atom.bat

@@ -40,7 +40,7 @@ rem "-J" is stripped, "-D" is left as is, and everything is appended to JAVA_OPT
 set _JAVA_PARAMS=
 set _APP_ARGS=
 
-set "APP_CLASSPATH=%APP_LIB_DIR%\io.appthreat.atom-2.1.13-classpath.jar"
+set "APP_CLASSPATH=%APP_LIB_DIR%\io.appthreat.atom-2.1.14-classpath.jar"
 set "APP_MAIN_CLASS=io.appthreat.atom.Atom"
 set "SCRIPT_CONF_FILE=%APP_HOME%\conf\application.ini"
 

package/plugins/composer/installed.php

@@ -1,9 +1,9 @@
 <?php return array(
     'root' => array(
         'name' => '__root__',
-        'pretty_version' => 'v2.1.13',
-        'version' => '2.1.13.0',
-        'reference' => 'cbf0caab75d535ac02719b3c97e9978955da5721',
+        'pretty_version' => 'v2.1.14',
+        'version' => '2.1.14.0',
+        'reference' => 'd1252e282dc3ddc3b51914f8e21fe48fd40d01ba',
         'type' => 'library',
         'install_path' => __DIR__ . '/../../',
         'aliases' => array(),
@@ -11,9 +11,9 @@
     ),
     'versions' => array(
         '__root__' => array(
-            'pretty_version' => 'v2.1.13',
-            'version' => '2.1.13.0',
-            'reference' => 'cbf0caab75d535ac02719b3c97e9978955da5721',
+            'pretty_version' => 'v2.1.14',
+            'version' => '2.1.14.0',
+            'reference' => 'd1252e282dc3ddc3b51914f8e21fe48fd40d01ba',
             'type' => 'library',
             'install_path' => __DIR__ . '/../../',
             'aliases' => array(),

package/plugins/rubyastgen/bundle/ruby/3.4.0/bundler/gems/ruby_ast_gen-d8b6ffa44317/ruby_ast_gen.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib".freeze]
   s.authors = ["David Baker Effendi".freeze, "Andrei Dreyer".freeze, "Team AppThreat".freeze]
   s.bindir = "exe".freeze
-  s.date = "2025-03-05"
+  s.date = "2025-03-10"
   s.description = "A Ruby parser than dumps the AST as JSON output for atom `rubysrc2cpg` frontend".freeze
   s.email = ["dave@whirlylabs.com".freeze, "andrei@whirlylabs.com".freeze, "hello@appthreat.com".freeze]
   s.executables = ["ruby_ast_gen".freeze]

package/plugins/rubyastgen/bundle/ruby/3.4.0/extensions/x86_64-linux/3.4.0/racc-1.8.1/gem_make.out

@@ -3,16 +3,16 @@ current directory: /home/runner/work/atom/atom/wrapper/nodejs/plugins/rubyastgen
 creating Makefile
 
 current directory: /home/runner/work/atom/atom/wrapper/nodejs/plugins/rubyastgen/bundle/ruby/3.4.0/gems/racc-1.8.1/ext/racc/cparse
-make DESTDIR\= sitearchdir\=./.gem.20250305-2774-qrpo6g sitelibdir\=./.gem.20250305-2774-qrpo6g clean
+make DESTDIR\= sitearchdir\=./.gem.20250310-2778-ho21ul sitelibdir\=./.gem.20250310-2778-ho21ul clean
 
 current directory: /home/runner/work/atom/atom/wrapper/nodejs/plugins/rubyastgen/bundle/ruby/3.4.0/gems/racc-1.8.1/ext/racc/cparse
-make DESTDIR\= sitearchdir\=./.gem.20250305-2774-qrpo6g sitelibdir\=./.gem.20250305-2774-qrpo6g
+make DESTDIR\= sitearchdir\=./.gem.20250310-2778-ho21ul sitelibdir\=./.gem.20250310-2778-ho21ul
 compiling cparse.c
 linking shared-object racc/cparse.so
 
 current directory: /home/runner/work/atom/atom/wrapper/nodejs/plugins/rubyastgen/bundle/ruby/3.4.0/gems/racc-1.8.1/ext/racc/cparse
-make DESTDIR\= sitearchdir\=./.gem.20250305-2774-qrpo6g sitelibdir\=./.gem.20250305-2774-qrpo6g install
-/usr/bin/install -c -m 0755 cparse.so ./.gem.20250305-2774-qrpo6g/racc
+make DESTDIR\= sitearchdir\=./.gem.20250310-2778-ho21ul sitelibdir\=./.gem.20250310-2778-ho21ul install
+/usr/bin/install -c -m 0755 cparse.so ./.gem.20250310-2778-ho21ul/racc
 
 current directory: /home/runner/work/atom/atom/wrapper/nodejs/plugins/rubyastgen/bundle/ruby/3.4.0/gems/racc-1.8.1/ext/racc/cparse
-make DESTDIR\= sitearchdir\=./.gem.20250305-2774-qrpo6g sitelibdir\=./.gem.20250305-2774-qrpo6g clean
+make DESTDIR\= sitearchdir\=./.gem.20250310-2778-ho21ul sitelibdir\=./.gem.20250310-2778-ho21ul clean

package/scalasem.js

@@ -0,0 +1,365 @@
+#!/usr/bin/env node
+// Usage: scalasem $(pwd) slices.json
+import { tmpdir } from "node:os";
+import { basename, dirname, join, relative } from "node:path";
+import { spawnSync } from "node:child_process";
+import { detectScala, detectScalac, getAllFiles } from "./utils.mjs";
+import process from "node:process";
+import {
+  existsSync,
+  mkdtempSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync
+} from "node:fs";
+
+function main(argvs) {
+  if (!detectScala() && !detectScalac()) {
+    console.warn("Scala is not installed!");
+    return false;
+  }
+  let configFiles = getAllFiles(argvs[0], "routes");
+  configFiles = configFiles.concat(getAllFiles(argvs[0], ".conf"));
+  let tastyFiles = getAllFiles(argvs[0], ".tasty");
+  if (!tastyFiles.length) {
+    let buildTool = "sbt";
+    const millFiles = getAllFiles(argvs[0], "build.mill");
+    if (millFiles.length) {
+      buildTool = "mill";
+    }
+    const cwd = process.env.ATOM_CWD || process.cwd();
+    let compileCommand =
+      process?.env[`${buildTool.toUpperCase()}_COMPILE_COMMAND`] || "compile";
+    if (process.env.SCALA_VERSION && buildTool === "sbt") {
+      compileCommand = `++${process.env.SCALA_VERSION} ${compileCommand}`;
+    } else {
+      // Detect crossScalaVersions
+      const scalaVersion = findScalaVersion(cwd);
+      if (scalaVersion) {
+        compileCommand = `++${scalaVersion} ${compileCommand}`;
+      }
+    }
+    console.log(`Executing '${buildTool} ${compileCommand}' in ${argvs[0]}`);
+    const result = spawnSync(buildTool, compileCommand.split(" "), {
+      encoding: "utf-8",
+      cwd,
+      stdio: "ignore",
+      stderr: "inherit",
+      env: process.env,
+      timeout: process.env.ATOM_TIMEOUT || process.env.ASTGEN_TIMEOUT
+    });
+    if (result.error || result.status !== 0) {
+      if (result.stderr) {
+        console.log(result.stderr);
+      }
+      return false;
+    }
+    tastyFiles = getAllFiles(argvs[0], ".tasty");
+    console.log(`Obtained ${tastyFiles.length} IR files after compilation.`);
+  }
+  const slicesFile =
+    argvs.length > 1 ? argvs[1] : join(argvs[0], "slices.json");
+  createSemanticSlices(tastyFiles, configFiles, slicesFile);
+}
+main(process.argv.slice(2));
+
+function findScalaVersion(cwd) {
+  let scalaVersion;
+  const buildSbtFile = join(cwd, "build.sbt");
+  if (existsSync(buildSbtFile)) {
+    const buildData = readFileSync(buildSbtFile, "utf-8");
+    for (let line of buildData.split("\n")) {
+      if (line.trim().includes("val ") && line.includes("scala")) {
+        const match = line.match(/"(3\.[^"]+)"/);
+        if (match) {
+          return match[1];
+        }
+      }
+      if (line.trim().includes("crossScalaVersions")) {
+        const crossVersions = line.split("crossScalaVersions").pop().trim();
+        if (crossVersions.includes("3.")) {
+          const match = crossVersions.match(/"(3\.[^"]+)"/);
+          if (match) {
+            return match[1];
+          }
+        }
+      }
+    }
+  }
+  return scalaVersion;
+}
+
+function createSemanticSlices(tastyFiles, configFiles, slicesFile) {
+  const outDir = mkdtempSync(join(tmpdir(), "scalasem-"));
+  const MAX_BUFFER =
+    Number.parseInt(process.env.ATOM_MAX_BUFFER) || 100 * 1024 * 1024;
+  const cwd = process.env.ATOM_CWD || process.cwd();
+  const slices = {};
+  slices.config = parseConfigFiles(configFiles);
+  for (const af of tastyFiles) {
+    const result = spawnSync(
+      process.env.SCALAC_CMD || "scalac",
+      ["-color:never", "-print-tasty", af],
+      {
+        encoding: "utf-8",
+        cwd,
+        env: process.env,
+        maxBuffer: MAX_BUFFER,
+        timeout: process.env.ATOM_TIMEOUT || process.env.ASTGEN_TIMEOUT
+      }
+    );
+    if (result.error || result.status !== 0) {
+      if (result.stderr) {
+        console.log(result.stderr);
+      }
+    }
+    if (result.stdout) {
+      let fileOutDir = join(outDir, relative(cwd, dirname(af)));
+      const scalaDir = relative(cwd, dirname(af)).replace(
+        new RegExp("target/scala-(.)*/classes"),
+        ""
+      );
+      if (fileOutDir.includes("classes")) {
+        fileOutDir = fileOutDir.replace(
+          new RegExp("target/scala-(.)*/classes"),
+          ""
+        );
+      }
+      mkdirSync(fileOutDir, { recursive: true });
+      const astFile = join(
+        fileOutDir,
+        basename(af).replace(".tasty", ".scala.ast")
+      );
+      const scalaFile = join(
+        scalaDir,
+        basename(af).replace(".tasty", ".scala")
+      );
+      writeFileSync(astFile, Buffer.from(result.stdout).toString());
+      const usages = parseTasty(astFile);
+      slices[usages.sourceFile || scalaFile] = usages;
+      rmSync(astFile);
+    }
+  }
+  const slicesJson = JSON.stringify(slices, null, null);
+  writeFileSync(slicesFile, slicesJson);
+  if (!Object.keys(slices).length) {
+    console.log("Empty slices file created.");
+  } else {
+    console.log(
+      `Slices file ${slicesFile} created successfully with ${
+        Object.keys(slices).length
+      } entries.`
+    );
+  }
+  if (outDir?.startsWith(tmpdir())) {
+    rmSync(outDir, { recursive: true });
+  }
+}
+
+function parseTasty(tastyAstFile) {
+  const astData = readFileSync(tastyAstFile, "utf-8");
+  let namesMode = false;
+  let treesMode = false;
+  let sourcePathsMode = false;
+  const literals = new Set();
+  const usedTypes = new Set();
+  const tags = new Set();
+  let sourceFile;
+  for (let line of astData.split("\n")) {
+    line = line.replace("\r", "").trim();
+    if (!line.length || line.startsWith("---")) {
+      continue;
+    }
+    if (line.startsWith("Names ") || line.startsWith("Names:")) {
+      namesMode = true;
+    }
+    if (line.startsWith("Trees ") || line.startsWith("Trees:")) {
+      namesMode = false;
+      treesMode = true;
+    }
+    if (line.startsWith("Positions ") || line.startsWith("positions:")) {
+      namesMode = false;
+      treesMode = false;
+    }
+    if (namesMode) {
+      // 3: api
+      if (line.includes(": ")) {
+        const literal = line.split(": ").pop().trim();
+        if (literal.length > 1) {
+          literals.add(literal);
+        }
+      }
+    }
+    if (treesMode && line.includes(" Signature(")) {
+      // 139:         SELECTin(12) 38 [<init>[Signed Signature(List(play.api.mvc.MessagesControllerComponents),play.api.mvc.MessagesAbstractController) @<init>]]
+      const signatureTypes = line
+        .split(" Signature(")
+        .pop()
+        .split(") ")[0]
+        .replaceAll("List(", "")
+        .replaceAll(")", "")
+        .split(",");
+      for (let sig of signatureTypes) {
+        sig = sig.trim();
+        if (
+          sig.length > 3 &&
+          !sig.startsWith("scala.") &&
+          !sig.startsWith("java.") &&
+          !sig.startsWith("javax.inject.")
+        ) {
+          usedTypes.add(sig);
+          if (sig.startsWith("play.api.")) {
+            tags.add("framework");
+          }
+          if (
+            sig.startsWith("play.api.data.Form") ||
+            sig.startsWith("play.api.mvc.Request") ||
+            sig.startsWith("play.twirl.api")
+          ) {
+            tags.add("framework-input");
+          }
+          if (
+            sig.startsWith("play.twirl.api.Html") ||
+            sig.startsWith("play.api.mvc.Result") ||
+            sig.startsWith("play.api.mvc.Action")
+          ) {
+            tags.add("framework-output");
+          }
+          if (
+            sig.startsWith("play.api.routing.") ||
+            sig.startsWith("play.core.routing") ||
+            sig.startsWith("router.RoutesPrefix")
+          ) {
+            tags.add("framework-route");
+          }
+          if (
+            sig.startsWith("slick.sql.") ||
+            sig.startsWith("play.db.") ||
+            sig.startsWith("slick.jdbc.")
+          ) {
+            tags.add("database");
+          }
+        }
+      }
+    }
+    if (line.includes("source paths:")) {
+      sourcePathsMode = true;
+    }
+    if (sourcePathsMode) {
+      if (line.includes(" [") && line.endsWith("]")) {
+        sourceFile = line.split(" [").pop().replace(/]/g, "");
+        sourcePathsMode = false;
+      } else if (line.includes(".scala") && line.includes(": ")) {
+        sourceFile = line.split(": ").pop().trim();
+        sourcePathsMode = false;
+      }
+    }
+    if (!namesMode && !treesMode && !sourcePathsMode) {
+      continue;
+    }
+  }
+  if (sourceFile?.includes("target")) {
+    tags.add("generated");
+  }
+  return {
+    sourceFile,
+    tags: Array.from(tags).sort(),
+    usedTypes: Array.from(usedTypes).sort(),
+    literals: Array.from(literals)
+  };
+}
+
+function parseConfigFiles(configFiles) {
+  const configMetadata = { routes: [] };
+  for (const aconfig of configFiles) {
+    if (aconfig.endsWith("routes")) {
+      const routes = parseRoutes(aconfig);
+      if (routes?.length) {
+        for (const aroute of routes) {
+          let duplicate = false;
+          for (const exisRoute of configMetadata.routes) {
+            if (
+              exisRoute.method === aroute.method &&
+              exisRoute.pattern === aroute.pattern
+            ) {
+              if (
+                exisRoute.controllerMethod &&
+                exisRoute.controllerMethod === aroute.controllerMethod
+              ) {
+                duplicate = true;
+                continue;
+              }
+            }
+          }
+          if (!duplicate) {
+            configMetadata["routes"].push(aroute);
+          }
+        }
+      }
+    }
+  }
+  if (configMetadata.routes.length) {
+    console.log("Found", configMetadata.routes.length, "routes.");
+  }
+  return configMetadata;
+}
+
+function parseRoutes(routesFile) {
+  const routes = [];
+  const routesData = readFileSync(routesFile, "utf-8");
+  for (let aline of routesData.split("\n")) {
+    aline = aline.replace("\r", "").trim();
+    if (aline.startsWith("#") || aline.startsWith("+")) {
+      continue;
+    }
+    const tmpA = aline.split(/\s+/);
+    if (tmpA.length < 2) {
+      continue;
+    }
+    // Ignore static assets
+    if (["/webjars"].includes(tmpA[1])) {
+      continue;
+    }
+    if (
+      [
+        "GET",
+        "PATCH",
+        "POST",
+        "OPTIONS",
+        "HEAD",
+        "DELETE",
+        "PUT",
+        "->"
+      ].includes(tmpA[0].toUpperCase())
+    ) {
+      let controllerMethod = tmpA.length > 2 ? tmpA[2] : undefined;
+      if (controllerMethod.includes("(")) {
+        controllerMethod = controllerMethod.split("(")[0];
+      }
+      // Exclude webjars
+      if (controllerMethod.startsWith("webjars.")) {
+        continue;
+      }
+      // Handle wildcards
+      if (tmpA[0] === "->") {
+        // We now need to parse a method called "routes" in the controllerMethod to identify the list of http methods
+        // Let's keep things simple for now
+        for (const m of ["GET", "PATCH", "POST", "DELETE", "PUT"]) {
+          routes.push({
+            method: m,
+            pattern: tmpA[1],
+            controllerMethod
+          });
+        }
+      } else {
+        routes.push({
+          method: tmpA[0],
+          pattern: tmpA[1],
+          controllerMethod
+        });
+      }
+    }
+  }
+  return routes;
+}

package/utils.mjs

@@ -28,7 +28,7 @@ const IGNORE_DIRS = process.env.ASTGEN_IGNORE_DIRS
 
 const IGNORE_FILE_PATTERN = new RegExp(
   process.env.ASTGEN_IGNORE_FILE_PATTERN ||
-    "(conf|config|test|spec|min|three|\\.d)\\.(js|ts|jsx|tsx)$",
+    "(test|spec|min|three|\\.d)\\.(js|ts|jsx|tsx)$",
   "i"
 );
 
@@ -69,7 +69,12 @@ export const getAllFiles = (dir, extn, files, result, regex) => {
         // ignore
       }
     } else {
-      if (regex.test(fileWithDir)) {
+      if (
+        regex.test(fileWithDir) ||
+        (extn &&
+          !extn.includes(".") &&
+          fileWithDir.toLowerCase().endsWith(extn.toLowerCase()))
+      ) {
         result.push(fileWithDir);
       }
     }
@@ -112,3 +117,23 @@ export const detectRuby = (versionNeeded) => {
   }
   return true;
 };
+
+export const detectScala = () => {
+  let result = spawnSync(process.env.SCALA_CMD || "scala", ["--version"], {
+    encoding: "utf-8"
+  });
+  if (result.status !== 0 || result.error) {
+    return false;
+  }
+  return true;
+};
+
+export const detectScalac = () => {
+  let result = spawnSync(process.env.SCALAC_CMD || "scalac", ["--version"], {
+    encoding: "utf-8"
+  });
+  if (result.status !== 0 || result.error) {
+    return false;
+  }
+  return true;
+};