@dxworks/insider 2.9.1

package/src/main/java/org/dxworks/insider/commands/VersionCommand.java

@@ -0,0 +1,34 @@
+package org.dxworks.insider.commands;
+
+import lombok.extern.slf4j.Slf4j;
+import org.dxworks.insider.InsiderFile;
+import org.dxworks.insider.configuration.InsiderConfiguration;
+
+import java.util.List;
+
+@Slf4j
+public class VersionCommand implements NoFilesCommand {
+    @Override
+    public boolean parse(List<String> args) {
+        if (args.size() != 1)
+            return false;
+
+        return VERSION.contains(args.get(0));
+    }
+
+    @Override
+    public void execute(List<InsiderFile> insiderFiles, List<String> args) {
+
+        System.out.println("Insider " + InsiderConfiguration.getInstance().getInsiderVersion());
+    }
+
+    @Override
+    public String usage() {
+        return "insider {-v | -version | --version | version}";
+    }
+
+    @Override
+    public String getName() {
+        return "version";
+    }
+}

package/src/main/java/org/dxworks/insider/configuration/InsiderConfiguration.java

@@ -0,0 +1,123 @@
+package org.dxworks.insider.configuration;
+
+import lombok.extern.slf4j.Slf4j;
+import org.dxworks.argumenthor.Argumenthor;
+import org.dxworks.argumenthor.config.ArgumenthorConfiguration;
+import org.dxworks.argumenthor.config.fields.impl.NumberField;
+import org.dxworks.argumenthor.config.fields.impl.StringField;
+import org.dxworks.argumenthor.config.fields.impl.StringListField;
+import org.dxworks.argumenthor.config.sources.impl.EnvSource;
+import org.dxworks.argumenthor.config.sources.impl.PropertiesSource;
+import org.dxworks.insider.technology.finder.LinguistService;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.dxworks.insider.constants.InsiderConstants.*;
+
+@Slf4j
+public class InsiderConfiguration {
+
+    private static InsiderConfiguration _instance = new InsiderConfiguration();
+    private static Argumenthor argumenthor;
+
+    private String insiderVersion = null;
+    private String projectID = null;
+    private String rootFolder = null;
+    private List<String> languages = null;
+    private String languagesFile = null;
+    private int depextMaxNamespaceLength = -1;
+
+    private InsiderConfiguration() {
+    }
+
+    public static InsiderConfiguration getInstance() {
+        return _instance;
+    }
+
+    public void load() {
+
+        System.out.println("Reading configuration");
+        readInsiderVersion();
+
+        ArgumenthorConfiguration argumenthorConfiguration = new ArgumenthorConfiguration(
+                new StringField(PROJECT_ID, null),
+                new StringField(ROOT_FOLDER, "."),
+                new StringListField(LANGUAGES, List.of(), ","),
+                new StringField(LINGUIST_FILE, DEFAULT_LINGUIST_FILE),
+                new NumberField(DEPEXT_MAX_NAMESPACE_LENGTH, DEFAULT_DEPEXT_MAX_NAMESPACE_LENGTH)
+        );
+
+        PropertiesSource propertiesSource = new PropertiesSource();
+        propertiesSource.setPath("config/insider-conf.properties");
+        argumenthorConfiguration.addSource(new EnvSource("INSIDER"));
+        argumenthorConfiguration.addSource(propertiesSource);
+        argumenthor = new Argumenthor(argumenthorConfiguration);
+
+        LinguistService.getInstance().initLinguist(getLanguagesFile());
+
+        System.out.println("Insider " + getInsiderVersion());
+        System.out.println("Project ID: " + getProjectID());
+        System.out.println("Root Folder: " + getRootFolder());
+        System.out.println("Languages: " + getLanguages());
+    }
+
+    public void readInsiderVersion() {
+        try {
+            insiderVersion = new String(getClass().getClassLoader().getResourceAsStream("insider-version").readAllBytes()).trim();
+        } catch (IOException e) {
+            log.warn("Could not read Insider Version", e);
+        }
+    }
+
+    public String getRootFolder() {
+        if (rootFolder == null) {
+            rootFolder = (String) argumenthor.getRawValue(ROOT_FOLDER);
+            if (rootFolder.endsWith("\\") || rootFolder.endsWith("/")) {
+                rootFolder = rootFolder.substring(0, rootFolder.length() - 1);
+            }
+        }
+
+        return rootFolder;
+    }
+
+    public String getProjectID() {
+        if (projectID == null) {
+            projectID = (String) argumenthor.getRawValue(PROJECT_ID);
+        }
+        return projectID;
+    }
+
+    public List<String> getLanguages() {
+        if (languages == null) {
+            languages = ((List<String>) argumenthor.getRawValue(LANGUAGES)).stream()
+                    .filter(s -> !s.isBlank()).collect(Collectors.toList());
+        }
+        return languages;
+    }
+
+    public String getLanguagesFile() {
+        if (languagesFile == null) {
+            languagesFile = (String) argumenthor.getRawValue(LINGUIST_FILE);
+        }
+        return languagesFile;
+    }
+
+    public int getDepextMaxNamespaceLength() {
+        if (depextMaxNamespaceLength == -1) {
+            try {
+            depextMaxNamespaceLength = (int) argumenthor.getRawValue(DEPEXT_MAX_NAMESPACE_LENGTH);
+            }
+            catch (Exception e) {
+                depextMaxNamespaceLength = DEFAULT_DEPEXT_MAX_NAMESPACE_LENGTH;
+            }
+        }
+
+        return depextMaxNamespaceLength;
+    }
+
+    public String getInsiderVersion() {
+        return insiderVersion;
+    }
+}

package/src/main/java/org/dxworks/insider/constants/InsiderConstants.java

@@ -0,0 +1,19 @@
+package org.dxworks.insider.constants;
+
+public interface InsiderConstants {
+
+    String PROJECT_ID = "projectID";
+    String ROOT_FOLDER = "rootFolder";
+    String LANGUAGES = "languages";
+    String DEPEXT_MAX_NAMESPACE_LENGTH = "depext.max.namespace.length";
+    String LINGUIST_FILE = "linguist.file";
+    String DEFAULT_LINGUIST_FILE = "languages.yml";
+    int DEFAULT_DEPEXT_MAX_NAMESPACE_LENGTH = 200;
+
+    String CONFIGURATION_FOLDER = "config";
+    String RESULTS_FOLDER = "results";
+
+    String CONFIGURATION_FILE = "insider-conf.properties";
+    String VERSION_FILE = ".insider-version";
+    String STARS = "*******************************************************************************";
+}

package/src/main/java/org/dxworks/insider/depext/AbstractImportsProcessor.java

@@ -0,0 +1,57 @@
+package org.dxworks.insider.depext;
+
+import org.dxworks.insider.InsiderFile;
+import org.dxworks.insider.utils.FileUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public abstract class AbstractImportsProcessor {
+    protected List<String> lines;
+    private final InsiderFile file;
+    protected String namespace;
+    private final List<ImportItem> importedItems = new ArrayList<>();
+
+    public AbstractImportsProcessor(InsiderFile file) {
+
+        lines = Arrays.asList(FileUtils.removeComments(file.getContent()).split("\n"));
+        this.file = file;
+
+        lines.stream()
+            .map(String::trim)
+            .map(it -> it.replaceAll("[^\\x00-\\x7F]", ""))
+            .forEach(trimmedLine -> {
+                if (namespace == null) {
+                    String foundNamespace = namespaceLine(trimmedLine);
+                    if (foundNamespace != null) namespace = foundNamespace;
+                }
+                List<ImportItem> crtImportItems = importLine(trimmedLine);
+                if (crtImportItems != null) importedItems.addAll(crtImportItems);
+            });
+
+    }
+
+    public ImportResult extract() {
+        ImportResult resultObject = new ImportResult();
+
+        resultObject.filename = file.getFullyQualifiedName();
+        resultObject.language = language();
+        resultObject.namespace = namespace();
+        resultObject.importedItems = importedItems;
+        resultObject.linesOfCode = (lines != null) ? lines.size() : 0;
+
+        return resultObject;
+    }
+
+    private String namespace() {
+        return namespace == null ? "" : namespace;
+    }
+
+    protected abstract String language();
+
+    protected abstract String namespaceLine(String trimmedLine);
+
+    protected abstract List<ImportItem> importLine(String trimmedLine);
+
+}

package/src/main/java/org/dxworks/insider/depext/CPPImportsProcessor.java

@@ -0,0 +1,55 @@
+package org.dxworks.insider.depext;
+
+import org.dxworks.insider.InsiderFile;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class CPPImportsProcessor extends AbstractImportsProcessor {
+    public static int counter = 0;
+
+    public CPPImportsProcessor(InsiderFile insiderFile) {
+        super(insiderFile);
+        counter++;
+    }
+
+    @Override
+    protected String language() {
+        return "cpp";
+    }
+
+    @Override
+    protected String namespaceLine(String trimmedLine) {
+        if (trimmedLine.startsWith("namespace")) {
+            String patternString = "(namespace\\s+)(\\w+)(.*)";
+
+            Matcher matcher = Pattern.compile(patternString).matcher(trimmedLine);
+            if (matcher.find()) return matcher.group(2);
+        }
+
+        return null;
+    }
+
+
+    @Override
+    protected List<ImportItem> importLine(String trimmedLine) {
+        if (trimmedLine.startsWith("#include ") == false) return null;
+
+        String patternString = "(#include\\s+<)([\\w\\d/.\\\\]+)(>.*)";
+
+        Matcher matcher = Pattern.compile(patternString).matcher(trimmedLine);
+        if (matcher.find())
+            return Collections.singletonList(new ImportItem(matcher.group(2), "library"));
+
+        patternString = "(#include\\s+\")([\\w/.]+)(\")";
+
+        matcher = Pattern.compile(patternString).matcher(trimmedLine);
+        if (matcher.find())
+            return Collections.singletonList(new ImportItem(matcher.group(2)));
+
+        return null;
+    }
+
+}

package/src/main/java/org/dxworks/insider/depext/CSharpImportsProcessor.java

@@ -0,0 +1,53 @@
+package org.dxworks.insider.depext;
+
+import org.dxworks.insider.InsiderFile;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class CSharpImportsProcessor extends AbstractImportsProcessor {
+    public static int counter = 0;
+
+    public CSharpImportsProcessor(InsiderFile insiderFile) {
+        super(insiderFile);
+        counter++;
+    }
+
+    @Override
+    protected String language() {
+        return "csharp";
+    }
+
+    @Override
+    protected String namespaceLine(String trimmedLine) {
+        return trimmedLine.startsWith("namespace") && trimmedLine.length() >= 10 ?
+            trimmedLine.substring(10).trim().replaceAll(";", "") : null;
+    }
+
+
+    @Override
+    protected List<ImportItem> importLine(String trimmedLine) {
+        if (trimmedLine.startsWith("using") == false) return null;
+        if (trimmedLine.contains("(")) return null;
+
+        Pattern p = Pattern.compile("(using|using static)\\s+([\\w.]+\\s*)=*\\s*([\\w.<>\\s,]*);");
+        Matcher m = p.matcher(trimmedLine.trim());
+        if (m.find() == false) return null;
+
+        String attribute = "";
+        String importedNamespace = "";
+
+        if (trimmedLine.startsWith("using static")) {
+            attribute = "static";
+            importedNamespace = m.group(3);
+        } else {
+            importedNamespace = trimmedLine.contains("=") ? m.group(3) : m.group(2);
+        }
+
+        importedNamespace = importedNamespace.replace(",", "");
+
+        return Collections.singletonList(new ImportItem(importedNamespace, attribute));
+    }
+}

package/src/main/java/org/dxworks/insider/depext/DependencyExtractor.java

@@ -0,0 +1,105 @@
+package org.dxworks.insider.depext;
+
+import org.dxworks.insider.InsiderFile;
+import org.dxworks.insider.configuration.InsiderConfiguration;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.PrintWriter;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+
+public class DependencyExtractor {
+    private static final List<String> cppExtensions = Arrays.asList(".c", ".cpp", ".hpp", ".cc", ".h", ".hh", ".cxx", ".hxx");
+    private static final String javaExtension = "java";
+    private static final String visualBasicExtension = ".vb";
+    private static final String kotlinExtension = "kt";
+    private static final String scalaExtension = "scala";
+    private static final String csExtension = "cs";
+    private static final String groovyExtension = "groovy";
+    private final List<ImportResult> resultsList = new ArrayList<>();
+    private int counterFiles = 0;
+    private int countIgnoredFiles = 0;
+
+    public void processAndWriteResults(List<InsiderFile> files) {
+        files.forEach(this::execute);
+        System.out.println("\n\nTotal files in folder: " + counterFiles);
+        System.out.println("Files skipped: " + countIgnoredFiles + "\n");
+
+        if (JavastackImportsProcessor.hasFiles()) System.out.println(JavastackImportsProcessor.statistics());
+        if (CSharpImportsProcessor.counter > 0)
+            System.out.println(CSharpImportsProcessor.counter + " C# files processed");
+        if (CPPImportsProcessor.counter > 0) System.out.println(CPPImportsProcessor.counter + " C++ files processed");
+        if (VBImportsProcessor.counter > 0) System.out.println(VBImportsProcessor.counter + " VB.NET files processed");
+
+
+        String outputFilename = "results/" + InsiderConfiguration.getInstance().getProjectID() + "-depext.csv";
+        try {
+            Path.of("results").toFile().mkdirs();
+            PrintWriter outputFile = new PrintWriter(outputFilename);
+
+            System.err.println("Writing " + resultsList.size() + " entries to output file: " + outputFilename);
+
+            outputFile.println(ImportResult.header());
+            resultsList.stream()
+                    .filter(it -> it.namespace.length() < InsiderConfiguration.getInstance().getDepextMaxNamespaceLength())
+                    .forEach(outputFile::println);
+            outputFile.close();
+        } catch (FileNotFoundException e) {
+            System.err.println("\nCannot write to file: " + outputFilename);
+        }
+    }
+
+    private void execute(InsiderFile insiderFile) {
+        AbstractImportsProcessor processor = createProcessor(insiderFile);
+
+        counterFiles++;
+        if (counterFiles % 1000 == 0) System.out.print("*");
+
+        if (processor != null) resultsList.add(processor.extract());
+        else countIgnoredFiles++;
+    }
+
+    private AbstractImportsProcessor createProcessor(InsiderFile insiderFile) {
+        String extension = extension(insiderFile.getPath());
+
+        if (extension == null) return null;
+
+        if (extension.endsWith(javaExtension)) return new JavastackImportsProcessor(insiderFile, "java");
+        else if (extension.endsWith(kotlinExtension)) return new JavastackImportsProcessor(insiderFile, "kotlin");
+        else if (extension.endsWith(scalaExtension)) return new JavastackImportsProcessor(insiderFile, "scala");
+        else if (extension.endsWith(groovyExtension)) return new JavastackImportsProcessor(insiderFile, "groovy");
+        else if (extension.endsWith(csExtension) && isRelevantCSharpFolder(insiderFile.getPath()))
+            return new CSharpImportsProcessor(insiderFile);
+        else if (cppExtensions.contains(extension)) return new CPPImportsProcessor(insiderFile);
+        else if (extension.endsWith(visualBasicExtension)) return new VBImportsProcessor(insiderFile);
+
+
+        return null;
+    }
+
+    private boolean isRelevantCSharpFolder(String path) {
+        if (path.contains(File.separator + ".vs" + File.separator)) return false;
+        if (path.contains(File.separator + "obj" + File.separator + "Debug" + File.separator)) return false;
+        if (path.contains(File.separator + "obj" + File.separator + "Release" + File.separator)) return false;
+        return true;
+    }
+
+    public boolean acceptFile(String path) {
+        List<String> acceptedExtensions = new ArrayList<>(cppExtensions);
+        acceptedExtensions.add(javaExtension);
+        acceptedExtensions.add(kotlinExtension);
+        acceptedExtensions.add(scalaExtension);
+        acceptedExtensions.add(groovyExtension);
+        acceptedExtensions.add(csExtension);
+        return acceptedExtensions.stream().anyMatch(path::endsWith);
+    }
+
+    private String extension(String filename) {
+        int extensionIndex = filename.lastIndexOf(".");
+        return (extensionIndex == -1) ? null : filename.substring(extensionIndex).toLowerCase();
+    }
+}

package/src/main/java/org/dxworks/insider/depext/ImportItem.java

@@ -0,0 +1,19 @@
+package org.dxworks.insider.depext;
+
+public class ImportItem {
+    String name;
+    String attribute;
+
+    public ImportItem(String _name, String _attribute) {
+        name = _name;
+        attribute = _attribute;
+    }
+
+    public ImportItem(String _name) {
+        this(_name, "");
+    }
+
+    public String toString() {
+        return attribute.isEmpty() ? name : name + "," + attribute;
+    }
+}

package/src/main/java/org/dxworks/insider/depext/ImportResult.java

@@ -0,0 +1,32 @@
+package org.dxworks.insider.depext;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class ImportResult {
+    String filename;
+    String language;
+    String namespace;
+    Integer linesOfCode;
+
+    List<ImportItem> importedItems;
+
+    public String toString() {
+        if (importedItems.isEmpty())
+            return filename + "," + language + "," + namespace + "," + linesOfCode + ",";
+
+        List<String> importLines = importedItems.stream().map(ImportItem::toString).collect(Collectors.toList());
+
+        String fileColumns = filename + "," + language + "," + namespace + "," + linesOfCode;
+
+        List<String> rows = new ArrayList<>();
+
+        importLines.forEach(importItem -> rows.add(fileColumns + "," + importItem));
+        return String.join("\n", rows);
+    }
+
+    public static String header() {
+        return "file,language,namespace,#lines,import,import_attribute";
+    }
+}

package/src/main/java/org/dxworks/insider/depext/JavastackImportsProcessor.java

@@ -0,0 +1,88 @@
+package org.dxworks.insider.depext;
+
+import org.dxworks.insider.InsiderFile;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class JavastackImportsProcessor extends AbstractImportsProcessor {
+    public static int javaCounter = 0;
+    public static int kotlinCounter = 0;
+    public static int scalaCounter = 0;
+    public static int groovyCounter = 0;
+
+    private final String language;
+
+    public JavastackImportsProcessor(InsiderFile insiderFile, String language) {
+        super(insiderFile);
+
+        this.language = language;
+
+        switch (this.language) {
+            case "java":
+                javaCounter++;
+                break;
+            case "scala":
+                scalaCounter++;
+                break;
+            case "kotlin":
+                kotlinCounter++;
+                break;
+            case "groovy":
+                groovyCounter++;
+                break;
+        }
+    }
+
+    public static boolean hasFiles() {
+        return javaCounter + kotlinCounter + scalaCounter + groovyCounter > 0;
+    }
+
+    public static String statistics() {
+        List<String> rows = new ArrayList<>();
+        if (javaCounter > 0) rows.add(javaCounter + " Java files processed");
+        if (kotlinCounter > 0) rows.add(kotlinCounter + " Kotlin files processed");
+        if (groovyCounter > 0) rows.add(groovyCounter + " Groovy files processed");
+        if (scalaCounter > 0) rows.add(scalaCounter + " Scala files processed");
+
+        return String.join("\n", rows);
+    }
+
+    @Override
+    protected String namespaceLine(String trimmedLine) {
+        return trimmedLine.startsWith("package") ? trimmedLine.substring(8).trim().replaceAll(";", "") : null;
+    }
+
+    @Override
+    protected String language() {
+        return language;
+    }
+
+    @Override
+    protected List<ImportItem> importLine(String trimmedLine) {
+        if (!trimmedLine.startsWith("import ")) return null;
+
+        if (trimmedLine.contains("(")) return null;
+        if (trimmedLine.contains("=>")) return null;
+
+        String attribute = "";
+        if (trimmedLine.startsWith("import static ")) {
+            trimmedLine = trimmedLine.substring(14);
+            attribute = "static";
+        } else trimmedLine = trimmedLine.substring(7);
+
+        trimmedLine = trimmedLine.trim().replaceAll(";", "").replace("}", "").replace("`", "").replace("._", "");
+
+        String[] parts = trimmedLine.split("\\{");
+        if (parts.length == 1) return Collections.singletonList(new ImportItem(trimmedLine, attribute));
+
+        String prefix = parts[0];
+
+        String[] listOfImports = parts[1].split(",");
+
+        List<ImportItem> importItems = new ArrayList<>();
+        for (String importItem : listOfImports) new ImportItem((prefix + importItem.trim()), attribute);
+        return importItems;
+    }
+}

package/src/main/java/org/dxworks/insider/depext/VBImportsProcessor.java

@@ -0,0 +1,41 @@
+package org.dxworks.insider.depext;
+
+import org.dxworks.insider.InsiderFile;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+class VBImportsProcessor extends AbstractImportsProcessor {
+    public static int counter = 0;
+
+    public VBImportsProcessor(InsiderFile insiderFile) {
+        super(insiderFile);
+        counter++;
+    }
+
+    @Override
+    protected String language() {
+        return "visualbasic";
+    }
+
+    @Override
+    protected String namespaceLine(String trimmedLine) {
+        if (trimmedLine.trim().startsWith("Namespace") == false) return null;
+        Pattern p = Pattern.compile("\\s*Namespace\\s([\\w.]+)(.*)");
+        Matcher m = p.matcher(trimmedLine.trim());
+        return m.find() ? m.group(1) : null;
+    }
+
+    protected List<ImportItem> importLine(String trimmedLine) {
+        Pattern p = Pattern.compile("Imports\\s([\\w.]+\\s*=*\\s*[\\w.]+)(.*)");
+        Matcher m = p.matcher(trimmedLine.trim());
+        if (m.find() == false) return null;
+
+        String[] importedEntity = m.group(1).split("=");
+
+        if (importedEntity.length == 1) return Collections.singletonList(new ImportItem(importedEntity[0].trim()));
+        return Collections.singletonList(new ImportItem(importedEntity[1].trim()));
+    }
+}

package/src/main/java/org/dxworks/insider/exceptions/InsiderException.java

@@ -0,0 +1,7 @@
+package org.dxworks.insider.exceptions;
+
+public class InsiderException extends RuntimeException {
+    public InsiderException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}