pmdtester 1.0.0.pre.beta3 → 1.0.0

data/lib/pmdtester/builders/html_report_builder.rb

@@ -4,6 +4,7 @@ module PmdTester
   # This class is the parent of all classes which is used to build html report
   class HtmlReportBuilder
     CSS_SRC_DIR = ResourceLocator.locate('resources/css')
+    NO_DIFFERENCES_MESSAGE = 'No differences found!'
 
     def build_html_report(title_name)
       html_builder = Nokogiri::HTML::Builder.new do |doc|
@@ -25,9 +26,31 @@ module PmdTester
       end
     end
 
+    def build_table_head(doc, *columns)
+      doc.thead do
+        doc.tr do
+          columns.each do |column|
+            doc.th column
+          end
+        end
+      end
+    end
+
+    def build_table_anchor_column(doc, prefix, index)
+      doc.td do
+        doc.a(id: "#{prefix}#{index}", href: "##{prefix}#{index}") { doc.text '#' }
+      end
+    end
+
     def copy_css(report_dir)
       css_dest_dir = "#{report_dir}/css"
       FileUtils.copy_entry(CSS_SRC_DIR, css_dest_dir)
     end
+
+    def build_table_content_for(doc, removed_size, new_size)
+      doc.font(color: 'red') { doc.text "-#{removed_size}" }
+      doc.text ' | '
+      doc.font(color: 'green') { doc.text "+#{new_size}" }
+    end
   end
 end

data/lib/pmdtester/builders/pmd_report_builder.rb

@@ -1,17 +1,19 @@
 # frozen_string_literal: true
 
 require 'fileutils'
+require 'rufus-scheduler'
 
 module PmdTester
   # Building pmd xml reports according to a list of standard
   # projects and branch of pmd source code
   class PmdReportBuilder
     include PmdTester
-    def initialize(branch_config, projects, local_git_repo, pmd_branch_name)
+    def initialize(branch_config, projects, local_git_repo, pmd_branch_name, threads = 1)
       @branch_config = branch_config
       @projects = projects
       @local_git_repo = local_git_repo
       @pmd_branch_name = pmd_branch_name
+      @threads = threads
       @pwd = Dir.getwd
 
       @pmd_branch_details = PmdBranchDetail.new(pmd_branch_name)
@@ -28,7 +30,7 @@ module PmdTester
       Cmd.execute(reset_cmd)
     end
 
-    def get_projects
+    def clone_projects
       logger.info 'Cloning projects started'
 
       @projects.each do |project|
@@ -46,29 +48,58 @@ module PmdTester
         end
         logger.info "Cloning #{project.name} completed"
       end
+
+      logger.info 'Cloning projects completed'
     end
 
     def get_pmd_binary_file
-      logger.info 'Start packaging PMD'
+      logger.info "#{@pmd_branch_name}: Start packaging PMD"
       Dir.chdir(@local_git_repo) do
-        checkout_cmd = "git checkout #{@pmd_branch_name}"
-        Cmd.execute(checkout_cmd)
+        current_head_sha = Cmd.execute('git rev-parse HEAD')
+        current_branch_sha = Cmd.execute("git rev-parse #{@pmd_branch_name}")
 
-        @pmd_branch_details.branch_last_sha = get_last_commit_sha
-        @pmd_branch_details.branch_last_message = get_last_commit_message
+        @pmd_version = determine_pmd_version
 
-        package_cmd = './mvnw clean package -Dpmd.skip=true -Dmaven.test.skip=true' \
-                      ' -Dmaven.checkstyle.skip=true -Dmaven.javadoc.skip=true'
-        Cmd.execute(package_cmd)
+        # in case we are already on the correct branch
+        # and a binary zip already exists...
+        if current_head_sha == current_branch_sha &&
+           File.exist?("pmd-dist/target/pmd-bin-#{@pmd_version}.zip")
+          logger.warn "#{@pmd_branch_name}: Skipping packaging - zip for " \
+                      "#{@pmd_version} already exists"
+        else
+          build_pmd
+        end
 
-        version_cmd = "./mvnw -q -Dexec.executable=\"echo\" -Dexec.args='${project.version}' " \
-                      '--non-recursive org.codehaus.mojo:exec-maven-plugin:1.5.0:exec'
-        @pmd_version = Cmd.execute(version_cmd)
+        @pmd_branch_details.branch_last_sha = get_last_commit_sha
+        @pmd_branch_details.branch_last_message = get_last_commit_message
 
+        logger.info "#{@pmd_branch_name}: Extracting the zip"
         unzip_cmd = "unzip -qo pmd-dist/target/pmd-bin-#{@pmd_version}.zip -d #{@pwd}/target"
         Cmd.execute(unzip_cmd)
       end
-      logger.info 'Packaging PMD completed'
+      logger.info "#{@pmd_branch_name}: Packaging PMD completed"
+    end
+
+    def build_pmd
+      logger.info "#{@pmd_branch_name}: Checking out the branch"
+      checkout_cmd = "git checkout #{@pmd_branch_name}"
+      Cmd.execute(checkout_cmd)
+
+      # determine the version again - it might be different in the other branch
+      @pmd_version = determine_pmd_version
+
+      logger.info "#{@pmd_branch_name}: Building PMD #{@pmd_version}..."
+      package_cmd = './mvnw clean package' \
+                    ' -Dmaven.test.skip=true' \
+                    ' -Dmaven.javadoc.skip=true' \
+                    ' -Dmaven.source.skip=true'
+      Cmd.execute(package_cmd)
+    end
+
+    def determine_pmd_version
+      version_cmd = "./mvnw -q -Dexec.executable=\"echo\" -Dexec.args='${project.version}' " \
+                    '--non-recursive org.codehaus.mojo:exec-maven-plugin:1.5.0:exec'
+      Cmd.execute(version_cmd)
     end
 
     def get_last_commit_sha
@@ -81,26 +112,40 @@ module PmdTester
       Cmd.execute(get_last_commit_message_cmd)
     end
 
-    def generate_pmd_report(src_root_dir, report_file)
+    def generate_pmd_report(project)
       run_path = "target/pmd-bin-#{@pmd_version}/bin/run.sh"
-      pmd_cmd = "#{run_path} pmd -d #{src_root_dir} -f xml -R #{@branch_config} " \
-                "-r #{report_file} -failOnViolation false"
+      pmd_cmd = "#{run_path} pmd -d #{project.local_source_path} -f xml " \
+                "-R #{project.get_config_path(@pmd_branch_name)} " \
+                "-r #{project.get_pmd_report_path(@pmd_branch_name)} " \
+                "-failOnViolation false -t #{@threads}"
       start_time = Time.now
       Cmd.execute(pmd_cmd)
       end_time = Time.now
       [end_time - start_time, end_time]
     end
 
+    def generate_config_for(project)
+      doc = Nokogiri::XML(File.read(@branch_config))
+      ruleset = doc.at_css('ruleset')
+      project.exclude_pattern.each do |exclude_pattern|
+        ruleset.add_child("<exclude-pattern>#{exclude_pattern}</exclude-pattern>")
+      end
+
+      File.open(project.get_config_path(@pmd_branch_name), 'w') do |x|
+        x << doc.to_s
+      end
+    end
+
     def generate_pmd_reports
       logger.info "Generating PMD report started -- branch #{@pmd_branch_name}"
-      get_pmd_binary_file
 
       sum_time = 0
       @projects.each do |project|
-        logger.info "Generating #{project.name}'s PMD report"
-        execution_time, end_time =
-          generate_pmd_report(project.local_source_path,
-                              project.get_pmd_report_path(@pmd_branch_name))
+        progress_logger = SimpleProgressLogger.new(project.name)
+        progress_logger.start
+        generate_config_for(project)
+        execution_time, end_time = generate_pmd_report(project)
+        progress_logger.stop
         sum_time += execution_time
 
         report_details = PmdReportDetail.new
@@ -117,7 +162,8 @@ module PmdTester
     end
 
     def build
-      get_projects
+      clone_projects
+      get_pmd_binary_file
       generate_pmd_reports
     end
   end

data/lib/pmdtester/builders/rule_set_builder.rb

@@ -29,6 +29,7 @@ module PmdTester
       output_filter_set(rule_sets)
       build_config_file(rule_sets)
       logger.debug "Dynamic configuration: #{[rule_sets]}"
+      rule_sets
     end
 
     def output_filter_set(rule_sets)
@@ -46,7 +47,7 @@ module PmdTester
         base = @options.base_branch
         patch = @options.patch_branch
         # We only need to support git here, since PMD's repo is using git.
-        diff_cmd = "git diff --name-only #{base}..#{patch} -- pmd-core pmd-java"
+        diff_cmd = "git diff --name-only #{base}..#{patch} -- pmd-core/src/main pmd-java/src/main"
         filenames = Cmd.execute(diff_cmd)
       end
       filenames.split("\n")
@@ -79,7 +80,7 @@ module PmdTester
     def build_config_file(rule_sets)
       if rule_sets.empty?
         logger.info NO_JAVA_RULES_CHANGED_MESSAGE
-        exit 0
+        return
       end
 
       doc = Nokogiri::XML(File.read(PATH_TO_ALL_JAVA_RULES))

data/lib/pmdtester/builders/simple_progress_logger.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'rufus-scheduler'
+
+module PmdTester
+  # Helper class that provides a simple progress logging
+  class SimpleProgressLogger
+    include PmdTester
+    def initialize(project_name)
+      @project_name = project_name
+    end
+
+    def start
+      logger.info "Starting to generate #{@project_name}'s PMD report"
+      message_counter = 1
+      @scheduler = Rufus::Scheduler.new
+      @scheduler.every '2m' do
+        logger.info "Still generating #{@project_name}'s PMD report (#{message_counter})..."
+        message_counter += 1
+      end
+    end
+
+    def stop
+      @scheduler.shutdown
+    end
+  end
+end

data/lib/pmdtester/builders/summary_report_builder.rb

@@ -29,7 +29,6 @@ module PmdTester
     def get_branch_details(branch_name)
       details = PmdBranchDetail.new(branch_name)
       details.load
-      details
     end
 
     def build_body(doc)
@@ -71,6 +70,10 @@ module PmdTester
                                @patch_details.branch_last_message)
         build_branch_table_row(doc, 'total execution time', @base_details.format_execution_time,
                                @patch_details.format_execution_time)
+        build_branch_table_row(doc, 'jdk version', @base_details.jdk_version,
+                               @patch_details.jdk_version)
+        build_branch_table_row(doc, 'language', @base_details.language,
+                               @patch_details.language)
         build_branch_config_table_row(doc)
       end
     end
@@ -122,8 +125,9 @@ module PmdTester
         doc.tr do
           doc.th 'project name'
           doc.th 'project branch/tag'
-          doc.th 'diff exist?'
-          doc.th 'introduce new errors?'
+          doc.th 'removed|new errors size'
+          doc.th 'removed|new violations size'
+          doc.th 'removed|new configerrors size'
         end
       end
     end
@@ -136,8 +140,18 @@ module PmdTester
               doc.a(href: project.diff_report_index_ref_path) { doc.text project.name }
             end
             doc.td project.tag
-            doc.td project.diffs_exist? ? 'Yes' : 'No'
-            doc.td project.introduce_new_errors? ? 'Yes' : 'No'
+            doc.td do
+              build_table_content_for(doc, project.removed_errors_size,
+                                      project.new_errors_size)
+            end
+            doc.td do
+              build_table_content_for(doc, project.removed_violations_size,
+                                      project.new_violations_size)
+            end
+            doc.td do
+              build_table_content_for(doc, project.removed_configerrors_size,
+                                      project.new_configerrors_size)
+            end
           end
         end
       end

data/lib/pmdtester/cmd.rb

@@ -7,20 +7,34 @@ module PmdTester
   class Cmd
     extend PmdTester
     def self.execute(cmd)
+      stdout, _stderr, _status = internal_execute(cmd)
+      stdout&.chomp!
+      stdout
+    end
+
+    def self.stderr_of(cmd)
+      _stdout, stderr, _status = internal_execute(cmd)
+      stderr
+    end
+
+    def self.internal_execute(cmd)
       logger.debug "execute command '#{cmd}'"
 
       stdout, stderr, status = Open3.capture3("#{cmd};")
 
       logger.debug stdout
       unless status.success?
+        logger.error stdout
         logger.error stderr
         raise CmdException.new(cmd, stderr)
       end
 
       stdout&.chomp!
 
-      stdout
+      [stdout, stderr, status]
     end
+
+    private_class_method :internal_execute
   end
 
   # The exception should be raised when the shell command failed.

data/lib/pmdtester/parsers/options.rb

@@ -25,6 +25,7 @@ module PmdTester
     attr_reader :config
     attr_reader :project_list
     attr_reader :mode
+    attr_reader :threads
     attr_reader :html_flag
     attr_reader :auto_config_flag
     attr_reader :debug_flag
@@ -40,6 +41,7 @@ module PmdTester
       @config = options[:c]
       @project_list = options[:l]
       @mode = options[:m]
+      @threads = options[:t]
       @html_flag = options[:f]
       @auto_config_flag = options[:a]
       @debug_flag = options[:d]
@@ -61,8 +63,8 @@ module PmdTester
           single: Set this option to 'single' if your patch branch contains changes
             for any option that can't work on master/base branch
           online: Set this option to 'online' if you want to download
-            'the PMD report of master/base branch rather than generating it locally
-          local: Default option is 'local'
+            the PMD report of master/base branch rather than generating it locally
+          local: Default option is 'local', PMD reports for the base and patch branches are generated locally.
       DOC
 
       Slop.parse argv do |o|
@@ -79,6 +81,8 @@ module PmdTester
                  'path to the file which contains the list of standard projects',
                  default: DEFAULT_LIST_PATH
         o.string '-m', '--mode', mode_message, default: 'local'
+        o.integer '-t', '--threads', 'Sets the number of threads used by PMD.' \
+              ' Set threads to 0 to disable multi-threading processing.', default: 1
         o.bool '-f', '--html-flag',
                'whether to not generate the html diff report in single mode'
         o.bool '-a', '--auto-gen-config',

data/lib/pmdtester/parsers/pmd_report_document.rb

@@ -7,12 +7,15 @@ module PmdTester
   class PmdReportDocument < Nokogiri::XML::SAX::Document
     attr_reader :violations
     attr_reader :errors
+    attr_reader :configerrors
     def initialize(branch_name, working_dir, filter_set = nil)
       @violations = PmdViolations.new
       @errors = PmdErrors.new
+      @configerrors = PmdConfigErrors.new
       @current_violations = []
       @current_violation = nil
       @current_error = nil
+      @current_configerror = nil
       @current_element = ''
       @filename = ''
       @filter_set = filter_set
@@ -34,6 +37,8 @@ module PmdTester
         @current_filename = remove_work_dir!(attrs['filename'])
         remove_work_dir!(attrs['msg'])
         @current_error = PmdError.new(attrs, @branch_name)
+      when 'configerror'
+        @current_configerror = PmdConfigError.new(attrs, @branch_name)
       end
     end
 
@@ -42,15 +47,13 @@ module PmdTester
     end
 
     def characters(string)
-      @current_violation.text = string unless @current_violation.nil?
+      @current_violation&.text = string
     end
 
     def end_element(name)
       case name
       when 'file'
-        unless @current_violations.empty?
-          @violations.add_violations_by_filename(@current_filename, @current_violations)
-        end
+        @violations.add_violations_by_filename(@current_filename, @current_violations)
         @current_filename = nil
       when 'violation'
         @current_violations.push(@current_violation) if match_filter_set?(@current_violation)
@@ -59,12 +62,15 @@ module PmdTester
         @errors.add_error_by_filename(@current_filename, @current_error)
         @current_filename = nil
         @current_error = nil
+      when 'configerror'
+        @configerrors.add_error(@current_configerror)
+        @current_configerror = nil
       end
     end
 
     def cdata_block(string)
       remove_work_dir!(string)
-      @current_error.text = string unless @current_error.nil?
+      @current_error&.text = string
     end
 
     def match_filter_set?(violation)

data/lib/pmdtester/pmd_branch_detail.rb

@@ -5,11 +5,15 @@ require 'json'
 module PmdTester
   # This class represents all details about branch of pmd
   class PmdBranchDetail
+    include PmdTester
+
     attr_accessor :branch_last_sha
     attr_accessor :branch_last_message
     attr_accessor :branch_name
     # The branch's execution time on all standard projects
     attr_accessor :execution_time
+    attr_reader :jdk_version
+    attr_reader :language
 
     def self.branch_filename(branch_name)
       branch_name&.tr('/', '_')
@@ -22,6 +26,9 @@ module PmdTester
       branch_filename = PmdBranchDetail.branch_filename(branch_name)
       @base_branch_dir = "target/reports/#{branch_filename}" unless @branch_name.nil?
       @execution_time = 0
+      # the result of command 'java -version' is going to stderr
+      @jdk_version = Cmd.stderr_of('java -version')
+      @language = ENV['LANG']
     end
 
     def load
@@ -31,20 +38,27 @@ module PmdTester
         @branch_last_message = hash['branch_last_message']
         @branch_name = hash['branch_name']
         @execution_time = hash['execution_time']
-        hash
+        @jdk_version = hash['jdk_version']
+        @language = hash['language']
       else
-        {}
+        @jdk_version = ''
+        @language = ''
+        logger.warn "#{branch_details_path} doesn't exist!"
       end
+      self
     end
 
     def save
       hash = { branch_last_sha: @branch_last_sha,
                branch_last_message: @branch_last_message,
                branch_name: @branch_name,
-               execution_time: @execution_time }
+               execution_time: @execution_time,
+               jdk_version: @jdk_version,
+               language: @language }
       file = File.new(branch_details_path, 'w')
       file.puts JSON.generate(hash)
       file.close
+      self
     end
 
     def branch_details_path