pmdtester 1.0.0.pre.beta3 → 1.1.2

data/lib/pmdtester/collection_by_file.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module PmdTester
+  # A collection of things, grouped by file.
+  #
+  # (Note: this replaces PmdErrors and PmdViolations)
+  class CollectionByFile
+    def initialize
+      # a hash of filename -> [list of items]
+      @hash = Hash.new([])
+      @total = 0
+    end
+
+    def add_all(filename, values)
+      return if values.empty?
+
+      if @hash.key?(filename)
+        @hash[filename].concat(values)
+      else
+        @hash[filename] = values
+      end
+      @total += values.size
+    end
+
+    def total_size
+      @total
+    end
+
+    def all_files
+      @hash.keys
+    end
+
+    def num_files
+      @hash.size
+    end
+
+    def all_values
+      @hash.values.flatten
+    end
+
+    def each_value(&block)
+      @hash.each_value do |vs|
+        vs.each(&block)
+      end
+    end
+
+    def [](fname)
+      @hash[fname]
+    end
+
+    def to_h
+      @hash
+    end
+  end
+end

data/lib/pmdtester/parsers/options.rb

@@ -16,6 +16,7 @@ module PmdTester
     SINGLE = 'single'
     DEFAULT_CONFIG_PATH = ResourceLocator.locate('config/all-java.xml')
     DEFAULT_LIST_PATH = ResourceLocator.locate('config/project-list.xml')
+    DEFAULT_BASELINE_URL_PREFIX = 'https://sourceforge.net/projects/pmd/files/pmd-regression-tester/'
 
     attr_reader :local_git_repo
     attr_reader :base_branch
@@ -25,10 +26,14 @@ module PmdTester
     attr_reader :config
     attr_reader :project_list
     attr_reader :mode
+    attr_reader :threads
     attr_reader :html_flag
     attr_reader :auto_config_flag
     attr_reader :debug_flag
     attr_accessor :filter_set
+    attr_reader :keep_reports
+    attr_reader :error_recovery
+    attr_reader :baseline_download_url_prefix
 
     def initialize(argv)
       options = parse(argv)
@@ -40,10 +45,19 @@ module PmdTester
       @config = options[:c]
       @project_list = options[:l]
       @mode = options[:m]
+      @threads = options[:t]
       @html_flag = options[:f]
       @auto_config_flag = options[:a]
       @debug_flag = options[:d]
       @filter_set = nil
+      @keep_reports = options.keep_reports?
+      @error_recovery = options.error_recovery?
+      url = options[:baseline_download_url]
+      @baseline_download_url_prefix = if url[-1] == '/'
+                                        url
+                                      else
+                                        "#{url}/"
+                                      end
 
       # if the 'config' option is selected then `config` overrides `base_config` and `patch_config`
       @base_config = @config if !@config.nil? && @mode == 'local'
@@ -61,8 +75,8 @@ module PmdTester
           single: Set this option to 'single' if your patch branch contains changes
             for any option that can't work on master/base branch
           online: Set this option to 'online' if you want to download
-            'the PMD report of master/base branch rather than generating it locally
-          local: Default option is 'local'
+            the PMD report of master/base branch rather than generating it locally
+          local: Default option is 'local', PMD reports for the base and patch branches are generated locally.
       DOC
 
       Slop.parse argv do |o|
@@ -79,13 +93,22 @@ module PmdTester
                  'path to the file which contains the list of standard projects',
                  default: DEFAULT_LIST_PATH
         o.string '-m', '--mode', mode_message, default: 'local'
+        o.integer '-t', '--threads', 'Sets the number of threads used by PMD.' \
+              ' Set threads to 0 to disable multi-threading processing.', default: 1
         o.bool '-f', '--html-flag',
                'whether to not generate the html diff report in single mode'
         o.bool '-a', '--auto-gen-config',
                'whether to generate configurations automatically based on branch differences,' \
                'this option only works in online and local mode'
+        o.bool '--keep-reports',
+               'whether to keep old reports and skip running PMD again if possible'
         o.bool '-d', '--debug',
                'whether change log level to DEBUG to see more information'
+        o.bool '--error-recovery',
+               'enable error recovery mode when executing PMD. Might help to analyze errors.'
+        o.string '--baseline-download-url',
+                 'download url prefix from where to download the baseline in online mode',
+                 default: DEFAULT_BASELINE_URL_PREFIX
         o.on '-v', '--version' do
           puts VERSION
           exit

data/lib/pmdtester/parsers/pmd_report_document.rb

@@ -7,74 +7,126 @@ module PmdTester
   class PmdReportDocument < Nokogiri::XML::SAX::Document
     attr_reader :violations
     attr_reader :errors
+    attr_reader :configerrors
+    attr_reader :infos_by_rules
+
     def initialize(branch_name, working_dir, filter_set = nil)
-      @violations = PmdViolations.new
-      @errors = PmdErrors.new
+      @violations = CollectionByFile.new
+      @errors = CollectionByFile.new
+      @configerrors = Hash.new { |hash, key| hash[key] = [] }
+
+      @infos_by_rules = {}
       @current_violations = []
       @current_violation = nil
       @current_error = nil
-      @current_element = ''
-      @filename = ''
+      @current_configerror = nil
       @filter_set = filter_set
       @working_dir = working_dir
       @branch_name = branch_name
+
+      @cur_text = String.new(capacity: 200)
     end
 
     def start_element(name, attrs = [])
       attrs = attrs.to_h
-      @current_element = name
 
       case name
       when 'file'
-        @current_violations = []
-        @current_filename = remove_work_dir!(attrs['name'])
+        handle_start_file attrs
       when 'violation'
-        @current_violation = PmdViolation.new(attrs, @branch_name)
+        handle_start_violation attrs
       when 'error'
-        @current_filename = remove_work_dir!(attrs['filename'])
-        remove_work_dir!(attrs['msg'])
-        @current_error = PmdError.new(attrs, @branch_name)
+        handle_start_error attrs
+      when 'configerror'
+        handle_start_configerror attrs
       end
     end
 
-    def remove_work_dir!(str)
-      str.sub!(/#{@working_dir}/, '')
+    def characters(string)
+      @cur_text << remove_work_dir!(string)
     end
 
-    def characters(string)
-      @current_violation.text = string unless @current_violation.nil?
+    def cdata_block(string)
+      @cur_text << remove_work_dir!(string)
     end
 
     def end_element(name)
       case name
       when 'file'
-        unless @current_violations.empty?
-          @violations.add_violations_by_filename(@current_filename, @current_violations)
-        end
+        @violations.add_all(@current_filename, @current_violations)
         @current_filename = nil
       when 'violation'
-        @current_violations.push(@current_violation) if match_filter_set?(@current_violation)
+        if match_filter_set?(@current_violation)
+          @current_violation.message = finish_text!
+          @current_violations.push(@current_violation)
+        end
         @current_violation = nil
       when 'error'
-        @errors.add_error_by_filename(@current_filename, @current_error)
+        @current_error.stack_trace = finish_text!
+        @errors.add_all(@current_filename, [@current_error])
         @current_filename = nil
         @current_error = nil
+      when 'configerror'
+        @configerrors[@current_configerror.rulename].push(@current_configerror)
+        @current_configerror = nil
       end
+      @cur_text.clear
     end
 
-    def cdata_block(string)
-      remove_work_dir!(string)
-      @current_error.text = string unless @current_error.nil?
+    private
+
+    # Modifies the string in place and returns it
+    # (this is what sub! does, except it returns nil if no replacement occurred)
+    def remove_work_dir!(str)
+      str.sub!(/#{@working_dir}/, '')
+      str
+    end
+
+    def finish_text!
+      res = @cur_text.strip!.dup.freeze
+      @cur_text.clear
+      res
     end
 
     def match_filter_set?(violation)
       return true if @filter_set.nil?
 
-      @filter_set.each do |ruleset|
-        return true if ruleset.eql?(violation.attrs['ruleset'].delete(' ').downcase)
-      end
+      ruleset_attr = violation.ruleset_name.delete(' ').downcase! << '.xml'
+      return true if @filter_set.include?(ruleset_attr)
+
+      rule_ref = "#{ruleset_attr}/#{violation.rule_name}"
+
+      @filter_set.include?(rule_ref)
+    end
+
+    def handle_start_file(attrs)
+      @current_filename = remove_work_dir!(attrs['name'])
+      @current_violations = []
+    end
+
+    def handle_start_violation(attrs)
+      @current_violation = PmdViolation.new(
+        branch: @branch_name,
+        fname: @current_filename,
+        info_url: attrs['externalInfoUrl'],
+        bline: attrs['beginline'].to_i,
+        rule_name: attrs['rule'],
+        ruleset_name: attrs['ruleset'].freeze
+      )
+    end
+
+    def handle_start_error(attrs)
+      @current_filename = remove_work_dir!(attrs['filename'])
+
+      @current_error = PmdError.new(
+        branch: @branch_name,
+        filename: @current_filename,
+        short_message: remove_work_dir!(attrs['msg'])
+      )
+    end
 
-      false
+    def handle_start_configerror(attrs)
+      @current_configerror = PmdConfigError.new(attrs, @branch_name)
     end
   end
 end

data/lib/pmdtester/parsers/projects_parser.rb

@@ -11,9 +11,7 @@ module PmdTester
       document = Nokogiri::XML(File.read(list_file))
 
       errors = schema.validate(document)
-      unless errors.empty?
-        raise ProjectsParserException.new(errors), "Schema validate failed: In #{list_file}"
-      end
+      raise ProjectsParserException.new(errors), "Schema validate failed: In #{list_file}" unless errors.empty?
 
       projects = []
       document.xpath('//project').each do |project|
@@ -23,7 +21,7 @@ module PmdTester
     end
 
     def schema_file_path
-      ResourceLocator.locate('config/projectlist_1_0_0.xsd')
+      ResourceLocator.locate('config/projectlist_1_1_0.xsd')
     end
   end
 

data/lib/pmdtester/pmd_branch_detail.rb

@@ -5,11 +5,16 @@ require 'json'
 module PmdTester
   # This class represents all details about branch of pmd
   class PmdBranchDetail
+    include PmdTester
+
     attr_accessor :branch_last_sha
     attr_accessor :branch_last_message
     attr_accessor :branch_name
     # The branch's execution time on all standard projects
     attr_accessor :execution_time
+    attr_accessor :jdk_version
+    attr_accessor :language
+    attr_accessor :pull_request
 
     def self.branch_filename(branch_name)
       branch_name&.tr('/', '_')
@@ -22,32 +27,51 @@ module PmdTester
       branch_filename = PmdBranchDetail.branch_filename(branch_name)
       @base_branch_dir = "target/reports/#{branch_filename}" unless @branch_name.nil?
       @execution_time = 0
+      # the result of command 'java -version' is going to stderr
+      @jdk_version = Cmd.stderr_of('java -version')
+      @language = ENV['LANG'] # the locale
+
+      prnum = ENV[PR_NUM_ENV_VAR]
+      @pull_request = prnum == 'false' ? nil : prnum
     end
 
-    def load
-      if File.exist?(branch_details_path)
-        hash = JSON.parse(File.read(branch_details_path))
-        @branch_last_sha = hash['branch_last_sha']
-        @branch_last_message = hash['branch_last_message']
-        @branch_name = hash['branch_name']
-        @execution_time = hash['execution_time']
-        hash
+    def self.load(branch_name, logger)
+      details = PmdBranchDetail.new(branch_name)
+      if File.exist?(details.path_to_save_file)
+        hash = JSON.parse(File.read(details.path_to_save_file))
+        details.branch_last_sha = hash['branch_last_sha']
+        details.branch_last_message = hash['branch_last_message']
+        details.branch_name = hash['branch_name']
+        details.execution_time = hash['execution_time']
+        details.jdk_version = hash['jdk_version']
+        details.language = hash['language']
+        details.pull_request = hash['pull_request']
       else
-        {}
+        details.jdk_version = ''
+        details.language = ''
+        logger&.warn "#{details.path_to_save_file} doesn't exist!"
       end
+      details
     end
 
     def save
       hash = { branch_last_sha: @branch_last_sha,
                branch_last_message: @branch_last_message,
                branch_name: @branch_name,
-               execution_time: @execution_time }
-      file = File.new(branch_details_path, 'w')
+               execution_time: @execution_time,
+               jdk_version: @jdk_version,
+               language: @language,
+               pull_request: @pull_request }
+
+      FileUtils.mkdir_p(@base_branch_dir) unless File.directory?(@base_branch_dir)
+
+      file = File.new(path_to_save_file, 'w')
       file.puts JSON.generate(hash)
       file.close
+      self
     end
 
-    def branch_details_path
+    def path_to_save_file
       "#{@base_branch_dir}/branch_info.json"
     end
 

data/lib/pmdtester/pmd_configerror.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module PmdTester
+  # This class represents a 'configerror' element of Pmd xml report
+  # and which Pmd branch the 'configerror' is from
+  class PmdConfigError
+    # The pmd branch type, 'base' or 'patch'
+    attr_reader :branch
+
+    # The schema of 'configerror' node:
+    # <xs:complexType name="configerror">
+    #   <xs:attribute name="rule" type="xs:string" use="required" />
+    #   <xs:attribute name="msg" type="xs:string" use="required" />
+    # </xs:complexType>
+    attr_reader :attrs
+    attr_accessor :old_error
+
+    def initialize(attrs, branch)
+      @attrs = attrs
+
+      @changed = false
+      @branch = branch
+    end
+
+    def rulename
+      @attrs['rule']
+    end
+
+    def msg
+      @attrs['msg']
+    end
+
+    def sort_key
+      rulename
+    end
+
+    def changed?
+      @changed
+    end
+
+    def eql?(other)
+      rulename.eql?(other.rulename) && msg.eql?(other.msg)
+    end
+
+    def try_merge?(other)
+      if branch != BASE &&
+         branch != other.branch &&
+         rulename == other.rulename &&
+         !changed? # not already changed
+        @changed = true
+        @old_error = other
+        true
+      end
+
+      false
+    end
+
+    def hash
+      [rulename, msg].hash
+    end
+  end
+end