pmdtester 1.6.2 → 1.7.0

data/lib/pmdtester/location.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module PmdTester
+  # This class represents a location in a source file
+  class Location
+    attr_reader :beginline, :endline, :begincolumn, :endcolumn
+
+    def initialize(beginline:, endline:, begincolumn:, endcolumn:)
+      @beginline = beginline
+      @endline = endline
+      @begincolumn = begincolumn
+      @endcolumn = endcolumn
+    end
+
+    def eql?(other)
+      beginline == other.beginline &&
+        endline == other.endline &&
+        begincolumn == other.begincolumn &&
+        endcolumn == other.endcolumn
+    end
+
+    def hash
+      [beginline, endline, begincolumn, endcolumn].hash
+    end
+
+    def to_s
+      if beginline == endline
+        if begincolumn == endcolumn
+          "#{beginline}:#{begincolumn}"
+        else
+          "#{beginline}:#{begincolumn}-#{endcolumn}"
+        end
+      else
+        "#{beginline}:#{begincolumn}-#{endline}:#{endcolumn}"
+      end
+    end
+  end
+end

data/lib/pmdtester/parsers/cpd_report_document.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+module PmdTester
+  # Parses a CPD report XML file into a set of duplications and errors
+  class CpdReportDocument < Nokogiri::XML::SAX::Document
+    attr_reader :duplications, :errors
+
+    def initialize(branch_name, working_dir)
+      super()
+      @duplications = []
+      @errors = []
+
+      @working_dir = working_dir
+      @branch_name = branch_name
+
+      @current_duplication = nil
+      @current_error = nil
+      @cur_text = String.new(capacity: 200)
+    end
+
+    def parse(file_path)
+      parser = Nokogiri::XML::SAX::Parser.new(self)
+      parser.parse(File.open(file_path)) if File.exist?(file_path)
+      self
+    end
+
+    def start_element(name, attrs = [])
+      attrs = attrs.to_h
+
+      case name
+      when 'duplication'
+        handle_start_duplication attrs
+      when 'file'
+        handle_start_file attrs
+      when 'codefragment'
+        handle_start_codefragment
+      when 'error'
+        handle_start_error attrs
+      end
+    end
+
+    def characters(string)
+      @cur_text << string
+    end
+
+    def cdata_block(string)
+      @cur_text << string
+    end
+
+    def end_element(name)
+      case name
+      when 'duplication'
+        handle_end_duplication
+      when 'codefragment'
+        handle_end_codefragment
+      when 'error'
+        handle_end_error
+      end
+    end
+
+    private
+
+    # Modifies the string in place and returns it
+    # (this is what sub! does, except it returns nil if no replacement occurred)
+    def remove_work_dir!(str)
+      str.sub!(%r{#{@working_dir}/}, '')
+      str
+    end
+
+    def finish_text!
+      remove_work_dir!(@cur_text)
+      res = @cur_text.dup.freeze
+      @cur_text.clear
+      res
+    end
+
+    def handle_start_duplication(attrs)
+      @current_duplication = { lines: attrs['lines'].to_i,
+                               tokens: attrs['tokens'].to_i,
+                               files: [],
+                               codefragment: '',
+                               branch: @branch_name }
+    end
+
+    def handle_end_duplication
+      @duplications << Duplication.new(
+        lines: @current_duplication[:lines],
+        tokens: @current_duplication[:tokens],
+        files: @current_duplication[:files],
+        codefragment: @current_duplication[:codefragment],
+        branch: @current_duplication[:branch]
+      )
+      @current_duplication = nil
+    end
+
+    def handle_start_file(attrs)
+      return if @current_duplication.nil?
+
+      file_info = DuplicationFileInfo.new(
+        path: remove_work_dir!(attrs['path']),
+        location: Location.new(
+          beginline: attrs['line'].to_i,
+          endline: attrs['endline'].to_i,
+          begincolumn: attrs['column'].to_i,
+          endcolumn: attrs['endcolumn'].to_i
+        ),
+        begintoken: attrs['begintoken'].to_i,
+        endtoken: attrs['endtoken'].to_i
+      )
+      @current_duplication[:files] << file_info
+    end
+
+    def handle_start_codefragment
+      @cur_text.clear
+    end
+
+    def handle_end_codefragment
+      @current_duplication[:codefragment] = finish_text!
+    end
+
+    def handle_start_error(attrs)
+      @current_error = PmdError.new(
+        filename: remove_work_dir!(attrs['filename']),
+        short_message: remove_work_dir!(attrs['msg']),
+        branch: @branch_name
+      )
+      @cur_text.clear
+    end
+
+    def handle_end_error
+      @current_error.stack_trace = finish_text!
+      @errors << @current_error
+      @current_error = nil
+    end
+  end
+
+  # Represents a duplication entry in a CPD report
+  class Duplication
+    attr_reader :lines, :tokens, :files, :codefragment, :branch, :old_lines, :old_tokens, :old_files, :old_codefragment
+
+    def initialize(lines:, tokens:, files:, codefragment:, branch:)
+      @lines = lines
+      @tokens = tokens
+      @files = files
+      @codefragment = codefragment
+      @branch = branch
+
+      @changed = false
+      @old_lines = nil
+      @old_tokens = nil
+      @old_files = nil
+      @old_codefragment = nil
+    end
+
+    def eql?(other)
+      return false unless other.is_a?(Duplication)
+
+      lines == other.lines &&
+        tokens == other.tokens &&
+        files.eql?(other.files) &&
+        codefragment == other.codefragment
+    end
+
+    def hash
+      [lines, tokens, files, codefragment].hash
+    end
+
+    def try_merge?(other)
+      if branch != BASE && branch != other.branch &&
+         !changed? && # not already changed
+         same_or_similar_locations?(other.files)
+        @changed = true
+        @old_lines = other.lines
+        @old_tokens = other.tokens
+        @old_files = other.files
+        @old_codefragment = other.codefragment
+        true
+      else
+        false
+      end
+    end
+
+    # only makes sense if this is a diff
+    def added?
+      branch != BASE && !changed?
+    end
+
+    # only makes sense if this is a diff
+    def changed?
+      @changed
+    end
+
+    # only makes sense if this is a diff
+    def removed?
+      branch == BASE
+    end
+
+    private
+
+    def same_or_similar_locations?(other_files)
+      files.each do |file_info|
+        other_files.each do |other_file_info|
+          return true if file_info.path == other_file_info.path &&
+                         (file_info.location.eql?(other_file_info.location) ||
+                          location_move?(file_info.location, other_file_info.location))
+        end
+      end
+      false
+    end
+
+    def location_move?(this_location, other_location)
+      (this_location.beginline - other_location.beginline).abs <= 5
+    end
+  end
+
+  # Represents a single file location of a duplication in a CPD report
+  class DuplicationFileInfo
+    attr_reader :path, :location, :begintoken, :endtoken
+
+    def initialize(path:, location:, begintoken:, endtoken:)
+      @path = path
+      @location = location
+      @begintoken = begintoken
+      @endtoken = endtoken
+    end
+
+    def eql?(other)
+      return false unless other.is_a?(DuplicationFileInfo)
+
+      path == other.path &&
+        location.eql?(other.location) &&
+        begintoken == other.begintoken &&
+        endtoken == other.endtoken
+    end
+
+    def hash
+      [path, location, begintoken, endtoken].hash
+    end
+  end
+end

data/lib/pmdtester/parsers/options.rb

@@ -5,6 +5,7 @@ require 'slop'
 module PmdTester
   class MissRequiredOptionError < StandardError; end
   class InvalidModeError < StandardError; end
+  class InvalidOptionError < StandardError; end
 
   # The Options is a class responsible of parsing all the
   # command line options
@@ -36,6 +37,8 @@ module PmdTester
     attr_reader :keep_reports
     attr_reader :error_recovery
     attr_reader :baseline_download_url_prefix
+    attr_reader :run_cpd
+    attr_reader :run_pmd
 
     def initialize(argv)
       options = parse(argv)
@@ -56,6 +59,8 @@ module PmdTester
       @keep_reports = options.keep_reports?
       @error_recovery = options.error_recovery?
       url = options[:baseline_download_url]
+      @run_cpd = !options.no_cpd?
+      @run_pmd = !options.no_pmd?
       @baseline_download_url_prefix = if url[-1] == '/'
                                         url
                                       else
@@ -116,6 +121,10 @@ module PmdTester
         o.string '--baseline-download-url',
                  'download url prefix from where to download the baseline in online mode',
                  default: DEFAULT_BASELINE_URL_PREFIX
+        o.bool '--no-cpd', 'do not execute CPD (Copy Paste Detector) and compare duplications; only execute PMD',
+               default: false
+        o.bool '--no-pmd', 'do not execute PMD and compare rule violations; only execute CPD (Copy Paste Detector)',
+               default: false
         o.on '-v', '--version' do
           puts VERSION
           exit
@@ -129,6 +138,7 @@ module PmdTester
 
     def check_options
       check_common_options
+      check_pmd_cpd_options
       case @mode
       when LOCAL
         check_local_options
@@ -164,6 +174,15 @@ module PmdTester
       check_option(ANY, 'patch branch name', @patch_branch)
     end
 
+    def check_pmd_cpd_options
+      return unless !@run_pmd && !@run_cpd
+
+      msg = 'Both "--no-cpd" and "--no-pmd" are given. At least one of PMD and CPD must be executed. ' \
+            'Please check your options.'
+      logger.error msg
+      raise InvalidOptionError, msg
+    end
+
     def check_option(mode, option_name, option)
       if option.nil?
         msg = "#{option_name} is required in #{mode} mode."

data/lib/pmdtester/parsers/pmd_report_document.rb

@@ -10,6 +10,7 @@ module PmdTester
     attr_reader :configerrors
 
     def initialize(branch_name, working_dir, filter_set = nil)
+      super()
       @violations = CollectionByFile.new
       @errors = CollectionByFile.new
       @configerrors = Hash.new { |hash, key| hash[key] = [] }
@@ -25,6 +26,12 @@ module PmdTester
       @cur_text = String.new(capacity: 200)
     end
 
+    def parse(file_path)
+      parser = Nokogiri::XML::SAX::Parser.new(self)
+      parser.parse(File.open(file_path)) if File.exist?(file_path)
+      self
+    end
+
     def start_element(name, attrs = [])
       attrs = attrs.to_h
 
@@ -109,7 +116,13 @@ module PmdTester
         info_url: attrs['externalInfoUrl'],
         bline: attrs['beginline'].to_i,
         rule_name: attrs['rule'],
-        ruleset_name: attrs['ruleset'].freeze
+        ruleset_name: attrs['ruleset'].freeze,
+        location: Location.new(
+          beginline: attrs['beginline'].to_i,
+          endline: attrs['endline'].to_i,
+          begincolumn: attrs['begincolumn'].to_i,
+          endcolumn: attrs['endcolumn'].to_i
+        )
       )
     end
 

data/lib/pmdtester/parsers/projects_parser.rb

@@ -17,7 +17,7 @@ module PmdTester
     end
 
     def schema_file_path
-      ResourceLocator.locate('config/projectlist_1_2_0.xsd')
+      ResourceLocator.locate('config/projectlist_1_3_0.xsd')
     end
   end
 

data/lib/pmdtester/pmd_branch_detail.rb

@@ -16,6 +16,9 @@ module PmdTester
     attr_accessor :execution_time
     attr_accessor :jdk_version
     attr_accessor :language
+    attr_accessor :cpu_info
+    attr_accessor :physical_memory
+    attr_accessor :os_info
     attr_accessor :pull_request
 
     def self.branch_filename(branch_name)
@@ -33,6 +36,10 @@ module PmdTester
       # the result of command 'java -version' is going to stderr
       @jdk_version = Cmd.stderr_of('java -version')
       @language = ENV.fetch('LANG') # the locale
+      system_info = PmdTester::SystemInfo.new
+      @cpu_info = system_info.cpu_info
+      @physical_memory = system_info.physical_memory
+      @os_info = system_info.uname
 
       prnum = ENV.fetch(PR_NUM_ENV_VAR, 'false')
       @pull_request = prnum == 'false' ? nil : prnum
@@ -41,19 +48,14 @@ module PmdTester
     def self.load(branch_name, logger)
       details = PmdBranchDetail.new(branch_name)
       if File.exist?(details.path_to_save_file)
-        hash = JSON.parse(File.read(details.path_to_save_file))
-        details.branch_last_sha = hash['branch_last_sha']
-        details.branch_last_message = hash['branch_last_message']
-        details.branch_name = hash['branch_name']
-        details.timestamp = hash['timestamp']
-        details.execution_time = hash['execution_time']
-        details.jdk_version = hash['jdk_version']
-        details.language = hash['language']
-        details.pull_request = hash['pull_request']
+        details.parse_from_json
       else
         details.timestamp = Time.now
         details.jdk_version = ''
         details.language = ''
+        details.cpu_info = ''
+        details.physical_memory = ''
+        details.os_info = ''
         logger&.warn "#{details.path_to_save_file} doesn't exist!"
       end
       details
@@ -66,6 +68,9 @@ module PmdTester
                timestamp: @timestamp,
                execution_time: @execution_time,
                jdk_version: @jdk_version,
+               cpu_info: @cpu_info,
+               physical_memory: @physical_memory,
+               os_info: @os_info,
                language: @language,
                pull_request: @pull_request }
 
@@ -92,5 +97,20 @@ module PmdTester
     def format_execution_time
       PmdReportDetail.convert_seconds(@execution_time)
     end
+
+    def parse_from_json
+      hash = JSON.parse(File.read(path_to_save_file))
+      @branch_last_sha = hash['branch_last_sha']
+      @branch_last_message = hash['branch_last_message']
+      @branch_name = hash['branch_name']
+      @timestamp = hash['timestamp']
+      @execution_time = hash['execution_time']
+      @jdk_version = hash['jdk_version']
+      @language = hash['language']
+      @cpu_info = hash['cpu_info']
+      @physical_memory = hash['physical_memory']
+      @os_info = hash['os_info']
+      @pull_request = hash['pull_request']
+    end
   end
 end

data/lib/pmdtester/pmd_report_detail.rb

@@ -3,29 +3,32 @@
 require 'json'
 
 module PmdTester
-  # This class represents all details about report of pmd
+  # This class represents all details about an execution of PMD
   class PmdReportDetail
     attr_accessor :execution_time
     attr_accessor :timestamp
     attr_accessor :working_dir
+    attr_accessor :cmdline
     attr_accessor :exit_code
-
-    def initialize(execution_time: 0, timestamp: '', working_dir: Dir.getwd, exit_code: nil)
-      @execution_time = execution_time
-      @timestamp = timestamp
-      @working_dir = working_dir
-      @exit_code = exit_code.nil? ? '?' : exit_code.to_s
-    end
+    attr_accessor :stdout
+    attr_accessor :stderr
+    attr_accessor :oom
+    attr_accessor :jfr_summary
 
     def save(report_info_path)
       hash = {
         execution_time: @execution_time,
         timestamp: @timestamp,
         working_dir: @working_dir,
-        exit_code: @exit_code
+        cmdline: @cmdline,
+        exit_code: @exit_code,
+        stdout: @stdout,
+        stderr: @stderr,
+        oom: @oom,
+        jfr_summary: @jfr_summary.to_h
       }
       file = File.new(report_info_path, 'w')
-      file.puts JSON.generate(hash)
+      file.puts JSON.pretty_generate(hash)
       file.close
     end
 
@@ -39,20 +42,58 @@ module PmdTester
       end
     end
 
-    def format_execution_time
+    def execution_time_formatted
       self.class.convert_seconds(@execution_time)
     end
 
-    def self.create(execution_time: 0, timestamp: '', working_dir: Dir.getwd, exit_code: nil, report_info_path:)
+    def to_h
+      {
+        'timestamp' => @timestamp,
+        'exit_code' => @exit_code,
+        'cmdline' => @cmdline,
+        'execution_time' => execution_time_formatted,
+        'oom' => @oom,
+        'jfr_summary' => @jfr_summary.to_h_for_liquid
+      }
+    end
+
+    def self.create(execution_time: 0, timestamp: '', working_dir: Dir.getwd, cmdline: '',
+                    exit_code: nil, stdout: '', stderr: '', oom: false,
+                    report_info_path:, jfr_summary: nil)
       detail = PmdReportDetail.new(execution_time: execution_time, timestamp: timestamp,
-                                   working_dir: working_dir, exit_code: exit_code)
+                                   working_dir: working_dir, cmdline: cmdline, oom: oom,
+                                   exit_code: exit_code, stdout: stdout, stderr: stderr, jfr_summary: jfr_summary)
       detail.save(report_info_path)
       detail
     end
 
+    def self.empty
+      new(execution_time: 0, timestamp: '', working_dir: Dir.getwd, cmdline: '',
+          exit_code: nil, stdout: '', stderr: '', oom: false)
+    end
+
     # convert seconds into HH::MM::SS
     def self.convert_seconds(seconds)
       Time.at(seconds.abs).utc.strftime('%H:%M:%S')
     end
+
+    private
+
+    def initialize(execution_time: 0, timestamp: '', working_dir: Dir.getwd, cmdline: '',
+                   exit_code: nil, stdout: '', stderr: '', oom: false, jfr_summary: nil)
+      @execution_time = execution_time
+      @timestamp = timestamp
+      @working_dir = working_dir
+      @cmdline = cmdline
+      @exit_code = exit_code.nil? ? '?' : exit_code.to_s
+      @stdout = stdout
+      @stderr = stderr
+      @oom = oom
+      @jfr_summary = if jfr_summary.instance_of? JfrSummary
+                       jfr_summary
+                     else
+                       JfrSummary.from_h(jfr_summary || {})
+                     end
+    end
   end
 end

data/lib/pmdtester/pmd_tester_utils.rb

@@ -7,12 +7,17 @@ module PmdTester
 
     # Parse the base and the patch report, compute their diff
     # Returns a +ReportDiff+
-    def build_report_diff(base_report_file, patch_report_file, base_info, patch_info, filter_set = nil)
+    def build_report_diff(base_report_file, patch_report_file, base_info, patch_info, filter_set = nil,
+                          rules_changed: true)
       base_details = PmdReportDetail.load(base_info)
-      patch_details = PmdReportDetail.load(patch_info)
-
       base_report = parse_pmd_report(base_report_file, BASE, base_details, filter_set)
-      patch_report = parse_pmd_report(patch_report_file, PATCH, patch_details)
+
+      if rules_changed
+        patch_details = PmdReportDetail.load(patch_info)
+        patch_report = parse_pmd_report(patch_report_file, PATCH, patch_details)
+      else
+        patch_report = base_report
+      end
 
       logger.info 'Calculating diffs'
       ReportDiff.new(base_report: base_report, patch_report: patch_report)
@@ -21,35 +26,58 @@ module PmdTester
     # Parse the +report_file+ to produce a +Report+.
     # For the schema of xml reports, refer to https://pmd.github.io/schema/report_2_0_0.xsd
     def parse_pmd_report(report_file, branch, report_details, filter_set = nil)
-      require 'nokogiri'
-
-      logger.info "Parsing #{report_file}"
+      logger.info "Parsing PMD Report #{report_file}"
       doc = PmdReportDocument.new(branch, report_details.working_dir, filter_set)
-      parser = Nokogiri::XML::SAX::Parser.new(doc)
-      parser.parse_file(report_file) if File.exist?(report_file)
+                             .parse(report_file)
       Report.new(
+        report_details: report_details,
         report_document: doc,
-        file: report_file,
+        file: report_file
+      )
+    end
 
-        timestamp: report_details.timestamp,
-        exec_time: report_details.execution_time,
-        exit_code: report_details.exit_code
+    # Parse the base and the patch CPD report, compute their diff
+    # Returns a +CpdReportDiff+
+    def build_cpd_report_diff(base_report_file, patch_report_file, base_info, patch_info, impl_changed:)
+      base_details = PmdReportDetail.load(base_info)
+      base_report = parse_cpd_report(base_report_file, BASE, base_details)
+
+      if impl_changed
+        patch_details = PmdReportDetail.load(patch_info)
+        patch_report = parse_cpd_report(patch_report_file, PATCH, patch_details)
+      else
+        patch_report = base_report
+      end
+
+      logger.info 'Calculating CPD diffs'
+      CpdReportDiff.new(base_report: base_report, patch_report: patch_report)
+    end
+
+    def parse_cpd_report(report_file, branch, report_details)
+      logger.info "Parsing CPD Report #{report_file}"
+      doc = CpdReportDocument.new(branch, report_details.working_dir).parse(report_file)
+      CpdReport.new(
+        report_details: report_details,
+        report_document: doc,
+        file: report_file
       )
     end
 
     # Fill the report_diff field of every project
-    def compute_project_diffs(projects, base_branch, patch_branch, filter_set = nil)
+    def compute_project_diffs(projects, base_branch, patch_branch, filter_set, rules_changed:, impl_changed:)
       projects.each do |project|
         logger.info "Preparing report for #{project.name}"
         logger.info "  with filter #{filter_set}" unless filter_set.nil?
-        project.compute_report_diff(base_branch, patch_branch, filter_set)
+        project.compute_report_diff(base_branch, patch_branch, filter_set,
+                                    rules_changed: rules_changed, impl_changed: impl_changed)
       end
     end
 
     # Build the diff reports and write them all
-    def build_html_reports(projects, base_branch_details, patch_branch_details, filter_set = nil)
+    def build_html_reports(projects, base_branch_details, patch_branch_details, filter_set,
+                           rules_changed:, impl_changed:)
       compute_project_diffs(projects, base_branch_details.branch_name, patch_branch_details.branch_name,
-                            filter_set)
+                            filter_set, rules_changed: rules_changed, impl_changed: impl_changed)
 
       SummaryReportBuilder.new.write_all_projects(projects,
                                                   base_branch_details,