pmdtester 1.0.1 → 1.1.0

data/config/projectlist_1_0_0.xsd

@@ -1,4 +1,5 @@
 <?xml version="1.0" ?>
+<!-- version 1.0.0 -->
 <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
   <xs:element name="projectlist">
     <xs:complexType>
@@ -25,4 +26,4 @@
       <xs:element name="exclude-pattern" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
     </xs:sequence>
   </xs:complexType>
-</xs:schema>
+</xs:schema>

data/config/projectlist_1_1_0.xsd

@@ -0,0 +1,31 @@
+<?xml version="1.0" ?>
+<!-- version 1.1.0 -->
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
+  <xs:element name="projectlist">
+    <xs:complexType>
+      <xs:sequence>
+        <xs:element name="description" type="xs:string" minOccurs="1" maxOccurs="1"/>
+        <xs:element name="project" type="project" minOccurs="1" maxOccurs="unbounded"/>
+      </xs:sequence>
+    </xs:complexType>
+  </xs:element>
+  <xs:complexType name="project">
+    <xs:sequence>
+      <xs:element name="name" type="xs:string" minOccurs="1" maxOccurs="1"/>
+      <xs:element name="type" minOccurs="1" maxOccurs="1">
+        <xs:simpleType>
+          <xs:restriction base="xs:string">
+            <xs:enumeration value="git"/>
+            <xs:enumeration value="hg"/>
+          </xs:restriction>
+        </xs:simpleType>
+      </xs:element>
+      <xs:element name="connection" type="xs:string" minOccurs="1" maxOccurs="1"/>
+      <xs:element name="webview-url" type="xs:string" minOccurs="0" maxOccurs="1"/>
+      <xs:element name="tag" type="xs:string" minOccurs="0" maxOccurs="1"/>
+      <xs:element name="exclude-pattern" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+      <xs:element name="build-command" type="xs:string" minOccurs="0" maxOccurs="1"/>
+      <xs:element name="auxclasspath-command" type="xs:string" minOccurs="0" maxOccurs="1"/>
+    </xs:sequence>
+  </xs:complexType>
+</xs:schema>

data/lib/pmdtester.rb

@@ -1,12 +1,15 @@
 # frozen_string_literal: true
 
 require 'logger'
+require 'logger/colors'
 
 require_relative 'pmdtester/cmd'
+require_relative 'pmdtester/collection_by_file'
 require_relative 'pmdtester/pmd_branch_detail'
 require_relative 'pmdtester/pmd_configerror'
 require_relative 'pmdtester/pmd_error'
 require_relative 'pmdtester/pmd_report_detail'
+require_relative 'pmdtester/pmd_tester_utils'
 require_relative 'pmdtester/pmd_violation'
 require_relative 'pmdtester/project'
 require_relative 'pmdtester/report_diff'
@@ -14,13 +17,10 @@ require_relative 'pmdtester/resource_locator'
 require_relative 'pmdtester/runner'
 
 require_relative 'pmdtester/builders/simple_progress_logger'
-require_relative 'pmdtester/builders/html_report_builder'
-require_relative 'pmdtester/builders/diff_builder'
-require_relative 'pmdtester/builders/diff_report/violations'
-require_relative 'pmdtester/builders/diff_report/configerrors'
-require_relative 'pmdtester/builders/diff_report/errors'
-require_relative 'pmdtester/builders/diff_report_builder'
+require_relative 'pmdtester/builders/project_builder'
+require_relative 'pmdtester/builders/project_hasher'
 require_relative 'pmdtester/builders/pmd_report_builder'
+require_relative 'pmdtester/builders/liquid_renderer'
 require_relative 'pmdtester/builders/rule_set_builder'
 require_relative 'pmdtester/builders/summary_report_builder'
 
@@ -32,9 +32,10 @@ require_relative 'pmdtester/parsers/projects_parser'
 # and unexpected behaviors will not be introduced to PMD project
 # after fixing an issue and new rules can work as expected.
 module PmdTester
-  VERSION = '1.0.1'
+  VERSION = '1.1.0'
   BASE = 'base'
   PATCH = 'patch'
+  PR_NUM_ENV_VAR = 'PMD_CI_PULL_REQUEST_NUMBER' # see PmdBranchDetail
 
   def logger
     PmdTester.logger

data/lib/pmdtester/builders/liquid_renderer.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require 'liquid'
+require 'json'
+
+module PmdTester
+  # A module to include in classes that use a Liquid template
+  # to generate content.
+  module LiquidRenderer
+    include PmdTester
+
+    def render_liquid(template_path, env)
+      to_render = File.read(ResourceLocator.resource(template_path))
+      includes = Liquid::LocalFileSystem.new(ResourceLocator.resource('_includes'), '%s.html')
+      Liquid::Template.file_system = includes
+      template = Liquid::Template.parse(to_render, error_mode: :strict)
+      template.render!(env, { strict_variables: true })
+    end
+
+    def render_and_write(template_path, target_file, env)
+      write_file(target_file, render_liquid(template_path, env))
+    end
+
+    def write_file(target_file, contents)
+      dir = File.dirname(target_file)
+      FileUtils.mkdir_p(dir) unless File.directory?(dir)
+
+      index = File.new(target_file, 'w')
+      index&.puts contents # may be nil when stubbing
+      logger&.info "Written #{target_file}"
+    ensure
+      index&.close
+    end
+
+    def copy_resource(dir, to_root)
+      src = ResourceLocator.resource(dir)
+      dest = "#{to_root}/#{dir}"
+      FileUtils.copy_entry(src, dest)
+    end
+  end
+
+  # Renders the index of a project diff report.
+  class LiquidProjectRenderer
+    include PmdTester
+    include ProjectHasher
+    include LiquidRenderer
+
+    def write_project_index(project, root)
+      liquid_env = {
+        'diff' => report_diff_to_h(project.report_diff),
+        'error_diffs' => errors_to_h(project),
+        'configerror_diffs' => configerrors_to_h(project),
+        'project_name' => project.name
+      }
+
+      # Renders index.html using liquid
+      write_file("#{root}/index.html", render_liquid('project_diff_report.html', liquid_env))
+      # generate array of violations in json
+      write_file("#{root}/project_data.js", dump_violations_json(project))
+    end
+
+    def dump_violations_json(project)
+      h = {
+        'source_link_base' => project.webview_url,
+        'source_link_template' => link_template(project),
+        **violations_to_hash(project)
+      }
+
+      project_data = JSON.fast_generate(h)
+      "let project = #{project_data}"
+    end
+  end
+end

data/lib/pmdtester/builders/pmd_report_builder.rb

@@ -8,92 +8,70 @@ module PmdTester
   # projects and branch of pmd source code
   class PmdReportBuilder
     include PmdTester
-    def initialize(branch_config, projects, local_git_repo, pmd_branch_name, threads = 1)
-      @branch_config = branch_config
+
+    def initialize(projects, options, branch_config, branch_name)
       @projects = projects
-      @local_git_repo = local_git_repo
-      @pmd_branch_name = pmd_branch_name
-      @threads = threads
+      @local_git_repo = options.local_git_repo
+      @threads = options.threads
+      @error_recovery = options.error_recovery
+      @branch_config = branch_config
+      @pmd_branch_name = branch_name
       @pwd = Dir.getwd
 
-      @pmd_branch_details = PmdBranchDetail.new(pmd_branch_name)
-    end
-
-    def execute_reset_cmd(type, tag)
-      case type
-      when 'git'
-        reset_cmd = "git reset --hard #{tag}"
-      when 'hg'
-        reset_cmd = "hg up #{tag}"
-      end
-
-      Cmd.execute(reset_cmd)
-    end
-
-    def clone_projects
-      logger.info 'Cloning projects started'
-
-      @projects.each do |project|
-        logger.info "Start cloning #{project.name} repository"
-        path = project.local_source_path
-        clone_cmd = "#{project.type} clone #{project.connection} #{path}"
-        if File.exist?(path)
-          logger.warn "Skipping clone, project path #{path} already exists"
-        else
-          Cmd.execute(clone_cmd)
-        end
-
-        Dir.chdir(path) do
-          execute_reset_cmd(project.type, project.tag)
-        end
-        logger.info "Cloning #{project.name} completed"
-      end
-
-      logger.info 'Cloning projects completed'
+      @pmd_branch_details = PmdBranchDetail.new(@pmd_branch_name)
+      @project_builder = ProjectBuilder.new(@projects)
     end
 
     def get_pmd_binary_file
       logger.info "#{@pmd_branch_name}: Start packaging PMD"
       Dir.chdir(@local_git_repo) do
-        current_head_sha = Cmd.execute('git rev-parse HEAD')
-        current_branch_sha = Cmd.execute("git rev-parse #{@pmd_branch_name}")
+        build_branch_sha = Cmd.execute("git rev-parse #{@pmd_branch_name}^{commit}")
+
+        checkout_build_branch # needs a clean working tree, otherwise fails
 
-        @pmd_version = determine_pmd_version
+        raise "Wrong branch #{get_last_commit_sha}" unless build_branch_sha == get_last_commit_sha
 
-        # in case we are already on the correct branch
-        # and a binary zip already exists...
-        if current_head_sha == current_branch_sha &&
-           File.exist?("pmd-dist/target/pmd-bin-#{@pmd_version}.zip")
-          logger.warn "#{@pmd_branch_name}: Skipping packaging - zip for " \
-                      "#{@pmd_version} already exists"
+        logger.debug "#{@pmd_branch_name}: PMD Version is #{@pmd_version} " \
+                     "(sha=#{build_branch_sha})"
+        distro_path = saved_distro_path(build_branch_sha)
+        if File.directory?(distro_path)
+          logger.info "#{@pmd_branch_name}: Skipping packaging - saved version exists " \
+                      " in #{distro_path}"
         else
-          build_pmd
+          build_pmd(into_dir: distro_path)
         end
 
-        @pmd_branch_details.branch_last_sha = get_last_commit_sha
+        # we're still on the build branch
+        @pmd_branch_details.branch_last_sha = build_branch_sha
         @pmd_branch_details.branch_last_message = get_last_commit_message
-
-        logger.info "#{@pmd_branch_name}: Extracting the zip"
-        unzip_cmd = "unzip -qo pmd-dist/target/pmd-bin-#{@pmd_version}.zip -d #{@pwd}/target"
-        Cmd.execute(unzip_cmd)
       end
       logger.info "#{@pmd_branch_name}: Packaging PMD completed"
     end
 
-    def build_pmd
-      logger.info "#{@pmd_branch_name}: Checking out the branch"
-      checkout_cmd = "git checkout #{@pmd_branch_name}"
-      Cmd.execute(checkout_cmd)
-
-      # determine the version again - it might be different in the other branch
-      @pmd_version = determine_pmd_version
+    # builds pmd on currently checked out branch
+    def build_pmd(into_dir:)
+      # in CI there might have been a build performed already. In that case
+      # we reuse the build result, otherwise we build PMD freshly
+      pmd_dist_target = "pmd-dist/target/pmd-bin-#{@pmd_version}.zip"
+      if File.exist?("#{@local_git_repo}/#{pmd_dist_target}")
+        # that's a warning, because we don't know, whether this build really
+        # belongs to the current branch or whether it's from a previous branch.
+        # In CI, that's not a problem, because the workspace is always fresh.
+        logger.warn "#{@pmd_branch_name}: Reusing already existing #{pmd_dist_target}"
+      else
+        logger.info "#{@pmd_branch_name}: Building PMD #{@pmd_version}..."
+        package_cmd = './mvnw clean package' \
+                      ' -Dmaven.test.skip=true' \
+                      ' -Dmaven.javadoc.skip=true' \
+                      ' -Dmaven.source.skip=true' \
+                      ' -Dcheckstyle.skip=true' \
+                      ' -T1C'
+        Cmd.execute(package_cmd)
+      end
 
-      logger.info "#{@pmd_branch_name}: Building PMD #{@pmd_version}..."
-      package_cmd = './mvnw clean package' \
-                    ' -Dmaven.test.skip=true' \
-                    ' -Dmaven.javadoc.skip=true' \
-                    ' -Dmaven.source.skip=true'
-      Cmd.execute(package_cmd)
+      logger.info "#{@pmd_branch_name}: Extracting the zip"
+      Cmd.execute("unzip -qo #{pmd_dist_target} -d pmd-dist/target/exploded")
+      Cmd.execute("mv pmd-dist/target/exploded/pmd-bin-#{@pmd_version} #{into_dir}")
     end
 
     def determine_pmd_version
@@ -103,7 +81,7 @@ module PmdTester
     end
 
     def get_last_commit_sha
-      get_last_commit_sha_cmd = 'git rev-parse HEAD'
+      get_last_commit_sha_cmd = 'git rev-parse HEAD^{commit}'
       Cmd.execute(get_last_commit_sha_cmd)
     end
 
@@ -113,27 +91,39 @@ module PmdTester
     end
 
     def generate_pmd_report(project)
-      run_path = "target/pmd-bin-#{@pmd_version}/bin/run.sh"
-      pmd_cmd = "#{run_path} pmd -d #{project.local_source_path} -f xml " \
+      error_recovery_options = @error_recovery ? 'PMD_JAVA_OPTS="-Dpmd.error_recovery -ea" ' : ''
+      run_path = "#{saved_distro_path(@pmd_branch_details.branch_last_sha)}/bin/run.sh"
+      pmd_cmd = "#{error_recovery_options}" \
+                "#{run_path} pmd -d #{project.local_source_path} -f xml " \
                 "-R #{project.get_config_path(@pmd_branch_name)} " \
                 "-r #{project.get_pmd_report_path(@pmd_branch_name)} " \
-                "-failOnViolation false -t #{@threads}"
+                "-failOnViolation false -t #{@threads} " \
+                "#{project.auxclasspath}"
       start_time = Time.now
-      Cmd.execute(pmd_cmd)
+      if File.exist?(project.get_pmd_report_path(@pmd_branch_name))
+        logger.warn "#{@pmd_branch_name}: Skipping PMD run - report " \
+                    "#{project.get_pmd_report_path(@pmd_branch_name)} already exists"
+      else
+        Cmd.execute(pmd_cmd)
+      end
       end_time = Time.now
       [end_time - start_time, end_time]
     end
 
     def generate_config_for(project)
+      logger.debug "Generating ruleset with excludes from #{@branch_config}"
       doc = Nokogiri::XML(File.read(@branch_config))
       ruleset = doc.at_css('ruleset')
+      ruleset.add_child("\n")
       project.exclude_pattern.each do |exclude_pattern|
-        ruleset.add_child("<exclude-pattern>#{exclude_pattern}</exclude-pattern>")
+        ruleset.add_child("    <exclude-pattern>#{exclude_pattern}</exclude-pattern>\n")
       end
 
       File.open(project.get_config_path(@pmd_branch_name), 'w') do |x|
         x << doc.to_s
       end
+
+      logger.debug "Created file #{project.get_config_path(@pmd_branch_name)}"
     end
 
     def generate_pmd_reports
@@ -141,16 +131,14 @@ module PmdTester
 
       sum_time = 0
       @projects.each do |project|
-        progress_logger = SimpleProgressLogger.new(project.name)
+        progress_logger = SimpleProgressLogger.new("generating #{project.name}'s PMD report")
         progress_logger.start
         generate_config_for(project)
         execution_time, end_time = generate_pmd_report(project)
         progress_logger.stop
         sum_time += execution_time
 
-        report_details = PmdReportDetail.new
-        report_details.execution_time = execution_time
-        report_details.timestamp = end_time
+        report_details = PmdReportDetail.new(execution_time: execution_time, timestamp: end_time)
         report_details.save(project.get_report_info_path(@pmd_branch_name))
         logger.info "#{project.name}'s PMD report was generated successfully"
       end
@@ -161,10 +149,46 @@ module PmdTester
       @pmd_branch_details
     end
 
+    # returns the branch details
     def build
-      clone_projects
+      @project_builder.clone_projects
+      @project_builder.build_projects
       get_pmd_binary_file
       generate_pmd_reports
     end
+
+    private
+
+    def checkout_build_branch
+      logger.info "#{@pmd_branch_name}: Checking out the branch"
+      # note that this would fail if the tree is dirty
+      Cmd.execute("git checkout #{@pmd_branch_name}")
+
+      # determine the version
+      @pmd_version = determine_pmd_version
+
+      return unless wd_has_dirty_git_changes
+
+      # working dir is dirty....
+      # we don't allow this because we need the SHA to address the zip file
+      logger.error "#{@pmd_branch_name}: Won\'t build without a clean working tree, " \
+                   'commit your changes'
+    end
+
+    def work_dir
+      "#{@pwd}/target"
+    end
+
+    # path to the unzipped distribution
+    # e.g. <cwd>/pmd-bin-<version>-<branch>-<sha>
+    def saved_distro_path(build_sha)
+      "#{work_dir}/pmd-bin-#{@pmd_version}" \
+      "-#{PmdBranchDetail.branch_filename(@pmd_branch_name)}" \
+      "-#{build_sha}"
+    end
+
+    def wd_has_dirty_git_changes
+      !Cmd.execute('git status --porcelain').empty?
+    end
   end
 end

data/lib/pmdtester/builders/project_builder.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'tempfile'
+
+module PmdTester
+  # Clones and builds the projects, that are configured in the project-list.xml
+  class ProjectBuilder
+    include PmdTester
+
+    def initialize(projects)
+      @projects = projects
+    end
+
+    def clone_projects
+      logger.info 'Cloning projects started'
+
+      @projects.each do |project|
+        logger.info "Start cloning #{project.name} repository"
+        path = project.local_source_path
+        clone_cmd = "#{project.type} clone #{project.connection} #{path}"
+        if File.exist?(path)
+          logger.warn "Skipping clone, project path #{path} already exists"
+        else
+          Cmd.execute(clone_cmd)
+        end
+
+        Dir.chdir(path) do
+          execute_reset_cmd(project.type, project.tag)
+        end
+        logger.info "Cloning #{project.name} completed"
+      end
+
+      logger.info 'Cloning projects completed'
+    end
+
+    def build_projects
+      logger.info 'Building projects started'
+
+      @projects.each do |project|
+        path = project.local_source_path
+        Dir.chdir(path) do
+          progress_logger = SimpleProgressLogger.new("building #{project.name} in #{path}")
+          progress_logger.start
+          prepare_project(project)
+          progress_logger.stop
+        end
+        logger.info "Building #{project.name} completed"
+      end
+
+      logger.info 'Building projects completed'
+    end
+
+    private
+
+    def prepare_project(project)
+      # Note: current working directory is the project directory,
+      # where the source code has been cloned to
+      if project.build_command
+        logger.debug "Executing build-command: #{project.build_command}"
+        run_as_script(Dir.getwd, project.build_command)
+      end
+      if project.auxclasspath_command
+        logger.debug "Executing auxclasspath-command: #{project.auxclasspath_command}"
+        auxclasspath = run_as_script(Dir.getwd, project.auxclasspath_command)
+        project.auxclasspath = "-auxclasspath #{auxclasspath}"
+      else
+        project.auxclasspath = ''
+      end
+    end
+
+    def run_as_script(path, command)
+      script = Tempfile.new(['pmd-regression-', '.sh'], path)
+      logger.debug "Creating script #{script.path}"
+      begin
+        script.write(command)
+        script.close
+        shell = 'sh -xe'
+        if command.start_with?('#!')
+          shell = command.lines[0].chomp[2..] # remove leading "#!"
+        end
+        stdout = Cmd.execute("#{shell} #{script.path}")
+      ensure
+        script.unlink
+      end
+      stdout
+    end
+
+    def execute_reset_cmd(type, tag)
+      case type
+      when 'git'
+        reset_cmd = "git checkout #{tag}; git reset --hard #{tag}"
+      when 'hg'
+        reset_cmd = "hg up #{tag}"
+      end
+
+      Cmd.execute(reset_cmd)
+    end
+  end
+end