simp-rake-helpers 5.1.4 → 5.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3172c665fe5f42370b01ef6a03b7530da556471e89a7b55a90e23b8cc5c899e5
-  data.tar.gz: 377fad7a7d2acccd401c1cf17e52548144a27bbcd220618e0caca66e7e3cadea
+  metadata.gz: 476fb45c0548c83de14eec268b1970b990a0c50ef8ead160f502bb3c34fa3bd6
+  data.tar.gz: 0d6dde21abf6befaf136b973f324d2cd812f0d36d49c0a6e48971499b0487802
 SHA512:
-  metadata.gz: fa1a4fc7b92292ceb0908368d43768ca70ec77ff31efd8e9d5c4def2dec3b90c213066f9843eaf258b070e4cfe53aeae8f0066cad24a062b41992e3da005f3f2
-  data.tar.gz: 3dd45d6438797ba3a8173ee33b511f90319725b5602930f3e75068bdc4002d485e58b4a58dbcdc684dc46f7c6af1dcd53766c39f44aae95b9c036a8cee976d74
+  metadata.gz: f090fd69b70d38e94f0a71ba57d3e76782b10a7ef6e7987c8a4b8665b17a29b2b5c9b7065c0b150979544f09203b8ac385fea194b2bf5672b44499868302a9e0
+  data.tar.gz: 3671a8e5759091753650ca03fcddfbc68ada6c85fcb12e6c33e6d803e3b9c3d78eafbcf858ef828632ba085d9fe831abd3d4d124227e83afaf7a408c8f310a6c

data/CHANGELOG.md

@@ -1,3 +1,21 @@
+### 5.2.0 / 2017-12-20
+* Create pkg:create_tag_changelog, which is a more thorough version
+  of Simp::Rake::Pupmod::Helpers changelog_annotation task.
+  - Now supports non-Puppet SIMP assets for which version and changelog
+    information is specified in an RPM spec files.
+  - Provides more extensive validation of date strings and changelog
+    entry ordering.
+  - Stops processing at the first invalid changelog entry, to minimize
+    non-catestrophic errors from old changelog entries.
+* Create pkg:compare_latest_tag, which is a more general replacement
+  for the Simp::Rake::Pupmod::Helpers compare_latest_tag task.
+  - Now supports non-Puppet SIMP assets for which version and changelog
+    information is specified in an RPM spec files.
+  - Does the same validation as the new pkg:create_tag_changelog task.
+* Fix broken acceptance tests 
+  - Remove logic to build SIMP 4 and SIMP 5 RPMs.
+  - Remove mock logic
+
 ### 5.1.4 / 2017-11-27
 * Switch back to using Gem::Version.new instead of Puppet's vercmp since
   Gem::Version matches the standard RPM version semantics and Puppet does not.

data/lib/simp/componentinfo.rb

@@ -0,0 +1,227 @@
+require 'date'
+
+module Simp; end
+
+# Class that provides component version, release, and changelog
+# information
+class Simp::ComponentInfo
+  attr_accessor :component_dir, :type, :version, :release, :changelog
+
+  # A helpful method for ensuring that the errors can be easily seen
+  ERR_MARKER = "WARNING: !!! "
+
+  # See https://fedoraproject.org/wiki/Packaging:Guidelines?rd=Packaging/Guidelines#Changelogs
+  # When matched against this regex
+  #   match 1 = date of the form {weekday} {month} {day} {year}
+  #   match 2 = author of the form {name} <{email}>
+  #   match 3 = version
+  #   match 4 = optional release qualifier; nil when absent
+  CHANGELOG_ENTRY_REGEX = /^\*\s+((?:Mon|Tue|Wed|Thu|Fri|Sat|Sun) (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [0-3]{1}[0-9]{1} \d{4})\s+(.+<.+>)(?:\s+|\s*-\s*)(\d+\.\d+\.\d+)(?:-(\S+))?\s*$/
+
+  # Load component information from appropriate component files
+  #
+  # Module
+  #   version:     Top-level 'version' key from the metadata.json file
+  #   release:     Not set
+  #   changelog:   Array of valid changelog entries derived from the
+  #                CHANGELOG file
+  #
+  # Asset
+  #   version:     Primary Version tag from build/<component>.spec
+  #   release:     Primary Release tag from build/<component>.spec
+  #   changelog:   Array of valid changelog entries derived from the
+  #                contents of the %changelog section in
+  #                build/<component>.spec.  Will be an empty if
+  #                %changelog is not present.
+  #
+  #  NOTES:
+  #  1.  The changelog is only parsed up to the first entry that
+  #      fails basic validation.
+  #      - First line must be of the form
+  #         * {date string} {author info} - {version}
+  #         * {date string} {author info} - {version}-{release}
+  #
+  #        where,
+  #         date string =  {weekday} {month} {day} {year}
+  #         author info =  {author name} <{author email}>
+  #      - Weekday must be correct for the specified date
+  #      - Entries must be separated by a blank line
+  #
+  #      NOTE: This currently does not support the valid RPM `%changelog`
+  #       format that places the version number on the next line:
+  #
+  #       * Fri Mar 02 2012 Maintenance
+  #       4.0.0-2
+  #       - Improved test stubs.
+  #
+  #       However, since we are not using this form for recent
+  #       changelogs and we stop processing upon reaching such
+  #       a changelog entry, this should *not* be an issue.
+  #  2.  When RPM spec files contain sub-packages, only the primary
+  #      package information is returned.
+  #  3.  Some assets have another version in a lib/.../version.rb.
+  #      Since there is no definitive way for this code to determine
+  #      that version, it will not be loaded here.
+  #
+  # Fails if any of the following occur:
+  # - The metadata.json file for a Puppet module component cannot be
+  #   parsed.
+  # - A top-level 'version' key does not exist in the metadata.json file.
+  # - The CHANGELOG file for a Puppet module component does not exist.
+  # - The RPM spec file for a non-Puppet module component does not exist.
+  # - More than 1 RPM spec file for a non-Puppet module component exists.
+  # - The version, release or changelog cannot be extracted from the RPM
+  #   spec file for a non-Puppet module.
+  #  - Any changelog entry below the first entry has a version greater
+  #   than that of the first entry.  Changelog entries must be
+  #   ordered from latest version to earliest version.
+  # - The changelog entries are out of date order.
+  #
+  # +component_dir+:: The root directory of the component project.
+  # +latest_version_only+:: Whether to only return the changelog
+  #  entries for  the latest version
+  # +verbose+:: Whether to log a changelog validation failure
+  #
+  def initialize(component_dir, latest_version_only = false, verbose = true)
+    @component_dir = component_dir
+
+    if File.exists?(File.join(@component_dir, 'metadata.json'))
+      @type = :module
+      load_module_info(latest_version_only, verbose)
+    else
+      @type = :asset
+      load_asset_info(latest_version_only, verbose)
+    end
+  end
+
+  private
+  def load_module_info(latest_version_only, verbose)
+    require 'json'
+    metadata_file = File.join(@component_dir, 'metadata.json')
+    metadata = JSON.parse(File.read(metadata_file))
+    @version = metadata['version']
+    @release = nil
+    fail("ERROR: Version missing from #{metadata_file}") if @version.nil?
+
+    changelog_file = File.join(component_dir, 'CHANGELOG')
+    unless File.exists?(changelog_file)
+      fail("ERROR: No CHANGELOG file found in #{component_dir}")
+    end
+    @changelog = parse_changelog(IO.read(changelog_file), latest_version_only, verbose)
+  end
+
+  def load_asset_info(latest_version_only, verbose)
+    rpm_spec_files = Dir.glob(File.join(@component_dir, 'build', '*.spec'))
+    if rpm_spec_files.empty?
+      fail("No RPM spec file found in #{File.join(@component_dir, 'build')}")
+    elsif rpm_spec_files.size > 1
+      fail("More than 1 RPM spec file found: #{rpm_spec_files.join(' ')}")
+    end
+
+    # Determine asset version, which we will ASSUME to be the main
+    # package version.  The RPM query, below, will return the main
+    # package followed by subpackages.
+    version_query = "rpm -q --queryformat '%{VERSION} %{RELEASE}\\n'" +
+      " --specfile #{rpm_spec_files[0]}"
+
+    rpm_version_list = `#{version_query} 2> /dev/null`
+    if $?.exitstatus != 0
+      fail("Could not extract version and release from #{rpm_spec_files[0]}." +
+        " To debug, execute:\n   #{version_query}")
+    end
+    @version, @release = rpm_version_list.split("\n")[0].split
+
+    changelog_query = "rpm -q --changelog --specfile #{rpm_spec_files[0]}"
+    raw_changelog = `#{changelog_query} 2> /dev/null`
+    if $?.exitstatus != 0
+      fail("Could not extract changelog from #{rpm_spec_files[0]}." +
+        " To debug, execute:\n   #{changelog_query}")
+    end
+    @changelog = parse_changelog(raw_changelog, latest_version_only, verbose)
+  end
+
+  # Return an array of changelog entries, optionally for only the
+  # latest version
+  #
+  # Iterates through the changelog entries from the newest to the
+  # oldest, performing basic validation.  Stops processing entries
+  # if an entry fails validation.
+  #
+  def parse_changelog(changelog, latest_version_only, verbose)
+    # split on the entry-separating lines
+    changelog_entries = changelog.split(/^\s*$/)
+    latest_version = nil # 1st version found is latest version
+    prev_entry_date = nil
+    changelogs = []
+    changelog_entries.each do |entry|
+      # split each entry into lines, removing the initial, empty line
+      # that occurs on all but the first entry
+      changelog_lines = entry.split("\n").delete_if { |line| line.empty? }
+      match = CHANGELOG_ENTRY_REGEX.match(changelog_lines[0])
+      if match.nil?
+        warn "WARNING: Parsing stopped at invalid changelog entry: \n#{entry}" if verbose
+        break
+      else
+        # verify 1st version is latest version
+        # NOTE:  There are edge cases in which comparisons between
+        # versions with and without release qualifiers may give answers
+        # that are not expected.  For example, '6.2.0' > '6.2.0-1'.
+        full_version = match[3]
+        full_version += "-#{match[4]}" unless match[4].nil?
+        current_version = Gem::Version.new(full_version)
+        latest_version = current_version if latest_version.nil?
+
+        if current_version > latest_version
+          fail("ERROR:  Changelog entries are not properly version ordered")
+        end
+
+        break if latest_version_only and (current_version < latest_version)
+
+        # verify dates are appropriately ordered (newest to oldest)
+        current_entry_date = Date.strptime(match[1], '%a %b %d %Y')
+        prev_entry_date = current_entry_date if prev_entry_date.nil?
+        if current_entry_date > prev_entry_date
+          fail("ERROR:  Changelog entries are not properly date ordered")
+        end
+
+        if valid_date_weekday?(match[1], verbose)
+          entry = {
+            :date    => match[1],
+            :version => match[3],
+            :release => match[4],
+            :content => changelog_lines
+          }
+          changelogs << entry
+        else
+          warn "WARNING: Parsing stopped at invalid changelog entry: \n#{entry}" if verbose
+          break
+        end
+      end
+    end
+
+    changelogs
+  end
+
+  # Validate the weekday in the already-format-verified changelog
+  # date string is correct for the date specified
+  #
+  # Returns false if the weekday is incorrect for date specified.
+  #
+  # +changelog_date+:: Date string of the form <weekday> <month> <day> <year>
+  # +verbose+:: Whether to log details about a weekday validation failure
+  def valid_date_weekday?(changelog_date, verbose)
+    date = Date.strptime(changelog_date, '%a %b %d %Y')
+    expected_weekday = date.strftime('%a')
+    actual_weekday = changelog_date.strip.split[0]
+
+    valid = true
+    if actual_weekday != expected_weekday
+      err_msg = ERR_MARKER + "'#{actual_weekday}' should be '#{expected_weekday}' for" +
+        " changelog timestamp '#{changelog_date}'"
+      warn err_msg if verbose
+      valid = false
+    end
+    return valid
+  end
+
+end

data/lib/simp/rake/helpers/version.rb

@@ -2,5 +2,5 @@ module Simp; end
 module Simp::Rake; end
 
 class Simp::Rake::Helpers
-  VERSION = '5.1.4'
+  VERSION = '5.2.0'
 end

data/lib/simp/rake/pkg.rb

@@ -6,6 +6,7 @@ require 'rake/clean'
 require 'rake/tasklib'
 require 'fileutils'
 require 'find'
+require 'simp/relchecks'
 require 'simp/rpm'
 require 'simp/rake/helpers/rpm_spec'
 
@@ -106,6 +107,8 @@ module Simp::Rake
       define_pkg_tar
       define_pkg_rpm
       define_pkg_check_version
+      define_pkg_compare_latest_tag
+      define_pkg_create_tag_changelog
       task :default => 'pkg:tar'
 
       Rake::Task['pkg:tar']
@@ -463,6 +466,118 @@ module Simp::Rake
       end
     end
 
+    def define_pkg_compare_latest_tag
+      namespace :pkg do
+        desc <<-EOM
+        Compare to latest tag.
+          ARGS:
+            * :tags_source => Set to the remote from which the tags for this
+                              project can be fetched. Defaults to 'origin'.
+            * :verbose => Set to 'true' if you want to see detailed messages
+
+          NOTES:
+          Compares mission-impacting (significant) files with the latest
+          tag and identifies the relevant files that have changed.
+
+          Fails if
+          (1) There is any version validation or changelog parsing failure
+              that would prevent an annotated changelog tag from being
+              created. (See pkg::create_tag_changelog)
+          (2) A version bump is required but not recorded in both the
+              CHANGELOG and metadata.json files.
+          (3) The latest version is < latest tag.
+
+          Changes to the following files/directories are not considered
+          significant:
+          - Any hidden file/directory (entry that begins with a '.')
+          - Gemfile
+          - Gemfile.lock
+          - Rakefile
+          - spec directory
+          - doc directory
+        EOM
+        task :compare_latest_tag, [:tags_source, :verbose] do |t,args|
+          tags_source = args[:tags_source].nil? ? 'origin' : args[:tags_source]
+          if args[:verbose].to_s == 'true'
+            verbose = true
+          else
+            verbose = false
+          end
+          Simp::RelChecks::compare_latest_tag(@base_dir, tags_source, verbose)
+        end
+      end
+    end
+
+    def define_pkg_create_tag_changelog
+      namespace :pkg do
+        # :pkg:create_tag_changelog
+        # -----------------------------
+        desc <<-EOM
+        Generate an appropriate changelog for an annotated tag from a
+        component's CHANGELOG or RPM spec file.  
+
+        The changelog text will be for the latest version and contain
+        1 or more changelog entries for that version, in reverse
+        chronological order.
+
+        ARGS:
+          * verbose => Set to 'true' if you want to see
+            non-catestrophic warning messages.
+
+        NOTES:
+          * Changelog entries must follow the following rules:
+            - An entry must start with * and be terminated
+              by a blank line.
+            - The first line must be of the form
+              * Wed Jul 05 2017 Author Name <author@simp.com> - 1.2.3-4
+            - The date string must be RPM compatible.
+            - Dates must be in reverse chronological order, with the
+              newest dates occurring at the top of the changelog.
+            - Both an author name and email are required.
+            - The author email must be contained in < >.
+            - The version is required and must be of the form
+              <major>.<minor>.<patch>.
+            - The version may contain a release qualifier.
+            - When the release qualifier is present, it must appear
+              at the end of the version string and be separated from
+              the version by a '-'.
+
+          * This task will fail if any of the following occur:
+            - The metadata.json file for a Puppet module component
+              cannot be parsed.
+            - The CHANGELOG file for a Puppet module component does
+              not exist.
+            - The CHANGELOG entries for the latest version are
+              malformed.
+            - The RPM spec file or a non-Puppet module component does
+              not exist.
+            - More than 1 RPM spec file for a non-Puppet module
+              component exists.
+            - No valid changelog entries for the version specified in
+              the metadata.json/spec file are found.
+            - The latest changelog version is greater than the version
+              in the metadata.json or the RPM spec file.
+            - The RPM release specified in the spec file does not match
+              the release in a changelog entry for the version.
+            - Any changelog entry below the first entry has a version
+              greater than that of the first entry.
+            - The changelog entries for the latest version are out of
+              date order.
+            - The weekday for a changelog entry for the latest version
+              does not match the date specified.
+
+        EOM
+        task :create_tag_changelog, [:verbose] do |t,args|
+          if args[:verbose].to_s == 'true'
+            verbose = true
+          else
+            verbose = false
+          end
+          puts Simp::RelChecks::create_tag_changelog(@base_dir, verbose)
+        end
+      end
+    end
+
     # ------------------------------------------------------------------------------
     # helper methods
     # ------------------------------------------------------------------------------

data/lib/simp/rake/pupmod/helpers.rb

@@ -251,9 +251,9 @@ class Simp::Rake::Pupmod::Helpers < ::Rake::TaskLib
             if curr_module_version < last_tag_version
               fail("ERROR: Version regression. '#{module_version}' < last tag '#{last_tag}'")
             elsif curr_module_version == last_tag_version
-              fail("ERROR: Version update beyond last tag '#{last_tag}' is required for changes to #{files_changed}")
+              fail("ERROR: Version update beyond last tag '#{last_tag}' is required for #{files_changed.count} changed files:\n  * #{files_changed.join("\n  * ")}")
             else
-              puts "  New tag of version '#{module_version}' is required for changes to #{files_changed}"
+              puts "NOTICE: New tag of version '#{module_version}' is required for #{files_changed.count} changed files:\n  * #{files_changed.join("\n  * ")}"
             end
           end
         end

data/lib/simp/relchecks.rb

@@ -0,0 +1,172 @@
+require 'date'
+require 'simp/componentinfo'
+
+module Simp; end
+
+# Class that provide release-related checks
+class Simp::RelChecks
+
+  # Compares mission-impacting (significant) files with the latest
+  # tag and identifies the relevant files that have changed.
+  #
+  # Fails if
+  # (1) There is any version validation or changelog parsing failure
+  #     that would prevent an annotated changelog tag from being
+  #     created. (See Simp::RelCheck::create_tag_changelog)
+  # (2) A version bump is required but not recorded in both the
+  #     CHANGELOG and metadata.json files.
+  # (3) The latest version is < latest tag.
+
+  # Changes to the following files/directories are not considered
+  # significant:
+  # - Any hidden file/directory (entry that begins with a '.')
+  # - Gemfile
+  # - Gemfile.lock
+  # - Rakefile
+  # - spec directory
+  # - doc directory
+  #
+  # +component_dir+:: The root directory of the component project.
+  # +tags_source+::   The remote from which the tags for this project
+  #                   can be fetched.
+  # +verbose+::       Set to 'true' if you want to see detailed messages
+  def self.compare_latest_tag(component_dir, tags_source = 'origin', verbose = false)
+    info, changelogs = load_and_validate_changelog(component_dir, verbose)
+    Dir.chdir(component_dir) do
+      # determine last tag
+      `git fetch -t #{tags_source} 2>/dev/null`
+      tags = `git tag -l`.split("\n")
+      puts "Available tags from #{tags_source} = #{tags}" if verbose
+      tags.delete_if { |tag| tag.include?('-') or (tag =~ /^v/) }
+
+      if tags.empty?
+        puts "  No tags exist from #{tags_source}"
+      else
+        last_tag = (tags.sort { |a,b| Gem::Version.new(a) <=> Gem::Version.new(b) })[-1]
+
+        # determine mission-impacting files that have changed
+        files_changed = `git diff tags/#{last_tag} --name-only`.strip.split("\n")
+        files_changed.delete_if do |file|
+          file[0] ==  '.' or file == 'Rakefile' or file =~ /^Gemfile|^spec\/|^doc\//
+        end
+
+        if files_changed.empty?
+          puts "  No new tag required: No significant files have changed since '#{last_tag}' tag"
+        else
+          curr_version = Gem::Version.new(info.version)
+          last_tag_version = Gem::Version.new(last_tag)
+
+          if curr_version < last_tag_version
+            fail("ERROR: Version regression. '#{info.version}' < last tag '#{last_tag}'")
+          elsif curr_version == last_tag_version
+            fail("ERROR: Version update beyond last tag '#{last_tag}' is required for #{files_changed.count} changed files:\n  * #{files_changed.join("\n  * ")}")
+          else
+            puts "NOTICE: New tag of version '#{info.version}' is required for #{files_changed.count} changed files:\n  * #{files_changed.join("\n  * ")}"
+          end
+        end
+      end
+    end
+  end
+
+  # Generate an appropriate changelog for an annotated tag from a
+  # component's CHANGELOG or RPM spec file.
+  #
+  # The changelog is only parsed up to the first entry that fails
+  # validation.
+  #
+  # Fails if any of the following occur:
+  # - The metadata.json file for a Puppet module component cannot be
+  #   parsed.
+  # - The CHANGELOG file for a Puppet module component does not exist.
+  # - The CHANGELOG entries for the latest version are malformed.
+  # - The RPM spec file for a non-Puppet module component does not exist.
+  # - More than 1 RPM spec file for a non-Puppet module component exists.
+  # - No valid changelog entries for the version specified in the
+  #   metadata.json/spec file are found.
+  # - The latest changelog version is greater than the version in the
+  #   metadata.json or the RPM spec file.
+  # - The RPM release specified in the spec file does not match the
+  #   release in a changelog entry for the version.
+  # - Any changelog entry below the first entry has a version greater
+  #   than that of the first entry.  Changelog entries must be
+  #   ordered from latest version to earliest version.
+  # - The changelog entries for the latest version are out of date
+  #   order.
+  # - The weekday for a changelog entry for the latest version
+  #   does not match the date specified.
+  #
+  # +component_dir+:: The root directory of the component project.
+  # +verbose+:: Whether to log non-catestrophic changelog parsing
+  #   failures.
+  def self.create_tag_changelog(component_dir, verbose = false)
+    info, changelogs = load_and_validate_changelog(component_dir,verbose)
+
+    result = "\nRelease of #{info.version}\n"
+    changelogs.each do |entry|
+      result += "\n#{entry[:content].first}\n"
+      if entry[:content].size > 1
+        entry[:content][1..-1].each do |line|
+          result += "  #{line}\n"
+        end
+      end
+    end
+    result
+  end
+
+  # Returns all changelog entries for the version
+  #
+  # Fails if
+  # - No valid entries for specified version are found
+  # - The latest changelog version is greater than the specified version
+  # - The release qualifier in a changelog entry for the specified
+  #   version does not match the specified release.
+  #
+  # +changelog_entries+:: Array containing valid changelog entries, each of
+  #   which is a Hash with :date, :version, :release, and :content keys
+  #
+  # +version+:: Target version for which one or more changelog entries
+  #   are to be extracted
+  #
+  # +release+:: Optional release qualifier for version
+  # +verbose+:: Whether to log non-catestrophic changelog parsing
+  #   failures.
+  def self.extract_version_changelog(changelog_entries, version,
+      release=nil, verbose=false)
+
+    version_entry_found = false
+    changelogs = []
+    changelog_entries.each do |entry|
+      if entry[:version] > version
+          fail("ERROR: Changelog entry for version > #{version} found: \n #{entry[:content].join("\n")}")
+        elsif entry[:version] < version
+          break
+        elsif entry[:version] == version
+          # If release is extracted from an RPM spec file, it may have
+          # a distribution at the end (e.g., 0.el7).  We can't extract
+          # distribution from a specfile (query returns 'none' for
+          # DISTRIBUTION and DISTAG tags), so make sure the beginning
+          # of the release matches.
+          if release and entry[:release] and release.match(/^#{entry[:release]}/).nil?
+            fail("ERROR: Version release does not match #{release}: \n #{entry[:content].join("\n")}")
+          end
+          changelogs << entry
+      end
+    end
+
+    if changelogs.empty?
+      fail("ERROR: No valid changelog entry for version #{version} found")
+    end
+    changelogs
+  end
+
+  def self.load_and_validate_changelog(component_dir, verbose)
+    # only get valid changelog entries for the latest version
+    # (up to the first malformed entry)
+    info = Simp::ComponentInfo.new(component_dir, true)
+
+    changelogs = extract_version_changelog(info.changelog, info.version,
+        info.release, verbose)
+
+    [info, changelogs]
+  end
+end