ra10ke 1.0.0 → 2.0.0

data/lib/ra10ke/deprecation.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require 'ra10ke/puppetfile_parser'
+require 'English'
+require 'puppet_forge'
+require 'table_print'
+require 'time'
+
+module Ra10ke::Deprecation
+  # Validate the git urls and refs
+  def define_task_deprecation(*)
+    desc 'Validate that no forge modules are deprecated'
+    task :deprecation do
+      valid = Ra10ke::Deprecation::Validation.new(get_puppetfile.puppetfile_path)
+      exit_code = 0
+      if valid.bad_mods?
+        exit_code = 1
+        message = "\nError: Puppetfile contains deprecated modules."
+        tp valid.sorted_mods, :name, :deprecated_at
+      else
+        message = 'Puppetfile contains no deprecated Forge modules.'
+      end
+      abort(message) if exit_code.positive?
+
+      puts message
+    end
+  end
+
+  class Validation
+    include Ra10ke::PuppetfileParser
+
+    attr_reader :puppetfile
+
+    def initialize(file)
+      file ||= './Puppetfile'
+      @puppetfile = File.expand_path(file)
+      abort("Puppetfile does not exist at #{puppetfile}") unless File.readable?(puppetfile)
+    end
+
+    # @return [Array[Hash]] array of module information and git status
+    def deprecated_modules
+      @deprecated_modules ||= begin
+        deprecated = forge_modules(puppetfile).map do |mod|
+          # For Ruby 2.4 support
+          begin # rubocop:disable Style/RedundantBegin
+            module_name = "#{mod[:namespace] || mod[:name]}-#{mod[:name]}"
+            forge_data = PuppetForge::Module.find(module_name)
+
+            next forge_data if forge_data.deprecated_at
+
+            nil
+          rescue Faraday::ResourceNotFound
+            nil
+          end
+        end
+        deprecated.compact.map do |mod|
+          { name: mod.slug, deprecated_at: Time.parse(mod.deprecated_at) }
+        end
+      end
+    end
+
+    # @return [Boolean] - true if there are any bad mods
+    def bad_mods?
+      deprecated_modules.any?
+    end
+
+    # @return [Hash] - sorts the mods based on good/bad
+    def sorted_mods
+      deprecated_modules.sort_by { |a| a[:name] }
+    end
+  end
+end

data/lib/ra10ke/git_repo.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require 'tempfile'
+
+module Ra10ke
+  class GitRepo
+    attr_reader :url
+
+    REMOTE_REFS_CMD = 'git ls-remote --symref'
+    CLONE_CMD = 'git clone --no-tags'
+    CURRENT_BRANCH_CMD = 'git symbolic-ref --short HEAD'
+    SHOW_CMD = 'git show'
+
+    def initialize(url)
+      @url = url
+    end
+
+    # Get the current branch for a Git repo
+    # @param path [String] - The path to the repository to check
+    # @return [String] - The current active branch of the Git repo
+    def self.current_branch(path)
+      Dir.chdir(path) do
+        data, success = run_command(CURRENT_BRANCH_CMD)
+        return success ? data.strip : nil
+      end
+    end
+
+    # @return [Array] - the raw data from the git ls-remote command as lines array
+    # return empty array if url or command failed
+    def remote_refs
+      @remote_refs ||= begin
+        data, success = run_command("#{REMOTE_REFS_CMD} #{url}")
+        success ? data.lines : []
+      end
+    end
+
+    # @return [Boolean] true if the git url is valid
+    def valid_url?
+      !remote_refs.empty?
+    end
+
+    # @return [Boolean] - return true if the commit sha is valid
+    # @param url [String] - the git string either https or ssh url
+    # @param ref [String] - the sha id
+    def valid_commit?(sha)
+      return false if sha.nil? || sha.empty?
+      return true if valid_ref?(sha)
+
+      # cloning is a last resort if for some reason we cannot
+      # remotely get via ls-remote
+      Dir.mktmpdir do |dir|
+        run_command("#{CLONE_CMD} #{url} #{dir}", silent: true)
+        Dir.chdir(dir) do
+          _, status = run_command("#{SHOW_CMD} #{sha}", silent: true)
+          status
+        end
+      end
+    end
+
+    # @return [Boolean] - return true if the ref is valid
+    # @param url [String] - the git string either https or ssh url
+    # @param ref [String] - the ref object, branch name, tag name, or commit sha, defaults to HEAD
+    def valid_ref?(ref = 'HEAD')
+      return false if ref.nil?
+
+      found = all_refs.find do |data|
+        # we don't need to bother with these types
+        next if data[:type] == :pull || data[:type] == :merge_request
+
+        # is the name equal to the tag or branch?  Is the commit sha equal?
+        data[:name].eql?(ref) || data[:sha].slice(0, 8).eql?(ref.slice(0, 8))
+      end
+      !found.nil?
+    end
+
+    # @return [Array] - an array of all the refs associated with the remote repository
+    # @param url [String] - the git string either https or ssh url
+    # @example
+    #   [{:sha=>"0ec707e431367bbe2752966be8ab915b6f0da754", :ref=>"refs/heads/74110ac", :type=>:branch, :subtype=>nil, :name=>"74110ac"},
+    #     :sha=>"07bb5d2d94db222dca5860eb29c184e8970f36f4", :ref=>"refs/pull/74/head", :type=>:pull, :subtype=>:head, :name=>"74"},
+    #     :sha=>"156ca9a8ea69e056e86355b27d944e59d1b3a1e1", :ref=>"refs/heads/master", :type=>:branch, :subtype=>nil, :name=>"master"},
+    #     :sha=>"fcc0532bbc5a5b65f3941738339e9cc7e3d767ce", :ref=>"refs/pull/249/head", :type=>:pull, :subtype=>:head, :name=>"249"},
+    #     :sha=>"8d54891fa5df75890ee15d53080c2a81b4960f92", :ref=>"refs/pull/267/head", :type=>:pull, :subtype=>:head, :name=>"267"}]
+    def all_refs
+      @all_refs ||= begin
+        remote_refs.each_with_object([]) do |line, refs|
+          sha, ref = line.split("\t")
+          next refs if line.include?('redirecting')
+          next refs if sha.eql?('ref: refs/heads/master')
+          _, type, name, subtype = ref.chomp.split('/')
+          next refs unless name
+
+          type = :tag if type.eql?('tags')
+          type = type.to_sym
+          subtype = subtype.to_sym if subtype
+          type = :branch if type.eql?(:heads)
+          refs << { sha: sha, ref: ref.chomp, type: type, subtype: subtype, name: name }
+        end
+      end
+    end
+
+    # @summary searches all the refs for a ref similar to the name provided
+    #          This is useful when looking for tags if a v or not a v
+    # @param ref_name [String]
+    # @return [String] the matching ref_name or nil
+    def get_ref_like(ref_name)
+      return nil unless valid_url?
+      ref = all_refs.find do |ref|
+        ref[:name].include?(ref_name)
+      end
+      ref
+    end
+
+    # useful for mocking easily
+    # @param cmd [String]
+    # @param silent [Boolean] set to true if you wish to send output to /dev/null, false by default
+    # @return [Array]
+    def self.run_command(cmd, silent: false)
+      out_args = silent ? '2>&1 > /dev/null' : '2>&1'
+      out = `#{cmd} #{out_args}`
+      [out, $CHILD_STATUS.success?]
+    end
+
+    def run_command(cmd, silent: false)
+      self.class.run_command(cmd, silent: silent)
+    end
+  end
+end

data/lib/ra10ke/puppetfile_parser.rb

@@ -12,6 +12,13 @@ module Ra10ke
         end
       end
 
+      # @return [Array] - returns a array of hashes that contain modules from the Forge
+      def forge_modules(file = puppetfile)
+        modules(file).reject do |mod|
+          mod[:args].key?(:git)
+        end
+      end
+
       # @param puppetfile [String] - the absolute path to the puppetfile
       # @return [Array] - returns an array of module hashes that represent the puppetfile
       # @example
@@ -20,11 +27,11 @@ module Ra10ke
       def modules(puppetfile)
         @modules ||= begin
           return [] unless File.exist?(puppetfile)
-
           all_lines = File.read(puppetfile).lines.map(&:strip_comment)
           # remove comments from all the lines
-          lines_without_comments = all_lines.reject { |line| line.match(/#.*\n/) }.join("\n")
+          lines_without_comments = all_lines.reject { |line| line.match(/#.*\n/) || line.empty? }.join("\n")
           lines_without_comments.split(/^mod/).map do |line|
+            
             next nil if line =~ /^forge/
             next nil if line.empty?
 

data/lib/ra10ke/solve.rb

@@ -6,6 +6,7 @@ require 'set'
 require 'solve'
 require 'yaml/store'
 require 'semverse/version'
+require 'fileutils'
 
 # How many versions to fetch from the Forge, at most
 FETCH_LIMIT = 3
@@ -35,6 +36,9 @@ module Ra10ke::Solve
       end
       # Actual new logic begins here:
       cache = (ENV['XDG_CACHE_DIR'] || File.expand_path('~/.cache'))
+
+      FileUtils.mkdir_p(cache)
+      
       # Metadata cache, since the Forge is slow:
       @metadata_cache = YAML::Store.new File.join(cache, 'ra10ke.metadata_cache')
       # The graph of available module versions

data/lib/ra10ke/validate.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
 require 'ra10ke/monkey_patches'
-require 'tempfile'
 require 'table_print'
 require 'ra10ke/puppetfile_parser'
 require 'English'
+require 'ra10ke/git_repo'
 
 module Ra10ke
   module Validate
@@ -13,7 +13,7 @@ module Ra10ke
     BAD_EMOJI = ENV['BAD_EMOJI'] || '😨'
 
     # Validate the git urls and refs
-    def define_task_validate(*args)
+    def define_task_validate(*)
       desc 'Validate the git urls and branches, refs, or tags'
       task :validate do
         gitvalididation = Ra10ke::Validate::Validation.new(get_puppetfile.puppetfile_path)
@@ -32,7 +32,7 @@ module Ra10ke
 
     class Validation
       include Ra10ke::PuppetfileParser
-      
+
       attr_reader :puppetfile
 
       def initialize(file)
@@ -41,69 +41,28 @@ module Ra10ke
         abort("Puppetfile does not exist at #{puppetfile}") unless File.readable?(puppetfile)
       end
 
-      # @return [Boolean] - return true if the ref is valid
-      # @param url [String] - the git string either https or ssh url
-      # @param ref [String] - the ref object, branch name, tag name, or commit sha, defaults to HEAD
-      def valid_ref?(url, ref = 'HEAD')
-        raise ArgumentError unless ref
-        found = all_refs(url).find do |data|
-          # we don't need to bother with these types
-          next if data[:type] == :pull || data[:type] == :merge_request     
-          # is the name equal to the tag or branch?  Is the commit sha equal?     
-          data[:name].eql?(ref) || data[:sha].slice(0,8).eql?(ref.slice(0,8))
-        end
-        !found.nil?
-      end
-
-      # @return [Array] - an array of all the refs associated with the remote repository
-      # @param url [String] - the git string either https or ssh url
-      # @example
-      #   [{:sha=>"0ec707e431367bbe2752966be8ab915b6f0da754", :ref=>"refs/heads/74110ac", :type=>:branch, :subtype=>nil, :name=>"74110ac"},
-      #     :sha=>"07bb5d2d94db222dca5860eb29c184e8970f36f4", :ref=>"refs/pull/74/head", :type=>:pull, :subtype=>:head, :name=>"74"},
-      #     :sha=>"156ca9a8ea69e056e86355b27d944e59d1b3a1e1", :ref=>"refs/heads/master", :type=>:branch, :subtype=>nil, :name=>"master"},
-      #     :sha=>"fcc0532bbc5a5b65f3941738339e9cc7e3d767ce", :ref=>"refs/pull/249/head", :type=>:pull, :subtype=>:head, :name=>"249"},
-      #     :sha=>"8d54891fa5df75890ee15d53080c2a81b4960f92", :ref=>"refs/pull/267/head", :type=>:pull, :subtype=>:head, :name=>"267"}]
-      def all_refs(url)
-        data = `git ls-remote --symref #{url}`
-        raise "Error downloading #{url}" unless $CHILD_STATUS.success?
-        data.lines.reduce([]) do |refs, line|
-          sha, ref = line.split("\t")
-          next refs if sha.eql?('ref: refs/heads/master')
-          _, type, name, subtype = ref.chomp.split('/')
-          next refs unless name 
-          type = :tag if type.eql?('tags')
-          type = type.to_sym
-          subtype = subtype.to_sym if subtype
-          type = :branch if type.eql?(:heads)
-          refs << { sha: sha, ref: ref.chomp, type: type, subtype: subtype, name: name }
-        end
-      end
-
-      # @return [Boolean] - return true if the commit sha is valid
-      # @param url [String] - the git string either https or ssh url
-      # @param ref [String] - the sha id
-      def valid_commit?(url, sha)
-        return false if sha.nil? || sha.empty?
-        return true if valid_ref?(url, sha)
-        Dir.mktmpdir do |dir|
-          `git clone --no-tags #{url} #{dir} 2>&1 > /dev/null`
-          Dir.chdir(dir) do
-            `git show #{sha} 2>&1 > /dev/null`
-            $CHILD_STATUS.success?
-          end
-        end
-      end
-
       # @return [Array[Hash]] array of module information and git status
       def all_modules
-        begin
+        @all_modules ||= begin
+          r10k_branch = Ra10ke::GitRepo.current_branch(File.dirname(puppetfile))
           git_modules(puppetfile).map do |mod|
-            ref = mod[:args][:ref] || mod[:args][:tag] || mod[:args][:branch]
-            valid_ref = valid_ref?(mod[:args][:git], ref) || valid_commit?(mod[:args][:git], mod[:args][:ref])
+            repo = Ra10ke::GitRepo.new(mod[:args][:git])
+            ref = mod[:args][:ref] || mod[:args][:tag] || mod[:args][:branch] || mod[:args][:commit]
+            # If using control_branch, try to guesstimate what the target branch should be
+            if ref == ':control_branch'
+              ref = ENV['CONTROL_BRANCH'] \
+                    || r10k_branch \
+                    || mod[:args][:default_branch_override] \
+                    || ENV['CONTROL_BRANCH_FALLBACK'] \
+                    || mod[:args][:default_branch] \
+                    || 'main'
+            end
+            valid_ref = repo.valid_ref?(ref) || repo.valid_commit?(mod[:args][:ref])
             {
               name: mod[:name],
-              url: mod[:args][:git],
+              url: repo.url,
               ref: ref,
+              valid_url?: repo.valid_url?,
               valid_ref?: valid_ref,
               status: valid_ref ? Ra10ke::Validate::GOOD_EMOJI : Ra10ke::Validate::BAD_EMOJI
             }

data/lib/ra10ke/version.rb

@@ -1,3 +1,3 @@
 module Ra10ke
-  VERSION = "1.0.0"
+  VERSION = "2.0.0"
 end

data/lib/ra10ke.rb

@@ -4,6 +4,7 @@ require 'ra10ke/version'
 require 'ra10ke/solve'
 require 'ra10ke/syntax'
 require 'ra10ke/dependencies'
+require 'ra10ke/deprecation'
 require 'ra10ke/duplicates'
 require 'ra10ke/install'
 require 'ra10ke/validate'
@@ -15,6 +16,7 @@ module Ra10ke
     include Ra10ke::Solve
     include Ra10ke::Syntax
     include Ra10ke::Dependencies
+    include Ra10ke::Deprecation
     include Ra10ke::Duplicates
     include Ra10ke::Install
     include Ra10ke::Validate
@@ -35,14 +37,18 @@ module Ra10ke
         define_task_solve_dependencies(*args)
         define_task_syntax(*args)
         define_task_dependencies(*args)
+        define_task_deprecation(*args)
         define_task_duplicates(*args)
         define_task_install(*args)
         define_task_validate(*args)
+        define_task_print_git_conversion(*args)
       end
     end
 
     def get_puppetfile
       R10K::Puppetfile.new(@basedir, @moduledir, @puppetfile_path, @puppetfile_name, @force)
+    rescue ArgumentError # R10k < 2.6.0
+      R10K::Puppetfile.new(@basedir, @moduledir, @puppetfile_path || File.join(@basedir, @puppetfile_name || 'Puppetfile'))
     end
   end
 end

data/ra10ke.gemspec

@@ -14,14 +14,17 @@ Gem::Specification.new do |spec|
 
   spec.files         = `git ls-files`.split($/)
   spec.require_paths = ["lib"]
-  spec.required_ruby_version = '>= 2.1.0'
+  spec.required_ruby_version = '>= 2.6.0'
 
   spec.add_dependency "rake"
   spec.add_dependency "puppet_forge"
-  spec.add_dependency "r10k"
+  spec.add_dependency "r10k", '>= 2.6.5'
   spec.add_dependency "git"
   spec.add_dependency "solve"
   spec.add_dependency 'semverse', '>= 2.0'
   spec.add_dependency 'table_print', '~> 1.5.6'
   spec.add_development_dependency 'rspec', '~> 3.6'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'simplecov'
 end
+

data/spec/fixtures/Puppetfile

@@ -2,7 +2,9 @@
 
 forge 'http://forge.puppetlabs.com'
 
+mod 'choria/choria',          '0.26.2'
 mod 'puppetlabs/inifile',     '2.2.0'
+mod 'puppetlabs-ruby',        '1.0.1'
 mod 'puppetlabs/stdlib',      '4.24.0'
 mod 'puppetlabs/concat',      '4.0.0'
 mod 'puppetlabs/ntp',         '6.4.1'

data/spec/fixtures/Puppetfile_deprecation_issue

@@ -0,0 +1,78 @@
+# used in profiles
+mod 'puppet/systemd', :latest
+mod 'puppet/lldpd', :latest
+mod 'puppet/ferm', :latest
+mod 'puppet/borg', :latest
+mod 'puppet/wireguard', :latest
+mod 'puppet/bird', :latest
+mod 'herculesteam/augeasproviders_pam', :latest
+mod 'herculesteam/augeasproviders_shellvar', :latest
+mod 'puppetlabs/vcsrepo', :latest
+mod 'saz/ssh', :latest
+mod 'puppet/r10k', :latest
+mod 'puppet/dbbackup', :latest
+mod 'puppet/mosquitto', :latest
+mod 'puppet/grafana', :latest
+mod 'puppet/nginx', :latest
+mod 'puppet/ssh_keygen', :latest
+mod 'puppet/unbound', :latest
+mod 'puppetlabs/inifile', :latest
+mod 'puppet/prometheus', :latest
+mod 'theforeman/foreman', :latest
+mod 'theforeman/puppet', :latest
+mod 'theforeman/foreman_proxy', :latest
+mod 'puppet/nftables', :latest
+mod 'herculesteam/augeasproviders_sysctl', :latest
+mod 'camptocamp/catalog_diff', :latest
+mod 'puppet/unattended_upgrades', :latest
+mod 'puppet/selinux', :latest
+mod 'choria/choria', :latest
+mod 'puppet/archive', :latest
+mod 'puppet/elasticsearch', :latest
+mod 'jsok/vault',
+  git: 'https://github.com/bastelfreak/puppet-vault',
+  ref: 'test'
+
+# dependencies
+mod 'herculesteam/augeasproviders_core', :latest
+mod 'puppetlabs/stdlib', :latest
+mod 'choria/mcollective', :latest
+mod 'puppetlabs/concat', :latest
+mod 'puppetlabs/apt', :latest
+mod 'puppetlabs/apache', :latest
+mod 'puppetlabs/postgresql', :latest
+mod 'puppet/extlib', :latest
+mod 'puppet/redis', :latest
+mod 'puppet/epel', :latest
+mod 'theforeman/puppetserver_foreman', :latest
+mod 'richardc/datacat', :latest
+mod 'theforeman/dns', :latest
+mod 'theforeman/dhcp', :latest
+mod 'theforeman/tftp', :latest
+mod 'puppetlabs/xinetd', :latest
+mod 'choria-mcollective_choria', :latest
+mod 'choria/mcollective_agent_puppet', :latest
+mod 'choria/mcollective_agent_package', :latest
+mod 'choria/mcollective_agent_service', :latest
+mod 'choria/mcollective_agent_filemgr', :latest
+mod 'choria/mcollective_agent_shell', :latest
+mod 'choria/mcollective_agent_nettest', :latest
+mod 'choria/mcollective_agent_puppetca', :latest
+mod 'choria/mcollective_agent_bolt_tasks', :latest
+mod 'choria/mcollective_agent_iptables', :latest
+mod 'choria/mcollective_agent_process', :latest
+mod 'choria/mcollective_util_actionpolicy', :latest
+mod 'optiz0r/mcollective_agent_puppet_env', :latest
+mod 'jay7x/mcollective_agent_query', :latest
+mod 'choria/mcollective_data_sysctl', :latest
+mod 'puppet/yum', :latest
+mod 'puppetlabs/java', :latest
+mod 'puppet/elastic_stack', :latest
+mod 'puppet/hashi_stack', :latest
+
+# core modules are not vendored on Gentoo
+mod 'puppetlabs-sshkeys_core', :latest
+mod 'puppetlabs-mount_core', :latest
+mod 'puppetlabs-augeas_core', :latest
+mod 'puppetlabs-selinux_core', :latest
+mod 'puppetlabs-yumrepo_core', :latest

data/spec/fixtures/Puppetfile_git_conversion

@@ -0,0 +1,28 @@
+mod 'choria',
+  :git => 'https://github.com/choria-io/puppet-choria',
+  :ref => '0.26.2'
+
+mod 'inifile',
+  :git => 'https://github.com/puppetlabs/puppetlabs-inifile',
+  :ref => '2.2.0'
+
+mod 'ruby',
+  :git => 'https://github.com/puppetlabs/puppetlabs-ruby.git',
+  :ref => 'v1.0.1'
+
+mod 'stdlib',
+  :git => 'https://github.com/puppetlabs/puppetlabs-stdlib',
+  :ref => '4.24.0'
+
+mod 'concat',
+  :git => 'https://github.com/puppetlabs/puppetlabs-concat',
+  :ref => '4.0.0'
+
+mod 'ntp',
+  :git => 'https://github.com/puppetlabs/puppetlabs-ntp',
+  :ref => '6.4.1'
+
+mod 'archive',
+  :git => 'https://github.com/voxpupuli/puppet-archive',
+  :ref => 'v3.1.1'
+

data/spec/fixtures/Puppetfile_with_bad_refs

@@ -7,4 +7,7 @@ mod 'gitlab',
 mod 'r10k',
   :git => 'https://github.com/acidprime/r10k',
   :tag => 'bad'
+  
+mod 'debug',
+  :git => 'https://github.com/nwops/typo'    
   

data/spec/fixtures/Puppetfile_with_commit

@@ -0,0 +1,3 @@
+mod 'ntp',
+  git: 'https://github.com/puppetlabs/puppetlabs-ntp',
+  commit: '81b34c6'

data/spec/fixtures/Puppetfile_with_control_branch

@@ -0,0 +1,23 @@
+mod 'puppet-hiera_master',
+  git: 'https://github.com/voxpupuli/puppet-hiera_master.git',
+  branch: master
+
+mod 'puppet-hiera_control',
+  git: 'https://github.com/voxpupuli/puppet-hiera_control.git',
+  branch: :control_branch
+
+mod 'puppet-hiera_controlwithdefault',
+  git: 'https://github.com/voxpupuli/puppet-hiera_control.git',
+  branch: :control_branch,
+  default_branch: 'master'
+
+mod 'puppet-hiera_controlwithdefaultoverride',
+  git: 'https://github.com/voxpupuli/puppet-hiera_control.git',
+  branch: :control_branch,
+  default_branch_override: 'master',
+  default_branch: 'devel'
+
+mod 'puppet-hiera_fakecontrol',
+  git: 'https://github.com/voxpupuli/puppet-hiera_fakecontrol.git',
+  branch: control_branch
+