pdksync 0.5.0 → 0.6.0

data/lib/pdksync/conf/puppet_abs_supported_platforms.yaml

@@ -0,0 +1,41 @@
+# PUPPET VERSION PLATFORM COMPATIBILITY
+#
+# Define the Puppet version as a root key, then specify the OS platform(s) and versions(s) that this Puppet version is
+# compatible with AND that you wish to test on. If you wish to exclude a platform from testing, simply omit it.
+#
+# OS names and versions must conform to the VMPooler nomenclature and conventions.
+#
+# Running 'bundle exec rake 'pdksync:generate_vmpooler_release_checks[7]' will generate an entry in the 'provision.yaml'
+# for each managed module that contains a configuration that satisfies both:
+# - The module's compatible platforms
+# - The Puppet version's compatible platforms (in this example: '7')
+#
+# NOTE: arch will always be assumed to be 'x86_64'
+---
+5:
+  centos: ['5', '6', '7', '8']
+  debian: ['8', '9', '10']
+  oracle: ['5', '6', '7']
+  redhat: ['5', '6', '7', '8']
+  sles: ['12', '15']
+  scientific: ['6', '7']
+  ubuntu: ['14.04', '16.04', '18.04']
+  win: ['2008r2', '2012r2', '2016', '2019', '10-pro']
+6:
+  centos: ['5', '6', '7', '8']
+  debian: ['8', '9', '10']
+  oracle: ['5', '6', '7']
+  redhat: ['5', '6', '7', '8']
+  sles: ['12', '15']
+  scientific: ['6', '7']
+  ubuntu: ['14.04', '16.04', '18.04', '20.04']
+  win: ['2008r2', '2012r2', '2016', '2019', '10-pro']
+7:
+  centos: ['7', '8']
+  debian: ['9', '10']
+  oracle: ['7']
+  redhat: ['7', '8']
+  sles: ['12', '15']
+  scientific: ['7']
+  ubuntu: ['18.04', '20.04']
+  win: ['2012r2', '2016', '2019', '10-pro']

data/lib/pdksync/configuration.rb

@@ -0,0 +1,155 @@
+require 'yaml'
+require 'pdk/version'
+require 'ostruct'
+
+# @summary
+#   A class used to contain a set of configuration variables
+# @note
+#   Configuration is loaded from `$HOME/.pdksync.yml`. If $HOME is not set, the config_path will use the current directory.
+#   The configuration filename and path can be overridden with env variable PDK_CONFIG_PATH
+#   Set PDKSYNC_LABEL to '' to disable adding a label during pdksync runs.
+module PdkSync
+  class Configuration < OpenStruct
+    SUPPORTED_SCM_PLATFORMS = [:github, :gitlab].freeze
+    PDKSYNC_FILE_NAME = 'pdksync.yml'.freeze
+
+    # Any key value added to the default config or custom config
+    # will automatically be a new configuration item and referenced
+    # via Configuration.new.<key_name> ie.
+    # c = Configuration.new
+    # c.api_endpoint
+    DEFAULT_CONFIG = {
+      namespace: 'puppetlabs',
+      pdksync_dir: 'modules_pdksync',
+      pdksync_gem_dir: 'gems_pdksync',
+      push_file_destination: 'origin',
+      create_pr_against: 'main',
+      managed_modules: 'managed_modules.yml',
+      pdksync_label: 'maintenance',
+      git_platform: :github,
+      git_base_uri: 'https://github.com',
+      gitlab_api_endpoint: 'https://gitlab.com/api/v4',
+      api_endpoint: nil,
+      pdk_templates_prefix: nil,
+      pdk_templates_ref: PDK::VERSION,
+      pdk_templates_url: 'https://github.com/puppetlabs/pdk-templates.git',
+      jenkins_platform: :jenkins,
+      jenkins_base_uri: 'https://jenkins.io',
+      jenkins_api_endpoint: '',
+      jenkins_server_url: '',
+      module_is_authoritive: true
+    }.freeze
+
+    # @param config_path [String] -  the path to the pdk config file
+    def initialize(config_path = ENV['PDKSYNC_CONFIG_PATH'])
+      @config_path = locate_config_path(config_path)
+      @custom_config = DEFAULT_CONFIG.merge(custom_config(@config_path))
+      @custom_config[:pdk_templates_ref] = "#{@custom_config[:pdk_templates_prefix]}#{@custom_config[:pdk_templates_ref]}"
+      super(@custom_config)
+      valid_scm?(git_platform)
+      valid_access_token?
+    end
+
+    # @return [Hash] - returns the access settings for git scm
+    def git_platform_access_settings
+      @git_platform_access_settings ||= {
+        access_token: access_token,
+        gitlab_api_endpoint: gitlab_api_endpoint || api_endpoint,
+        api_endpoint: api_endpoint
+
+      }
+    end
+
+    def jenkins_platform_access_settings
+      @jenkins_platform_access_settings ||= {
+        jenkins_username: ENV['JENKINS_USERNAME'].freeze,
+        jenkins_password: ENV['JENKINS_PASSWORD'].freeze,
+        jenkins_api_endpoint: ''
+      }
+    end
+
+    # @return [Hash] - returns the access settings for gemfury account
+    def gemfury_access_settings
+      valid_access_token_gem_fury?
+      @gemfury_access_token = access_token_gem_fury
+    end
+
+    # @return [String] return a rendered string for pdk to use the templates
+    def templates
+      "--template-url=#{pdk_templates_url} --template-ref=#{pdk_templates_ref}"
+    end
+
+    # @param path [String] path to the pdksync config file in yaml format
+    # @return [Hash] the custom configuration as a hash
+    def custom_config(path = nil)
+      return {} unless path
+      return {} unless File.exist?(path)
+      c = (YAML.load_file(path) || {}).transform_keys_to_symbols
+      c[:git_base_uri] ||= 'https://gitlab.com' if c[:git_platform].eql?(:gitlab)
+      c
+    end
+
+    # @return [String] the path the pdksync config file, nil if not found
+    def locate_config_path(custom_file = nil)
+      files = [
+        custom_file,
+        PDKSYNC_FILE_NAME,
+        File.join(ENV['HOME'], PDKSYNC_FILE_NAME)
+      ]
+      files.find { |file| file && File.exist?(file) }
+    end
+
+    private
+
+    # @return [Boolean] true if the supported platforms were specified correctly
+    # @param scm [Symbol] - the scm type (:github or :gitlab)
+    def valid_scm?(scm)
+      unless SUPPORTED_SCM_PLATFORMS.include?(scm)
+        raise ArgumentError, "Unsupported Git hosting platform '#{scm}'."\
+          " Supported platforms are: #{SUPPORTED_SCM_PLATFORMS.join(', ')}"
+      end
+      true
+    end
+
+    # @return [Boolean] true if the access token for the scm platform was supplied
+    def valid_access_token?
+      if access_token.nil?
+        raise ArgumentError, "Git platform access token for #{git_platform.capitalize} not set"\
+          " - use 'export #{git_platform.upcase}_TOKEN=\"<your token>\"' to set"
+      end
+      true
+    end
+
+    # @return [Boolean] true if the access token for the gemfury was supplied
+    def valid_access_token_gem_fury?
+      if access_token_gem_fury.nil?
+        raise 'Gemfury access token not set'\
+        " - use 'export GEMFURY_TOKEN=\"<your token>\"' to set"
+      end
+      true
+    end
+
+    # @return [String] the platform specific access token
+    def access_token
+      case git_platform
+      when :github
+        ENV['GITHUB_TOKEN'].freeze
+      when :gitlab
+        ENV['GITLAB_TOKEN'].freeze
+      end
+    end
+
+    # @return [String] the gem_fury access token
+    def access_token_gem_fury
+      ENV['GEMFURY_TOKEN'].freeze
+    end
+  end
+end
+
+# monkey patch
+class Hash
+  # take keys of hash and transform those to a symbols
+  def transform_keys_to_symbols
+    each_with_object({}) { |(k, v), memo| memo[k.to_sym] = v; }
+  end
+end

data/lib/pdksync/githubclient.rb

@@ -10,7 +10,9 @@ class PdkSync::GithubClient
   #   supplied access token
   # @param access_token
   #   The Github access token, required to access the Github API
-  def initialize(access_token)
+  def initialize(access_token, api_endpoint = nil)
+    # USE ENV['OCTOKIT_API_ENDPOINT'] or pass in the api_endpoint
+    Octokit.configure { |c| c.api_endpoint = api_endpoint } unless api_endpoint.nil?
     @client = Octokit::Client.new(access_token: access_token.to_s)
     @client.user.login
   end

data/lib/pdksync/gitplatformclient.rb

@@ -26,11 +26,11 @@ class PdkSync::GitPlatformClient
               when :github
                 require 'pdksync/githubclient'
 
-                PdkSync::GithubClient.new(access_token)
+                PdkSync::GithubClient.new(access_token, git_platform_access_settings[:api_endpoint])
               when :gitlab
                 require 'pdksync/gitlabclient'
 
-                gitlab_api_endpoint = git_platform_access_settings[:gitlab_api_endpoint]
+                gitlab_api_endpoint = git_platform_access_settings[:gitlab_api_endpoint] || git_platform_access_settings[:api_endpoint]
                 PdkSync::GitlabClient.new(access_token, gitlab_api_endpoint)
               end
   end

data/lib/pdksync/jenkinsclient.rb

@@ -0,0 +1,50 @@
+require 'jenkins_api_client'
+
+# @summary
+#   This class wraps Gitlab::JenkinsCLient and provides the method implementations
+#   required by pdksync main to access the Jenkins API for creating jobs in jenkins
+class PdkSync::JenkinsClient
+  # @summary
+  #   Creates a new Jenkins::Client and logs in the user based on the
+  #   supplied user credentials and the Jenkins API endpoint URL
+  # @param [String] jenkins_platform_access_settings
+  #   The Jenkins credentials, required to access the Jenkins API
+
+  def initialize(jenkins_server_url, jenkins_platform_access_settings)
+    jenkins_username = jenkins_platform_access_settings[:jenkins_username]
+    jenkins_password = jenkins_platform_access_settings[:jenkins_password]
+    @client = JenkinsApi::Client.new('server_url' => jenkins_server_url,
+                                     'username'   => jenkins_username,
+                                     'password'   => jenkins_password)
+  end
+
+  # @summary
+  #   Creates a new adhoc job against the jenkins
+  #   platform
+  # @param [String] github_repo
+  #   Repo or Module for which the adhoc job to be created
+  # @param [String] github_branch
+  #   The target branch against which to create the adhoc job
+  # @return
+  #   Build Id returned by the job
+  def create_adhoc_job(github_repo, github_branch, github_user, job_name)
+    # params to start the build
+    job_params = { 'GITHUB_USER' => github_user,
+                   'GITHUB_REPO' => github_repo,
+                   'GITHUB_REF'  => github_branch }
+    # job name
+    # Wait for up to 30 seconds, attempt to cancel queued build
+    opts = { 'build_start_timeout' => 30,
+             'cancel_on_build_start_timeout' => true,
+             'completion_proc' => lambda { |build_number, cancelled| # rubocop:disable Style/Lambda
+               if build_number
+                 PdkSync::Logger.info "Wait over: build #{build_number} started"
+               else
+                 PdkSync::Logger.info "Wait over: build not started, build #{cancelled ? '' : 'NOT '} cancelled"
+               end
+             } }
+
+    build_id = @client.job.build(job_name, job_params || {}, opts)
+    build_id
+  end
+end

data/lib/pdksync/logger.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+class String
+  def colorize(color_code)
+    "\e[#{color_code}m#{self}\e[0m"
+  end
+
+  def red
+    colorize(31)
+  end
+
+  def green
+    colorize(32)
+  end
+
+  def fatal
+    red
+  end
+
+  def yellow
+    colorize(33)
+  end
+
+  def light_blue
+    colorize(36)
+  end
+end
+
+module PdkSync
+  class Logger
+    def self.warn(message)
+      logger.warn(message)
+    end
+
+    def self.info(message)
+      logger.info(message)
+    end
+
+    def self.debug(message)
+      logger.debug(message)
+    end
+
+    def self.fatal(message)
+      logger.fatal(message)
+    end
+
+    def self.crit(message)
+      logger.error(message)
+    end
+
+    def self.log_file
+      if ENV['PDKSYNC_LOG_FILENAME'] && File.exist?(ENV['PDKSYNC_LOG_FILENAME'])
+        ENV['PDKSYNC_LOG_FILENAME']
+      else
+        STDOUT
+      end
+    end
+
+    def self.logger(file = PdkSync::Logger.log_file)
+      @logger ||= begin
+        log = ::Logger.new(file)
+        log.level = log_level
+        log.progname = 'PdkSync'
+        log.formatter = proc do |severity, datetime, progname, msg|
+          if PdkSync::Logger.log_file == STDOUT
+            "#{severity} - #{progname}: #{msg}\n".send(color(severity))
+          else
+            "#{datetime} #{severity} - #{progname}: #{msg}\n".send(color(severity))
+          end
+        end
+        log
+      end
+    end
+
+    def logger
+      @logger ||= PdkSync::Logger.logger
+    end
+
+    def self.color(severity)
+      case severity
+      when ::Logger::Severity::WARN, 'WARN'
+        :yellow
+      when ::Logger::Severity::INFO, 'INFO'
+        :green
+      when ::Logger::Severity::FATAL, 'FATAL'
+        :fatal
+      when ::Logger::Severity::ERROR, 'ERROR'
+        :fatal
+      when ::Logger::Severity::DEBUG, 'DEBUG'
+        :light_blue
+      else
+        :green
+      end
+    end
+
+    def self.log_level
+      level = ENV['LOG_LEVEL'].downcase if ENV['LOG_LEVEL']
+      case level
+      when 'warn'
+        ::Logger::Severity::WARN
+      when 'fatal'
+        ::Logger::Severity::FATAL
+      when 'debug'
+        ::Logger::Severity::DEBUG
+      when 'info'
+        ::Logger::Severity::INFO
+      when 'error'
+        ::Logger::Severity::ERROR
+      else
+        ::Logger::Severity::INFO
+      end
+    end
+  end
+end

data/lib/pdksync/rake_tasks.rb

@@ -10,6 +10,23 @@ task :pdksync, [:additional_title] do |_task, args|
   PdkSync.main(steps: [:use_pdk_ref, :clone, :pdk_update, :create_commit, :push, :create_pr], args: args)
 end
 
+desc 'Run full gem_testing process, clone repository, gemfile update, create pr. Additional title information can be added to the title, which will be appended before the reference section.'
+task :gem_testing, [:additional_title, :gem_to_test, :gem_line, :gem_sha_finder, :gem_sha_replacer, :gem_version_finder, :gem_version_replacer, :gem_branch_finder, :gem_branch_replacer] do |_task, args| # rubocop:disable Metrics/LineLength
+  args = { branch_name: 'pdksync_gem_testing{ref}',
+           commit_message: 'pdksync_gem_testing{ref}',
+           pr_title: 'pdksync_gem_testing{ref}',
+           additional_title: args[:additional_title],
+           gem_to_test: args[:gem_to_test],
+           gem_line: args[:gem_line],
+           gem_sha_finder: args[:gem_sha_finder],
+           gem_sha_replacer: args[:gem_sha_replacer],
+           gem_version_finder: args[:gem_version_finder],
+           gem_version_replacer: args[:gem_version_replacer],
+           gem_branch_finder: args[:gem_branch_finder],
+           gem_branch_replacer: args[:gem_branch_replacer] }
+  PdkSync.main(steps: [:use_gem_ref, :clone, :gem_file_update, :create_commit, :push, :create_pr], args: args)
+end
+
 namespace :pdksync do
   desc 'Runs PDK convert against modules'
   task :pdk_convert do
@@ -21,27 +38,89 @@ namespace :pdksync do
     PdkSync.main(steps: [:pdk_validate])
   end
 
-  desc "Run a command against modules eg rake 'run_a_command[complex command here -f -gx]'"
-  task :run_a_command, [:command] do |_task, args|
-    PdkSync.main(steps: [:run_a_command], args: args[:command])
+  desc "Run a command against modules eg rake 'run_a_command[complex command here -f -gx, 'background']'"
+  task :run_a_command, [:command, :option] do |_task, args|
+    PdkSync.main(steps: [:run_a_command], args: args)
+  end
+
+  desc "Gem File Update'gem_file_update[gem_to_test, gem_line, gem_sha_finder, gem_sha_replacer, gem_version_finder, gem_version_replacer, gem_branch_finder, gem_branch_replacer]'"
+  task :gem_file_update, [:gem_to_test, :gem_line, :gem_sha_finder, :gem_sha_replacer, :gem_version_finder, :gem_version_replacer, :gem_branch_finder, :gem_branch_replacer] do |_task, args|
+    PdkSync.main(steps: [:gem_file_update], args: args)
+  end
+
+  desc "Run test against modules eg rake 'run_tests_locally[litmus, 'provision_type']'"
+  task :run_tests_locally, [:provision_type, :puppet_collection] do |_task, args|
+    PdkSync.main(steps: [:run_tests_locally], args: args)
+  end
+
+  desc "Fetch run results against modules eg rake 'fetch_test_results_locally[litmus]'"
+  task :fetch_test_results_locally do
+    PdkSync.main(steps: [:fetch_test_results_locally])
+  end
+
+  desc "Run test in jenkins for traditional modules eg rake 'run_tests_jenkins['jenkins_server_url', 'branchname']'"
+  task :run_tests_jenkins, [:jenkins_server_url, :github_branch, :test_framework, :github_user] do |_task, args|
+    PdkSync.main(steps: [:run_tests_jenkins], args: args)
+  end
+
+  desc 'Multi Gem Testing, multi_gem_testing[gem_name, version_file, build_gem, gem_path, gemfury_username]'
+  task :multi_gem_testing, [:gem_name, :version_file, :build_gem, :gem_path, :gemfury_username] do |_task, args|
+    PdkSync.main(steps: [:multi_gem_testing], args: args)
+  end
+
+  desc 'Multi Gem File Update, multigem_file_update[gem_name, gemfury_username]'
+  task :multigem_file_update, [:gem_name, :gemfury_username] do |_task, args|
+    PdkSync.main(steps: [:multigem_file_update], args: args)
   end
 
   desc 'Display the current configuration of pdksync'
   task :show_config do
-    include PdkSync::Constants
+    config = PdkSync::Configuration.new
     puts 'Please note that you can override any of the configuration by using an additional file at `$HOME/.pdksync.yml`.'.bold.red
-    puts 'PDKSync Configuration'.bold.yellow
-    puts '- Git hosting platform: '.bold + PdkSync::Constants::GIT_PLATFORM.to_s.cyan
-    puts '- Git base URI: '.bold + PdkSync::Constants::GIT_BASE_URI.to_s.cyan
-    if PdkSync::Constants::GIT_PLATFORM == :gitlab
-      puts '- Gitlab API endpoint: '.bold + PdkSync::Constants::GITLAB_API_ENDPOINT.to_s.cyan
+    puts "\nPDKSync Configuration".bold.yellow
+    config.to_h.each do |key, value|
+      puts "- #{key}: ".bold + value.to_s.cyan
     end
-    puts '- Namespace: '.bold + PdkSync::Constants::NAMESPACE.to_s.cyan
-    puts '- PDKSync Dir: '.bold + PdkSync::Constants::PDKSYNC_DIR.to_s.cyan
-    puts '- Push File Destination: '.bold + PdkSync::Constants::PUSH_FILE_DESTINATION.to_s.cyan
-    puts '- Create PR Against: '.bold + PdkSync::Constants::CREATE_PR_AGAINST.to_s.cyan
-    puts '- Managed Modules: '.bold + PdkSync::Constants::MANAGED_MODULES.to_s.cyan
-    puts '- Default PDKSync Label: '.bold + PdkSync::Constants::PDKSYNC_LABEL.to_s.cyan
+  end
+
+  desc "Fetch run results against traditional modules eg rake 'fetch_traditional_test_results'"
+  task :test_results_jenkins, [:jenkins_server_url] do |_task, args|
+    PdkSync.main(steps: [:test_results_jenkins], args: args)
+  end
+
+  desc 'Add a provision list key to provision.yaml'
+  task :add_provision_list, [:key, :provisioner, :images] do |_task, args|
+    PdkSync.main(steps: [:add_provision_list], args: args)
+  end
+
+  desc 'Generates release checks in provision.yaml based on module compatible platforms and puppet version'
+  task :generate_vmpooler_release_checks, [:puppet_version] do |_task, args|
+    PdkSync.main(steps: [:generate_vmpooler_release_checks], args: args)
+  end
+
+  desc 'Update the metadata.json OS supported list to that defined in conf/supported_os_list.yaml'
+  task :update_os_support do |_task, args|
+    PdkSync.main(steps: [:update_os_support], args: args)
+  end
+
+  desc 'Remove a platform version from the metadata.json'
+  task :remove_platform_from_metadata, [:os, :version] do |_task, args|
+    PdkSync.main(steps: [:remove_platform_from_metadata], args: args)
+  end
+
+  desc 'Add a platform version to the metadata.json'
+  task :add_platform_to_metadata, [:os, :version] do |_task, args|
+    PdkSync.main(steps: [:add_platform_to_metadata], args: args)
+  end
+
+  desc 'Add or update a requirement in the metadata.json'
+  task :update_requirements, [:name, :key, :value] do |_task, args|
+    PdkSync.main(steps: [:update_requirements], args: args)
+  end
+
+  desc 'Normalise the OS and OS versions in the metadata.json'
+  task :normalize_metadata_supported_platforms do |_task, args|
+    PdkSync.main(steps: [:normalize_metadata_supported_platforms], args: args)
   end
 end
 
@@ -51,6 +130,11 @@ namespace :git do
     PdkSync.main(steps: [:clone])
   end
 
+  desc 'Clone managed gem'
+  task :clone_gem, [:gem_name] do |_task, args|
+    PdkSync.main(steps: [:clone_gem], args: args)
+  end
+
   desc "Stage commits for modules, branchname and commit message eg rake 'git:create_commit[flippity, commit messagez]'"
   task :create_commit, [:branch_name, :commit_message] do |_task, args|
     PdkSync.main(steps: [:create_commit], args: args)