modulesync 0.8.2 → 1.2.0

data/lib/modulesync.rb

@@ -9,7 +9,7 @@ require 'modulesync/settings'
 require 'modulesync/util'
 require 'monkey_patches'
 
-module ModuleSync
+module ModuleSync # rubocop:disable Metrics/ModuleLength
   include Constants
 
   def self.config_defaults
@@ -22,31 +22,39 @@ module ModuleSync
   end
 
   def self.local_file(config_path, file)
-    "#{config_path}/#{MODULE_FILES_DIR}/#{file}"
+    File.join(config_path, MODULE_FILES_DIR, file)
   end
 
-  def self.module_file(project_root, puppet_module, file)
-    "#{project_root}/#{puppet_module}/#{file}"
+  def self.module_file(project_root, namespace, puppet_module, *parts)
+    File.join(project_root, namespace, puppet_module, *parts)
   end
 
-  def self.local_files(path)
-    if File.exist?(path)
-      # only select *.erb files, and strip the extension. This way all the code will only have to handle bare paths, except when reading the actual ERB text
-      local_files = Find.find(path).find_all { |p| p =~ /.erb$/ && !File.directory?(p) }.collect { |p| p.chomp('.erb') }.to_a
+  # List all template files.
+  #
+  # Only select *.erb files, and strip the extension. This way all the code will only have to handle bare paths,
+  # except when reading the actual ERB text
+  def self.find_template_files(local_template_dir)
+    if File.exist?(local_template_dir)
+      Find.find(local_template_dir).find_all { |p| p =~ /.erb$/ && !File.directory?(p) }
+          .collect { |p| p.chomp('.erb') }
+          .to_a
     else
-      puts "#{path} does not exist. Check that you are working in your module configs directory or that you have passed in the correct directory with -c."
+      $stdout.puts "#{local_template_dir} does not exist." \
+        ' Check that you are working in your module configs directory or' \
+        ' that you have passed in the correct directory with -c.'
       exit
     end
   end
 
-  def self.module_files(local_files, path)
-    local_files.map { |file| file.sub(/#{path}/, '') }
+  def self.relative_names(file_list, path)
+    file_list.map { |file| file.sub(/#{path}/, '') }
   end
 
-  def self.managed_modules(path, filter, negative_filter)
-    managed_modules = Util.parse_config(path)
+  def self.managed_modules(config_file, filter, negative_filter)
+    managed_modules = Util.parse_config(config_file)
     if managed_modules.empty?
-      puts "No modules found at #{path}. Check that you specified the right configs directory containing managed_modules.yml."
+      $stdout.puts "No modules found in #{config_file}." \
+        ' Check that you specified the right :configs directory and :managed_modules_conf file.'
       exit
     end
     managed_modules.select! { |m| m =~ Regexp.new(filter) } unless filter.nil?
@@ -71,72 +79,109 @@ module ModuleSync
   end
 
   def self.manage_file(filename, settings, options)
+    namespace = settings.additional_settings[:namespace]
     module_name = settings.additional_settings[:puppet_module]
     configs = settings.build_file_configs(filename)
+    target_file = module_file(options[:project_root], namespace, module_name, filename)
     if configs['delete']
-      Renderer.remove(module_file(options[:project_root], module_name, filename))
+      Renderer.remove(target_file)
     else
       templatename = local_file(options[:configs], filename)
       begin
         erb = Renderer.build(templatename)
-        template = Renderer.render(erb, configs)
-        Renderer.sync(template, module_file(options[:project_root], module_name, filename))
+        # Meta data passed to the template as @metadata[:name]
+        metadata = {
+          :module_name => module_name,
+          :workdir     => module_file(options[:project_root], namespace, module_name),
+          :target_file => target_file,
+        }
+        template = Renderer.render(erb, configs, metadata)
+        Renderer.sync(template, target_file)
       rescue # rubocop:disable Lint/RescueWithoutErrorClass
-        STDERR.puts "Error while rendering #{filename}"
+        $stderr.puts "Error while rendering #{filename}"
         raise
       end
     end
   end
 
   def self.manage_module(puppet_module, module_files, module_options, defaults, options)
-    puts "Syncing #{puppet_module}"
     namespace, module_name = module_name(puppet_module, options[:namespace])
+    git_repo = File.join(namespace, module_name)
     unless options[:offline]
-      git_base = options[:git_base]
-      git_uri = "#{git_base}#{namespace}"
-      Git.pull(git_uri, module_name, options[:branch], options[:project_root], module_options || {})
+      Git.pull(options[:git_base], git_repo, options[:branch], options[:project_root], module_options || {})
     end
-    module_configs = Util.parse_config("#{options[:project_root]}/#{module_name}/#{MODULE_CONF_FILE}")
+
+    module_configs = Util.parse_config(module_file(options[:project_root], namespace, module_name, MODULE_CONF_FILE))
     settings = Settings.new(defaults[GLOBAL_DEFAULTS_KEY] || {},
                             defaults,
                             module_configs[GLOBAL_DEFAULTS_KEY] || {},
                             module_configs,
                             :puppet_module => module_name,
-                            :git_base => git_base,
+                            :git_base => options[:git_base],
                             :namespace => namespace)
     settings.unmanaged_files(module_files).each do |filename|
-      puts "Not managing #{filename} in #{module_name}"
+      $stdout.puts "Not managing #{filename} in #{module_name}"
     end
 
     files_to_manage = settings.managed_files(module_files)
     files_to_manage.each { |filename| manage_file(filename, settings, options) }
 
     if options[:noop]
-      Git.update_noop(module_name, options)
+      Git.update_noop(git_repo, options)
     elsif !options[:offline]
-      Git.update(module_name, files_to_manage, options)
+      pushed = Git.update(git_repo, files_to_manage, options)
+      pushed && options[:pr] && @pr.manage(namespace, module_name, options)
     end
   end
 
   def self.update(options)
     options = config_defaults.merge(options)
-    defaults = Util.parse_config("#{options[:configs]}/#{CONF_FILE}")
+    defaults = Util.parse_config(File.join(options[:configs], CONF_FILE))
+    if options[:pr]
+      unless options[:branch]
+        $stderr.puts 'A branch must be specified with --branch to use --pr!'
+        raise
+      end
+
+      @pr = create_pr_manager if options[:pr]
+    end
 
-    path = "#{options[:configs]}/#{MODULE_FILES_DIR}"
-    local_files = self.local_files(path)
-    module_files = self.module_files(local_files, path)
+    local_template_dir = File.join(options[:configs], MODULE_FILES_DIR)
+    local_files = find_template_files(local_template_dir)
+    module_files = relative_names(local_files, local_template_dir)
 
-    managed_modules = self.managed_modules("#{options[:configs]}/managed_modules.yml", options[:filter], options[:negative_filter])
+    managed_modules = self.managed_modules(File.join(options[:configs], options[:managed_modules_conf]),
+                                           options[:filter],
+                                           options[:negative_filter])
 
+    errors = false
     # managed_modules is either an array or a hash
     managed_modules.each do |puppet_module, module_options|
       begin
         manage_module(puppet_module, module_files, module_options, defaults, options)
       rescue # rubocop:disable Lint/RescueWithoutErrorClass
-        STDERR.puts "Error while updating #{puppet_module}"
+        $stderr.puts "Error while updating #{puppet_module}"
         raise unless options[:skip_broken]
-        puts "Skipping #{puppet_module} as update process failed"
+        errors = true
+        $stdout.puts "Skipping #{puppet_module} as update process failed"
       end
     end
+    exit 1 if errors && options[:fail_on_warnings]
+  end
+
+  def self.create_pr_manager
+    github_token = ENV.fetch('GITHUB_TOKEN', '')
+    gitlab_token = ENV.fetch('GITLAB_TOKEN', '')
+
+    if !github_token.empty?
+      require 'modulesync/pr/github'
+      ModuleSync::PR::GitHub.new(github_token, ENV.fetch('GITHUB_BASE_URL', 'https://api.github.com'))
+    elsif !gitlab_token.empty?
+      require 'modulesync/pr/github'
+      ModuleSync::PR::GitLab.new(gitlab_token, ENV.fetch('GITLAB_BASE_URL', 'https://gitlab.com/api/v4'))
+    else
+      $stderr.puts 'Environment variables GITHUB_TOKEN or GITLAB_TOKEN must be set to use --pr!'
+      raise
+    end
   end
 end

data/lib/modulesync/cli.rb

@@ -43,7 +43,8 @@ module ModuleSync
                    :default => CLI.defaults[:git_base] || 'git@github.com:'
       class_option :namespace,
                    :aliases => '-n',
-                   :desc => 'Remote github namespace (user or organization) to clone from and push to. Defaults to puppetlabs',
+                   :desc => 'Remote github namespace (user or organization) to clone from and push to.' \
+                              ' Defaults to puppetlabs',
                    :default => CLI.defaults[:namespace] || 'puppetlabs'
       class_option :filter,
                    :aliases => '-f',
@@ -53,8 +54,9 @@ module ModuleSync
                    :desc => 'A regular expression to skip repositories.'
       class_option :branch,
                    :aliases => '-b',
-                   :desc => 'Branch name to make the changes in. Defaults to master.',
-                   :default => CLI.defaults[:branch] || 'master'
+                   :desc => 'Branch name to make the changes in.' \
+                              ' Defaults to the default branch of the upstream repository, but falls back to "master".',
+                   :default => CLI.defaults[:branch]
 
       desc 'update', 'Update the modules in managed_modules.yml'
       option :message,
@@ -63,7 +65,8 @@ module ModuleSync
              :default => CLI.defaults[:message]
       option :configs,
              :aliases => '-c',
-             :desc => 'The local directory or remote repository to define the list of managed modules, the file templates, and the default values for template variables.'
+             :desc => 'The local directory or remote repository to define the list of managed modules,' \
+                        ' the file templates, and the default values for template variables.'
       option :remote_branch,
              :aliases => '-r',
              :desc => 'Remote branch name to push the changes to. Defaults to the branch name.',
@@ -85,6 +88,17 @@ module ModuleSync
              :type => :boolean,
              :desc => 'No-op mode',
              :default => false
+      option :pr,
+             :type => :boolean,
+             :desc => 'Submit pull/merge request',
+             :default => false
+      option :pr_title,
+             :desc => 'Title of pull/merge request',
+             :default => CLI.defaults[:pr_title] || 'Update to module template files'
+      option :pr_labels,
+             :type => :array,
+             :desc => 'Labels to add to the pull/merge request',
+             :default => CLI.defaults[:pr_labels] || []
       option :offline,
              :type => :boolean,
              :desc => 'Do not run any Git commands. Allows the user to manage Git outside of ModuleSync.',
@@ -104,13 +118,21 @@ module ModuleSync
       option :tag_pattern,
              :desc => 'The pattern to use when tagging releases.'
       option :pre_commit_script,
-             :desc => 'A script to be run before commiting',
+             :desc => 'A script to be run before committing',
              :default => CLI.defaults[:pre_commit_script]
+      option :fail_on_warnings,
+             :type => :boolean,
+             :aliases => '-F',
+             :desc => 'Produce a failure exit code when there are warnings' \
+                        ' (only has effect when --skip_broken is enabled)',
+             :default => false
 
       def update
         config = { :command => 'update' }.merge(options)
         config = Util.symbolize_keys(config)
-        raise Thor::Error, 'No value provided for required option "--message"' unless config[:noop] || config[:message] || config[:offline]
+        raise Thor::Error, 'No value provided for required option "--message"' unless config[:noop] \
+                                                                                      || config[:message] \
+                                                                                      || config[:offline]
         config[:git_opts] = { 'amend' => config[:amend], 'force' => config[:force] }
         ModuleSync.update(config)
       end

data/lib/modulesync/git.rb

@@ -2,7 +2,7 @@ require 'git'
 require 'puppet_blacksmith'
 
 module ModuleSync
-  module Git
+  module Git # rubocop:disable Metrics/ModuleLength
     include Constants
 
     def self.remote_branch_exists?(repo, branch)
@@ -18,7 +18,17 @@ module ModuleSync
         repo.diff("#{local_branch}..origin/#{remote_branch}").any?
     end
 
+    def self.default_branch(repo)
+      symbolic_ref = repo.branches.find { |b| b.full =~ %r{remotes/origin/HEAD} }
+      return unless symbolic_ref
+      %r{remotes/origin/HEAD\s+->\s+origin/(?<branch>.+?)$}.match(symbolic_ref.full)[:branch]
+    end
+
     def self.switch_branch(repo, branch)
+      unless branch
+        branch = default_branch(repo)
+        puts "Using repository's default branch: #{branch}"
+      end
       return if repo.current_branch == branch
 
       if local_branch_exists?(repo, branch)
@@ -36,12 +46,13 @@ module ModuleSync
     end
 
     def self.pull(git_base, name, branch, project_root, opts)
+      puts "Syncing #{name}"
       Dir.mkdir(project_root) unless Dir.exist?(project_root)
 
       # Repo needs to be cloned in the cwd
       if !Dir.exist?("#{project_root}/#{name}") || !Dir.exist?("#{project_root}/#{name}/.git")
         puts 'Cloning repository fresh'
-        remote = opts[:remote] || (git_base.start_with?('file://') ? "#{git_base}/#{name}" : "#{git_base}/#{name}.git")
+        remote = opts[:remote] || (git_base.start_with?('file://') ? "#{git_base}#{name}" : "#{git_base}#{name}.git")
         local = "#{project_root}/#{name}"
         puts "Cloning from #{remote}"
         repo = ::Git.clone(remote, local)
@@ -95,12 +106,19 @@ module ModuleSync
       repo.push('origin', tag)
     end
 
+    def self.checkout_branch(repo, branch)
+      selected_branch = branch || repo.current_branch || 'master'
+      repo.branch(selected_branch).checkout
+      selected_branch
+    end
+    private_class_method :checkout_branch
+
     # Git add/rm, git commit, git push
     def self.update(name, files, options)
       module_root = "#{options[:project_root]}/#{name}"
       message = options[:message]
       repo = ::Git.open(module_root)
-      repo.branch(options[:branch]).checkout
+      branch = checkout_branch(repo, options[:branch])
       files.each do |file|
         if repo.status.deleted.include?(file)
           repo.remove(file)
@@ -119,11 +137,11 @@ module ModuleSync
         end
         repo.commit(message, opts_commit)
         if options[:remote_branch]
-          if remote_branch_differ?(repo, options[:branch], options[:remote_branch])
-            repo.push('origin', "#{options[:branch]}:#{options[:remote_branch]}", opts_push)
+          if remote_branch_differ?(repo, branch, options[:remote_branch])
+            repo.push('origin', "#{branch}:#{options[:remote_branch]}", opts_push)
           end
         else
-          repo.push('origin', options[:branch], opts_push)
+          repo.push('origin', branch, opts_push)
         end
         # Only bump/tag if pushing didn't fail (i.e. there were changes)
         m = Blacksmith::Modulefile.new("#{module_root}/metadata.json")
@@ -134,11 +152,14 @@ module ModuleSync
       rescue ::Git::GitExecuteError => git_error
         if git_error.message =~ /working (directory|tree) clean/
           puts "There were no files to update in #{name}. Not committing."
+          return false
         else
           puts git_error
-          exit(1)
+          raise
         end
       end
+
+      true
     end
 
     # Needed because of a bug in the git gem that lists ignored files as
@@ -154,7 +175,7 @@ module ModuleSync
       puts "Using no-op. Files in #{name} may be changed but will not be committed."
 
       repo = ::Git.open("#{options[:project_root]}/#{name}")
-      repo.branch(options[:branch]).checkout
+      checkout_branch(repo, options[:branch])
 
       puts 'Files changed:'
       repo.diff('HEAD', '--').each do |diff|

data/lib/modulesync/pr/github.rb

@@ -0,0 +1,41 @@
+require 'octokit'
+require 'modulesync/util'
+
+module ModuleSync
+  module PR
+    # GitHub creates and manages pull requests on github.com or GitHub
+    # Enterprise installations.
+    class GitHub
+      def initialize(token, endpoint)
+        Octokit.configure do |c|
+          c.api_endpoint = endpoint
+        end
+        @api = Octokit::Client.new(:access_token => token)
+      end
+
+      def manage(namespace, module_name, options)
+        repo_path = File.join(namespace, module_name)
+        head = "#{namespace}:#{options[:branch]}"
+
+        pull_requests = @api.pull_requests(repo_path, :state => 'open', :base => 'master', :head => head)
+        if pull_requests.empty?
+          pr = @api.create_pull_request(repo_path, 'master', options[:branch], options[:pr_title], options[:message])
+          $stdout.puts "Submitted PR '#{options[:pr_title]}' to #{repo_path} - merges #{options[:branch]} into master"
+        else
+          # Skip creating the PR if it exists already.
+          $stdout.puts "Skipped! #{pull_requests.length} PRs found for branch #{options[:branch]}"
+        end
+
+        # PR labels can either be a list in the YAML file or they can pass in a comma
+        # separated list via the command line argument.
+        pr_labels = ModuleSync::Util.parse_list(options[:pr_labels])
+
+        # We only assign labels to the PR if we've discovered a list > 1. The labels MUST
+        # already exist. We DO NOT create missing labels.
+        return if pr_labels.empty?
+        $stdout.puts "Attaching the following labels to PR #{pr['number']}: #{pr_labels.join(', ')}"
+        @api.add_labels_to_an_issue(repo_path, pr['number'], pr_labels)
+      end
+    end
+  end
+end

data/lib/modulesync/pr/gitlab.rb

@@ -0,0 +1,40 @@
+require 'gitlab'
+require 'modulesync/util'
+
+module ModuleSync
+  module PR
+    # GitLab creates and manages merge requests on gitlab.com or private GitLab
+    # installations.
+    class GitLab
+      def initialize(token, endpoint)
+        @api = Gitlab::Client.new(
+          :endpoint => endpoint,
+          :private_token => token
+        )
+      end
+
+      def manage(namespace, module_name, options)
+        repo_path = File.join(namespace, module_name)
+
+        head = "#{namespace}:#{options[:branch]}"
+        merge_requests = @api.merge_requests(repo_path,
+                                             :state => 'opened',
+                                             :source_branch => head,
+                                             :target_branch => 'master')
+        if merge_requests.empty?
+          mr_labels = ModuleSync::Util.parse_list(options[:pr_labels])
+          mr = @api.create_merge_request(repo_path, options[:pr_title],
+                                         :source_branch => options[:branch],
+                                         :target_branch => 'master',
+                                         :labels => mr_labels)
+          $stdout.puts "Submitted MR '#{options[:pr_title]}' to #{repo_path} - merges #{options[:branch]} into master"
+          return if mr_labels.empty?
+          $stdout.puts "Attached the following labels to MR #{mr.iid}: #{mr_labels.join(', ')}"
+        else
+          # Skip creating the MR if it exists already.
+          $stdout.puts "Skipped! #{merge_requests.length} MRs found for branch #{options[:branch]}"
+        end
+      end
+    end
+  end
+end

data/lib/modulesync/renderer.rb

@@ -4,8 +4,9 @@ require 'find'
 module ModuleSync
   module Renderer
     class ForgeModuleFile
-      def initialize(configs = {})
+      def initialize(configs = {}, metadata = {})
         @configs = configs
+        @metadata = metadata
       end
     end
 
@@ -26,8 +27,8 @@ module ModuleSync
       File.delete(file) if File.exist?(file)
     end
 
-    def self.render(_template, configs = {})
-      ForgeModuleFile.new(configs).render
+    def self.render(_template, configs = {}, metadata = {})
+      ForgeModuleFile.new(configs, metadata).render
     end
 
     def self.sync(template, target_name)