modulesync 0.9.0 → 1.3.0

data/lib/modulesync.rb

@@ -9,7 +9,7 @@ require 'modulesync/settings'
 require 'modulesync/util'
 require 'monkey_patches'
 
-module ModuleSync
+module ModuleSync # rubocop:disable Metrics/ModuleLength
   include Constants
 
   def self.config_defaults
@@ -22,31 +22,39 @@ module ModuleSync
   end
 
   def self.local_file(config_path, file)
-    "#{config_path}/#{MODULE_FILES_DIR}/#{file}"
+    File.join(config_path, MODULE_FILES_DIR, file)
   end
 
-  def self.module_file(project_root, puppet_module, file)
-    "#{project_root}/#{puppet_module}/#{file}"
+  def self.module_file(project_root, namespace, puppet_module, *parts)
+    File.join(project_root, namespace, puppet_module, *parts)
   end
 
-  def self.local_files(path)
-    if File.exist?(path)
-      # only select *.erb files, and strip the extension. This way all the code will only have to handle bare paths, except when reading the actual ERB text
-      local_files = Find.find(path).find_all { |p| p =~ /.erb$/ && !File.directory?(p) }.collect { |p| p.chomp('.erb') }.to_a
+  # List all template files.
+  #
+  # Only select *.erb files, and strip the extension. This way all the code will only have to handle bare paths,
+  # except when reading the actual ERB text
+  def self.find_template_files(local_template_dir)
+    if File.exist?(local_template_dir)
+      Find.find(local_template_dir).find_all { |p| p =~ /.erb$/ && !File.directory?(p) }
+          .collect { |p| p.chomp('.erb') }
+          .to_a
     else
-      puts "#{path} does not exist. Check that you are working in your module configs directory or that you have passed in the correct directory with -c."
+      $stdout.puts "#{local_template_dir} does not exist." \
+        ' Check that you are working in your module configs directory or' \
+        ' that you have passed in the correct directory with -c.'
       exit
     end
   end
 
-  def self.module_files(local_files, path)
-    local_files.map { |file| file.sub(/#{path}/, '') }
+  def self.relative_names(file_list, path)
+    file_list.map { |file| file.sub(/#{path}/, '') }
   end
 
   def self.managed_modules(config_file, filter, negative_filter)
     managed_modules = Util.parse_config(config_file)
     if managed_modules.empty?
-      puts "No modules found in #{config_file}. Check that you specified the right :configs directory and :managed_modules_conf file."
+      $stdout.puts "No modules found in #{config_file}." \
+        ' Check that you specified the right :configs directory and :managed_modules_conf file.'
       exit
     end
     managed_modules.select! { |m| m =~ Regexp.new(filter) } unless filter.nil?
@@ -71,62 +79,93 @@ module ModuleSync
   end
 
   def self.manage_file(filename, settings, options)
+    namespace = settings.additional_settings[:namespace]
     module_name = settings.additional_settings[:puppet_module]
     configs = settings.build_file_configs(filename)
+    target_file = module_file(options[:project_root], namespace, module_name, filename)
     if configs['delete']
-      Renderer.remove(module_file(options[:project_root], module_name, filename))
+      Renderer.remove(target_file)
     else
       templatename = local_file(options[:configs], filename)
       begin
         erb = Renderer.build(templatename)
-        template = Renderer.render(erb, configs)
-        Renderer.sync(template, module_file(options[:project_root], module_name, filename))
+        # Meta data passed to the template as @metadata[:name]
+        metadata = {
+          :module_name => module_name,
+          :workdir     => module_file(options[:project_root], namespace, module_name),
+          :target_file => target_file,
+        }
+        template = Renderer.render(erb, configs, metadata)
+        Renderer.sync(template, target_file)
       rescue # rubocop:disable Lint/RescueWithoutErrorClass
-        STDERR.puts "Error while rendering #{filename}"
+        $stderr.puts "Error while rendering #{filename}"
         raise
       end
     end
   end
 
   def self.manage_module(puppet_module, module_files, module_options, defaults, options)
-    puts "Syncing #{puppet_module}"
-    namespace, module_name = module_name(puppet_module, options[:namespace])
+    default_namespace = options[:namespace]
+    if module_options.is_a?(Hash) && module_options.key?(:namespace)
+      default_namespace = module_options[:namespace]
+    end
+    namespace, module_name = module_name(puppet_module, default_namespace)
+    git_repo = File.join(namespace, module_name)
     unless options[:offline]
-      git_base = options[:git_base]
-      git_uri = "#{git_base}#{namespace}"
-      Git.pull(git_uri, module_name, options[:branch], options[:project_root], module_options || {})
+      Git.pull(options[:git_base], git_repo, options[:branch], options[:project_root], module_options || {})
     end
-    module_configs = Util.parse_config("#{options[:project_root]}/#{module_name}/#{MODULE_CONF_FILE}")
+
+    module_configs = Util.parse_config(module_file(options[:project_root], namespace, module_name, MODULE_CONF_FILE))
     settings = Settings.new(defaults[GLOBAL_DEFAULTS_KEY] || {},
                             defaults,
                             module_configs[GLOBAL_DEFAULTS_KEY] || {},
                             module_configs,
                             :puppet_module => module_name,
-                            :git_base => git_base,
+                            :git_base => options[:git_base],
                             :namespace => namespace)
     settings.unmanaged_files(module_files).each do |filename|
-      puts "Not managing #{filename} in #{module_name}"
+      $stdout.puts "Not managing #{filename} in #{module_name}"
     end
 
     files_to_manage = settings.managed_files(module_files)
     files_to_manage.each { |filename| manage_file(filename, settings, options) }
 
     if options[:noop]
-      Git.update_noop(module_name, options)
+      Git.update_noop(git_repo, options)
     elsif !options[:offline]
-      Git.update(module_name, files_to_manage, options)
+      pushed = Git.update(git_repo, files_to_manage, options)
+      pushed && options[:pr] && pr(module_options).manage(namespace, module_name, options)
     end
   end
 
+  def self.config_path(file, options)
+    return file if Pathname.new(file).absolute?
+    File.join(options[:configs], file)
+  end
+
+  def config_path(file, options)
+    self.class.config_path(file, options)
+  end
+
   def self.update(options)
     options = config_defaults.merge(options)
-    defaults = Util.parse_config("#{options[:configs]}/#{CONF_FILE}")
+    defaults = Util.parse_config(config_path(CONF_FILE, options))
+    if options[:pr]
+      unless options[:branch]
+        $stderr.puts 'A branch must be specified with --branch to use --pr!'
+        raise
+      end
+
+      @pr = create_pr_manager if options[:pr]
+    end
 
-    path = "#{options[:configs]}/#{MODULE_FILES_DIR}"
-    local_files = self.local_files(path)
-    module_files = self.module_files(local_files, path)
+    local_template_dir = config_path(MODULE_FILES_DIR, options)
+    local_files = find_template_files(local_template_dir)
+    module_files = relative_names(local_files, local_template_dir)
 
-    managed_modules = self.managed_modules("#{options[:configs]}/#{options[:managed_modules_conf]}", options[:filter], options[:negative_filter])
+    managed_modules = self.managed_modules(config_path(options[:managed_modules_conf], options),
+                                           options[:filter],
+                                           options[:negative_filter])
 
     errors = false
     # managed_modules is either an array or a hash
@@ -134,12 +173,47 @@ module ModuleSync
       begin
         manage_module(puppet_module, module_files, module_options, defaults, options)
       rescue # rubocop:disable Lint/RescueWithoutErrorClass
-        STDERR.puts "Error while updating #{puppet_module}"
+        $stderr.puts "Error while updating #{puppet_module}"
         raise unless options[:skip_broken]
         errors = true
-        puts "Skipping #{puppet_module} as update process failed"
+        $stdout.puts "Skipping #{puppet_module} as update process failed"
       end
     end
     exit 1 if errors && options[:fail_on_warnings]
   end
+
+  def self.pr(module_options)
+    github_conf = module_options[:github]
+    gitlab_conf = module_options[:gitlab]
+
+    if !github_conf.nil?
+      base_url = github_conf[:base_url] || ENV.fetch('GITHUB_BASE_URL', 'https://api.github.com')
+      require 'modulesync/pr/github'
+      ModuleSync::PR::GitHub.new(github_conf[:token], base_url)
+    elsif !gitlab_conf.nil?
+      base_url = gitlab_conf[:base_url] || ENV.fetch('GITLAB_BASE_URL', 'https://gitlab.com/api/v4')
+      require 'modulesync/pr/gitlab'
+      ModuleSync::PR::GitLab.new(gitlab_conf[:token], base_url)
+    elsif @pr.nil?
+      $stderr.puts 'No GitHub or GitLab token specified for --pr!'
+      raise
+    else
+      @pr
+    end
+  end
+
+  def self.create_pr_manager
+    github_token = ENV.fetch('GITHUB_TOKEN', '')
+    gitlab_token = ENV.fetch('GITLAB_TOKEN', '')
+
+    if !github_token.empty?
+      require 'modulesync/pr/github'
+      ModuleSync::PR::GitHub.new(github_token, ENV.fetch('GITHUB_BASE_URL', 'https://api.github.com'))
+    elsif !gitlab_token.empty?
+      require 'modulesync/pr/gitlab'
+      ModuleSync::PR::GitLab.new(gitlab_token, ENV.fetch('GITLAB_BASE_URL', 'https://gitlab.com/api/v4'))
+    else
+      warn '--pr specified without environment variables GITHUB_TOKEN or GITLAB_TOKEN'
+    end
+  end
 end

data/lib/modulesync/cli.rb

@@ -43,7 +43,8 @@ module ModuleSync
                    :default => CLI.defaults[:git_base] || 'git@github.com:'
       class_option :namespace,
                    :aliases => '-n',
-                   :desc => 'Remote github namespace (user or organization) to clone from and push to. Defaults to puppetlabs',
+                   :desc => 'Remote github namespace (user or organization) to clone from and push to.' \
+                              ' Defaults to puppetlabs',
                    :default => CLI.defaults[:namespace] || 'puppetlabs'
       class_option :filter,
                    :aliases => '-f',
@@ -53,7 +54,8 @@ module ModuleSync
                    :desc => 'A regular expression to skip repositories.'
       class_option :branch,
                    :aliases => '-b',
-                   :desc => 'Branch name to make the changes in. Defaults to the default branch of the upstream repository, but falls back to "master".',
+                   :desc => 'Branch name to make the changes in.' \
+                              ' Defaults to the default branch of the upstream repository, but falls back to "master".',
                    :default => CLI.defaults[:branch]
 
       desc 'update', 'Update the modules in managed_modules.yml'
@@ -63,7 +65,10 @@ module ModuleSync
              :default => CLI.defaults[:message]
       option :configs,
              :aliases => '-c',
-             :desc => 'The local directory or remote repository to define the list of managed modules, the file templates, and the default values for template variables.'
+             :desc => 'The local directory or remote repository to define the list of managed modules,' \
+                        ' the file templates, and the default values for template variables.'
+      option :managed_modules_conf,
+             :desc => 'The file name to define the list of managed modules'
       option :remote_branch,
              :aliases => '-r',
              :desc => 'Remote branch name to push the changes to. Defaults to the branch name.',
@@ -85,6 +90,20 @@ module ModuleSync
              :type => :boolean,
              :desc => 'No-op mode',
              :default => false
+      option :pr,
+             :type => :boolean,
+             :desc => 'Submit pull/merge request',
+             :default => false
+      option :pr_title,
+             :desc => 'Title of pull/merge request',
+             :default => CLI.defaults[:pr_title] || 'Update to module template files'
+      option :pr_labels,
+             :type => :array,
+             :desc => 'Labels to add to the pull/merge request',
+             :default => CLI.defaults[:pr_labels] || []
+      option :pr_target_branch,
+             :desc => 'Target branch for the pull/merge request',
+             :default => CLI.defaults[:pr_target_branch] || 'master'
       option :offline,
              :type => :boolean,
              :desc => 'Do not run any Git commands. Allows the user to manage Git outside of ModuleSync.',
@@ -104,18 +123,21 @@ module ModuleSync
       option :tag_pattern,
              :desc => 'The pattern to use when tagging releases.'
       option :pre_commit_script,
-             :desc => 'A script to be run before commiting',
+             :desc => 'A script to be run before committing',
              :default => CLI.defaults[:pre_commit_script]
       option :fail_on_warnings,
              :type => :boolean,
              :aliases => '-F',
-             :desc => 'Produce a failure exit code when there are warnings (only has effect when --skip_broken is enabled)',
+             :desc => 'Produce a failure exit code when there are warnings' \
+                        ' (only has effect when --skip_broken is enabled)',
              :default => false
 
       def update
         config = { :command => 'update' }.merge(options)
         config = Util.symbolize_keys(config)
-        raise Thor::Error, 'No value provided for required option "--message"' unless config[:noop] || config[:message] || config[:offline]
+        raise Thor::Error, 'No value provided for required option "--message"' unless config[:noop] \
+                                                                                      || config[:message] \
+                                                                                      || config[:offline]
         config[:git_opts] = { 'amend' => config[:amend], 'force' => config[:force] }
         ModuleSync.update(config)
       end

data/lib/modulesync/git.rb

@@ -46,12 +46,13 @@ module ModuleSync
     end
 
     def self.pull(git_base, name, branch, project_root, opts)
+      puts "Syncing #{name}"
       Dir.mkdir(project_root) unless Dir.exist?(project_root)
 
       # Repo needs to be cloned in the cwd
       if !Dir.exist?("#{project_root}/#{name}") || !Dir.exist?("#{project_root}/#{name}/.git")
         puts 'Cloning repository fresh'
-        remote = opts[:remote] || (git_base.start_with?('file://') ? "#{git_base}/#{name}" : "#{git_base}/#{name}.git")
+        remote = opts[:remote] || (git_base.start_with?('file://') ? "#{git_base}#{name}" : "#{git_base}#{name}.git")
         local = "#{project_root}/#{name}"
         puts "Cloning from #{remote}"
         repo = ::Git.clone(remote, local)
@@ -151,11 +152,14 @@ module ModuleSync
       rescue ::Git::GitExecuteError => git_error
         if git_error.message =~ /working (directory|tree) clean/
           puts "There were no files to update in #{name}. Not committing."
+          return false
         else
           puts git_error
           raise
         end
       end
+
+      true
     end
 
     # Needed because of a bug in the git gem that lists ignored files as

data/lib/modulesync/pr/github.rb

@@ -0,0 +1,47 @@
+require 'octokit'
+require 'modulesync/util'
+
+module ModuleSync
+  module PR
+    # GitHub creates and manages pull requests on github.com or GitHub
+    # Enterprise installations.
+    class GitHub
+      def initialize(token, endpoint)
+        Octokit.configure do |c|
+          c.api_endpoint = endpoint
+        end
+        @api = Octokit::Client.new(:access_token => token)
+      end
+
+      def manage(namespace, module_name, options)
+        repo_path = File.join(namespace, module_name)
+        head = "#{namespace}:#{options[:branch]}"
+        target_branch = options[:pr_target_branch] || 'master'
+
+        pull_requests = @api.pull_requests(repo_path, :state => 'open', :base => target_branch, :head => head)
+        if pull_requests.empty?
+          pr = @api.create_pull_request(repo_path,
+                                        target_branch,
+                                        options[:branch],
+                                        options[:pr_title],
+                                        options[:message])
+          $stdout.puts \
+            "Submitted PR '#{options[:pr_title]}' to #{repo_path} - merges #{options[:branch]} into #{target_branch}"
+        else
+          # Skip creating the PR if it exists already.
+          $stdout.puts "Skipped! #{pull_requests.length} PRs found for branch #{options[:branch]}"
+        end
+
+        # PR labels can either be a list in the YAML file or they can pass in a comma
+        # separated list via the command line argument.
+        pr_labels = ModuleSync::Util.parse_list(options[:pr_labels])
+
+        # We only assign labels to the PR if we've discovered a list > 1. The labels MUST
+        # already exist. We DO NOT create missing labels.
+        return if pr_labels.empty?
+        $stdout.puts "Attaching the following labels to PR #{pr['number']}: #{pr_labels.join(', ')}"
+        @api.add_labels_to_an_issue(repo_path, pr['number'], pr_labels)
+      end
+    end
+  end
+end

data/lib/modulesync/pr/gitlab.rb

@@ -0,0 +1,42 @@
+require 'gitlab'
+require 'modulesync/util'
+
+module ModuleSync
+  module PR
+    # GitLab creates and manages merge requests on gitlab.com or private GitLab
+    # installations.
+    class GitLab
+      def initialize(token, endpoint)
+        @api = Gitlab::Client.new(
+          :endpoint => endpoint,
+          :private_token => token
+        )
+      end
+
+      def manage(namespace, module_name, options)
+        repo_path = File.join(namespace, module_name)
+
+        head = "#{namespace}:#{options[:branch]}"
+        target_branch = options[:pr_target_branch] || 'master'
+        merge_requests = @api.merge_requests(repo_path,
+                                             :state => 'opened',
+                                             :source_branch => head,
+                                             :target_branch => target_branch)
+        if merge_requests.empty?
+          mr_labels = ModuleSync::Util.parse_list(options[:pr_labels])
+          mr = @api.create_merge_request(repo_path, options[:pr_title],
+                                         :source_branch => options[:branch],
+                                         :target_branch => target_branch,
+                                         :labels => mr_labels)
+          $stdout.puts \
+            "Submitted MR '#{options[:pr_title]}' to #{repo_path} - merges #{options[:branch]} into #{target_branch}"
+          return if mr_labels.empty?
+          $stdout.puts "Attached the following labels to MR #{mr.iid}: #{mr_labels.join(', ')}"
+        else
+          # Skip creating the MR if it exists already.
+          $stdout.puts "Skipped! #{merge_requests.length} MRs found for branch #{options[:branch]}"
+        end
+      end
+    end
+  end
+end

data/lib/modulesync/renderer.rb

@@ -4,8 +4,9 @@ require 'find'
 module ModuleSync
   module Renderer
     class ForgeModuleFile
-      def initialize(configs = {})
+      def initialize(configs = {}, metadata = {})
         @configs = configs
+        @metadata = metadata
       end
     end
 
@@ -26,8 +27,8 @@ module ModuleSync
       File.delete(file) if File.exist?(file)
     end
 
-    def self.render(_template, configs = {})
-      ForgeModuleFile.new(configs).render
+    def self.render(_template, configs = {}, metadata = {})
+      ForgeModuleFile.new(configs, metadata).render
     end
 
     def self.sync(template, target_name)

data/lib/modulesync/settings.rb

@@ -19,10 +19,9 @@ module ModuleSync
 
     def build_file_configs(target_name)
       file_def = lookup_config(defaults, target_name)
-      file_md  = lookup_config(module_defaults, target_name)
       file_mc  = lookup_config(module_configs, target_name)
 
-      global_defaults.merge(file_def).merge(file_md).merge(file_mc).merge(additional_settings)
+      global_defaults.merge(file_def).merge(module_defaults).merge(file_mc).merge(additional_settings)
     end
 
     def managed?(target_name)