modulesync 0.10.0 → 2.0.0

data/Rakefile

@@ -19,3 +19,28 @@ Cucumber::Rake::Task.new do |t|
 end
 
 task :test => %i[clean spec cucumber rubocop]
+task :default => %i[test]
+
+begin
+  require 'github_changelog_generator/task'
+  GitHubChangelogGenerator::RakeTask.new :changelog do |config|
+    config.header = "# Changelog\n\nAll notable changes to this project will be documented in this file."
+    config.exclude_labels = %w[duplicate question invalid wontfix wont-fix modulesync skip-changelog]
+    config.user = 'voxpupuli'
+    config.project = 'modulesync'
+    config.future_release = Gem::Specification.load("#{config.project}.gemspec").version
+  end
+
+  # Workaround for https://github.com/github-changelog-generator/github-changelog-generator/issues/715
+  require 'rbconfig'
+  if RbConfig::CONFIG['host_os'] =~ /linux/
+    task :changelog do
+      puts 'Fixing line endings...'
+      changelog_file = File.join(__dir__, 'CHANGELOG.md')
+      changelog_txt = File.read(changelog_file)
+      new_contents = changelog_txt.gsub(/\r\n/, "\n")
+      File.open(changelog_file, 'w') { |file| file.puts new_contents }
+    end
+  end
+rescue LoadError
+end

data/features/step_definitions/git_steps.rb

@@ -23,7 +23,7 @@ Given 'a remote module repository' do
   CONFIG
 end
 
-Given /a remote module repository with "(.+?)" as the default branch/ do |branch| # rubocop:disable Lint/AmbiguousRegexpLiteral
+Given Regexp.new(/a remote module repository with "(.+?)" as the default branch/) do |branch|
   steps %(
     Given a directory named "sources"
     And I run `git clone --mirror https://github.com/maestrodev/puppet-test sources/puppet-test`

data/features/update.feature

@@ -31,7 +31,7 @@ Feature: update
       Files added:
       test
       """
-    Given I run `cat modules/puppet-test/test`
+    Given I run `cat modules/maestrodev/puppet-test/test`
     Then the output should contain "aruba"
 
   Scenario: Using skip_broken option and adding a new file to repo without write access
@@ -113,14 +113,14 @@ Feature: update
     Then the exit status should be 0
     And the output should match:
       """
-      Warning: using './moduleroot//test' as template without '.erb' suffix
+      Warning: using './moduleroot/test' as template without '.erb' suffix
       """
     And the output should match:
       """
       Files added:
       test
       """
-    Given I run `cat modules/puppet-test/test`
+    Given I run `cat modules/maestrodev/puppet-test/test`
     Then the output should contain "aruba"
 
   Scenario: Adding a new file using global values
@@ -153,7 +153,7 @@ Feature: update
       Files added:
       test
       """
-    Given I run `cat modules/puppet-test/test`
+    Given I run `cat modules/maestrodev/puppet-test/test`
     Then the output should contain "aruba"
 
   Scenario: Adding a new file overriding global values
@@ -189,7 +189,7 @@ Feature: update
       Files added:
       test
       """
-    Given I run `cat modules/puppet-test/test`
+    Given I run `cat modules/maestrodev/puppet-test/test`
     Then the output should contain "aruba"
 
   Scenario: Adding a new file ignoring global values
@@ -225,7 +225,7 @@ Feature: update
       Files added:
       test
       """
-    Given I run `cat modules/puppet-test/test`
+    Given I run `cat modules/maestrodev/puppet-test/test`
     Then the output should contain "aruba"
 
   Scenario: Adding a file that ERB can't parse
@@ -342,7 +342,7 @@ Feature: update
       Files changed:
       +diff --git a/Gemfile b/Gemfile
       """
-    Given I run `cat modules/puppet-test/Gemfile`
+    Given I run `cat modules/maestrodev/puppet-test/Gemfile`
     Then the output should contain:
       """
       source 'https://somehost.com'
@@ -405,7 +405,7 @@ Feature: update
       Not managing Gemfile in puppet-test
       """
     And the exit status should be 0
-    Given I run `cat modules/puppet-test/Gemfile`
+    Given I run `cat modules/maestrodev/puppet-test/Gemfile`
     Then the output should contain:
       """
       source 'https://rubygems.org'
@@ -488,8 +488,8 @@ Feature: update
       """
       some spec_helper fud
       """
-    And a directory named "modules/puppetlabs-apache/spec"
-    And a file named "modules/puppetlabs-apache/spec/spec_helper.rb" with:
+    And a directory named "modules/puppetlabs/puppetlabs-apache/spec"
+    And a file named "modules/puppetlabs/puppetlabs-apache/spec/spec_helper.rb" with:
       """
       This is a fake spec_helper!
       """
@@ -499,7 +499,7 @@ Feature: update
       Not managing spec/spec_helper.rb in puppetlabs-apache
       """
     And the exit status should be 0
-    Given I run `cat modules/puppetlabs-apache/spec/spec_helper.rb`
+    Given I run `cat modules/puppetlabs/puppetlabs-apache/spec/spec_helper.rb`
     Then the output should contain:
       """
       This is a fake spec_helper!
@@ -538,7 +538,7 @@ Feature: update
       Files added:
       spec/spec_helper.rb
       """
-    Given I run `cat modules/puppet-test/spec/spec_helper.rb`
+    Given I run `cat modules/maestrodev/puppet-test/spec/spec_helper.rb`
     Then the output should contain:
       """
       require 'puppetlabs_spec_helper/module_helper'
@@ -577,7 +577,7 @@ Feature: update
     Given a file named "managed_modules.yml" with:
       """
       ---
-        - puppet-test
+        - maestrodev/puppet-test
       """
     And a file named "modulesync.yml" with:
       """
@@ -597,8 +597,8 @@ Feature: update
         require '<%= required %>'
       <% end %>
       """
-    Given I run `git init modules/puppet-test`
-    Given a file named "modules/puppet-test/.git/config" with:
+    Given I run `git init modules/maestrodev/puppet-test`
+    Given a file named "modules/maestrodev/puppet-test/.git/config" with:
       """
       [core]
           repositoryformatversion = 0
@@ -667,7 +667,7 @@ Feature: update
       Files added:
       test
       """
-    Given I run `cat modules/puppet-test/test`
+    Given I run `cat modules/maestrodev/puppet-test/test`
     Then the output should contain "aruba"
 
   Scenario: When specifying configurations in managed_modules.yml and using a filter
@@ -702,9 +702,9 @@ Feature: update
       Files added:
       test
       """
-    Given I run `cat modules/puppet-test/test`
+    Given I run `cat modules/maestrodev/puppet-test/test`
     Then the output should contain "aruba"
-    And a directory named "modules/puppet-blacksmith" should not exist
+    And a directory named "modules/maestrodev/puppet-blacksmith" should not exist
 
   Scenario: When specifying configurations in managed_modules.yml and using a negative filter
     Given a file named "managed_modules.yml" with:
@@ -738,15 +738,15 @@ Feature: update
       Files added:
       test
       """
-    Given I run `cat modules/puppet-test/test`
+    Given I run `cat modules/maestrodev/puppet-test/test`
     Then the output should contain "aruba"
-    And a directory named "modules/puppet-blacksmith" should not exist
+    And a directory named "modules/maestrodev/puppet-blacksmith" should not exist
 
   Scenario: Updating a module with a .sync.yml file
     Given a file named "managed_modules.yml" with:
       """
       ---
-        - puppet-test
+        - maestrodev/puppet-test
       """
     And a file named "modulesync.yml" with:
       """
@@ -756,6 +756,9 @@ Feature: update
     And a file named "config_defaults.yml" with:
       """
       ---
+      :global:
+        global-default: some-default
+        global-to-overwrite: to-be-overwritten
       spec/spec_helper.rb:
         require:
           - puppetlabs_spec_helper/module_helper
@@ -766,8 +769,14 @@ Feature: update
         require '<%= required %>'
       <% end %>
       """
-    Given I run `git init modules/puppet-test`
-    Given a file named "modules/puppet-test/.git/config" with:
+    And a file named "moduleroot/global-test.md.erb" with:
+      """
+      <%= @configs['global-default'] %>
+      <%= @configs['global-to-overwrite'] %>
+      <%= @configs['module-default'] %>
+      """
+    Given I run `git init modules/maestrodev/puppet-test`
+    Given a file named "modules/maestrodev/puppet-test/.git/config" with:
       """
       [core]
           repositoryformatversion = 0
@@ -780,9 +789,12 @@ Feature: update
           url = https://github.com/maestrodev/puppet-test.git
           fetch = +refs/heads/*:refs/remotes/origin/*
       """
-    Given a file named "modules/puppet-test/.sync.yml" with:
+    Given a file named "modules/maestrodev/puppet-test/.sync.yml" with:
       """
       ---
+      :global:
+        global-to-overwrite: it-is-overwritten
+        module-default: some-value
       spec/spec_helper.rb:
         unmanaged: true
       """
@@ -792,6 +804,13 @@ Feature: update
       """
       Not managing spec/spec_helper.rb in puppet-test
       """
+    Given I run `cat modules/maestrodev/puppet-test/global-test.md`
+    Then the output should match:
+      """
+      some-default
+      it-is-overwritten
+      some-value
+      """
 
   Scenario: Module with custom namespace
     Given a file named "managed_modules.yml" with:
@@ -824,9 +843,9 @@ Feature: update
       Files added:
       test
       """
-    Given I run `cat modules/puppet-test/.git/config`
+    Given I run `cat modules/maestrodev/puppet-test/.git/config`
     Then the output should contain "url = https://github.com/maestrodev/puppet-test.git"
-    Given I run `cat modules/puppet-lib-file_concat/.git/config`
+    Given I run `cat modules/electrical/puppet-lib-file_concat/.git/config`
     Then the output should contain "url = https://github.com/electrical/puppet-lib-file_concat.git"
 
   Scenario: Modifying an existing file with values exposed by the module
@@ -858,7 +877,7 @@ Feature: update
       Files changed:
       +diff --git a/README.md b/README.md
       """
-    Given I run `cat modules/puppet-test/README.md`
+    Given I run `cat modules/maestrodev/puppet-test/README.md`
     Then the output should contain:
       """
       echo 'https://github.com/maestrodev'
@@ -886,6 +905,28 @@ Feature: update
     Then the output should not contain "error"
     Then the output should not contain "rejected"
 
+  Scenario: Creating a GitHub PR with an update
+    Given a mocked git configuration
+    And a remote module repository
+    And a directory named "moduleroot"
+    And I set the environment variables to:
+      | variable     | value  |
+      | GITHUB_TOKEN | foobar |
+    When I run `msync update --noop --branch managed_update --pr`
+    Then the output should contain "Would submit PR "
+    And the exit status should be 0
+
+  Scenario: Creating a GitLab MR with an update
+    Given a mocked git configuration
+    And a remote module repository
+    And a directory named "moduleroot"
+    And I set the environment variables to:
+      | variable     | value  |
+      | GITLAB_TOKEN | foobar |
+    When I run `msync update --noop --branch managed_update --pr`
+    Then the output should contain "Would submit MR "
+    And the exit status should be 0
+
   Scenario: Repository with a default branch other than master
     Given a mocked git configuration
     And a remote module repository with "develop" as the default branch
@@ -903,3 +944,36 @@ Feature: update
     When I run `msync update -m "Update Gemfile"`
     Then the exit status should be 0
     Then the output should contain "Using repository's default branch: develop"
+
+  Scenario: Adding a new file from a template using meta data
+    And a file named "config_defaults.yml" with:
+      """
+      ---
+      """
+    Given a file named "managed_modules.yml" with:
+      """
+      ---
+        - puppet-test
+      """
+    And a file named "modulesync.yml" with:
+      """
+      ---
+        namespace: maestrodev
+        git_base: https://github.com/
+      """
+    And a directory named "moduleroot"
+    And a file named "moduleroot/test.erb" with:
+      """
+      module: <%= @metadata[:module_name] %>
+      target: <%= @metadata[:target_file] %>
+      workdir: <%= @metadata[:workdir] %>
+      """
+    When I run `msync update --noop`
+    Then the exit status should be 0
+    Given I run `cat modules/maestrodev/puppet-test/test`
+    Then the output should contain:
+      """
+      module: puppet-test
+      target: modules/maestrodev/puppet-test/test
+      workdir: modules/maestrodev/puppet-test
+      """

data/lib/modulesync.rb

@@ -1,5 +1,4 @@
 require 'fileutils'
-require 'octokit'
 require 'pathname'
 require 'modulesync/cli'
 require 'modulesync/constants'
@@ -10,13 +9,7 @@ require 'modulesync/settings'
 require 'modulesync/util'
 require 'monkey_patches'
 
-GITHUB_TOKEN = ENV.fetch('GITHUB_TOKEN', '')
-
-Octokit.configure do |c|
-  c.api_endpoint = ENV.fetch('GITHUB_BASE_URL', 'https://api.github.com')
-end
-
-module ModuleSync
+module ModuleSync # rubocop:disable Metrics/ModuleLength
   include Constants
 
   def self.config_defaults
@@ -29,31 +22,39 @@ module ModuleSync
   end
 
   def self.local_file(config_path, file)
-    "#{config_path}/#{MODULE_FILES_DIR}/#{file}"
+    File.join(config_path, MODULE_FILES_DIR, file)
   end
 
-  def self.module_file(project_root, puppet_module, file)
-    "#{project_root}/#{puppet_module}/#{file}"
+  def self.module_file(project_root, namespace, puppet_module, *parts)
+    File.join(project_root, namespace, puppet_module, *parts)
   end
 
-  def self.local_files(path)
-    if File.exist?(path)
-      # only select *.erb files, and strip the extension. This way all the code will only have to handle bare paths, except when reading the actual ERB text
-      local_files = Find.find(path).find_all { |p| p =~ /.erb$/ && !File.directory?(p) }.collect { |p| p.chomp('.erb') }.to_a
+  # List all template files.
+  #
+  # Only select *.erb files, and strip the extension. This way all the code will only have to handle bare paths,
+  # except when reading the actual ERB text
+  def self.find_template_files(local_template_dir)
+    if File.exist?(local_template_dir)
+      Find.find(local_template_dir).find_all { |p| p =~ /.erb$/ && !File.directory?(p) }
+          .collect { |p| p.chomp('.erb') }
+          .to_a
     else
-      puts "#{path} does not exist. Check that you are working in your module configs directory or that you have passed in the correct directory with -c."
+      $stdout.puts "#{local_template_dir} does not exist." \
+        ' Check that you are working in your module configs directory or' \
+        ' that you have passed in the correct directory with -c.'
       exit
     end
   end
 
-  def self.module_files(local_files, path)
-    local_files.map { |file| file.sub(/#{path}/, '') }
+  def self.relative_names(file_list, path)
+    file_list.map { |file| file.sub(/#{path}/, '') }
   end
 
   def self.managed_modules(config_file, filter, negative_filter)
     managed_modules = Util.parse_config(config_file)
     if managed_modules.empty?
-      puts "No modules found in #{config_file}. Check that you specified the right :configs directory and :managed_modules_conf file."
+      $stdout.puts "No modules found in #{config_file}." \
+        ' Check that you specified the right :configs directory and :managed_modules_conf file.'
       exit
     end
     managed_modules.select! { |m| m =~ Regexp.new(filter) } unless filter.nil?
@@ -78,100 +79,144 @@ module ModuleSync
   end
 
   def self.manage_file(filename, settings, options)
+    namespace = settings.additional_settings[:namespace]
     module_name = settings.additional_settings[:puppet_module]
     configs = settings.build_file_configs(filename)
+    target_file = module_file(options[:project_root], namespace, module_name, filename)
     if configs['delete']
-      Renderer.remove(module_file(options[:project_root], module_name, filename))
+      Renderer.remove(target_file)
     else
       templatename = local_file(options[:configs], filename)
       begin
         erb = Renderer.build(templatename)
-        template = Renderer.render(erb, configs)
-        Renderer.sync(template, module_file(options[:project_root], module_name, filename))
+        # Meta data passed to the template as @metadata[:name]
+        metadata = {
+          :module_name => module_name,
+          :workdir     => module_file(options[:project_root], namespace, module_name),
+          :target_file => target_file,
+        }
+        template = Renderer.render(erb, configs, metadata)
+        Renderer.sync(template, target_file)
       rescue # rubocop:disable Lint/RescueWithoutErrorClass
-        STDERR.puts "Error while rendering #{filename}"
+        $stderr.puts "Error while rendering #{filename}"
         raise
       end
     end
   end
 
   def self.manage_module(puppet_module, module_files, module_options, defaults, options)
-    if options[:pr] && !GITHUB_TOKEN
-      STDERR.puts 'Environment variable GITHUB_TOKEN must be set to use --pr!'
-      raise unless options[:skip_broken]
+    default_namespace = options[:namespace]
+    if module_options.is_a?(Hash) && module_options.key?(:namespace)
+      default_namespace = module_options[:namespace]
     end
-
-    puts "Syncing #{puppet_module}"
-    namespace, module_name = module_name(puppet_module, options[:namespace])
+    namespace, module_name = module_name(puppet_module, default_namespace)
+    git_repo = File.join(namespace, module_name)
     unless options[:offline]
-      git_base = options[:git_base]
-      git_uri = "#{git_base}#{namespace}"
-      Git.pull(git_uri, module_name, options[:branch], options[:project_root], module_options || {})
+      Git.pull(options[:git_base], git_repo, options[:branch], options[:project_root], module_options || {})
     end
-    module_configs = Util.parse_config("#{options[:project_root]}/#{module_name}/#{MODULE_CONF_FILE}")
+
+    module_configs = Util.parse_config(module_file(options[:project_root], namespace, module_name, MODULE_CONF_FILE))
     settings = Settings.new(defaults[GLOBAL_DEFAULTS_KEY] || {},
                             defaults,
                             module_configs[GLOBAL_DEFAULTS_KEY] || {},
                             module_configs,
                             :puppet_module => module_name,
-                            :git_base => git_base,
+                            :git_base => options[:git_base],
                             :namespace => namespace)
     settings.unmanaged_files(module_files).each do |filename|
-      puts "Not managing #{filename} in #{module_name}"
+      $stdout.puts "Not managing #{filename} in #{module_name}"
     end
 
     files_to_manage = settings.managed_files(module_files)
     files_to_manage.each { |filename| manage_file(filename, settings, options) }
 
     if options[:noop]
-      Git.update_noop(module_name, options)
+      Git.update_noop(git_repo, options)
+      options[:pr] && pr(module_options).manage(namespace, module_name, options)
     elsif !options[:offline]
-      # Git.update() returns a boolean: true if files were pushed, false if not.
-      pushed = Git.update(module_name, files_to_manage, options)
-      return nil unless pushed && options[:pr]
-
-      # We only do GitHub PR work if the GITHUB_TOKEN variable is set in the environment.
-      repo_path = "#{namespace}/#{module_name}"
-      puts "Submitting PR '#{options[:pr_title]}' on GitHub to #{repo_path} - merges #{options[:branch]} into master"
-      github = Octokit::Client.new(:access_token => GITHUB_TOKEN)
-      pr = github.create_pull_request(repo_path, 'master', options[:branch], options[:pr_title], options[:message])
-      puts "PR created at #{pr['html_url']}"
-
-      # PR labels can either be a list in the YAML file or they can pass in a comma
-      # separated list via the command line argument.
-      pr_labels = Util.parse_list(options[:pr_labels])
-
-      # We only assign labels to the PR if we've discovered a list > 1. The labels MUST
-      # already exist. We DO NOT create missing labels.
-      unless pr_labels.empty?
-        puts "Attaching the following labels to PR #{pr['number']}: #{pr_labels.join(', ')}"
-        github.add_labels_to_an_issue(repo_path, pr['number'], pr_labels)
-      end
+      pushed = Git.update(git_repo, files_to_manage, options)
+      pushed && options[:pr] && pr(module_options).manage(namespace, module_name, options)
     end
   end
 
+  def self.config_path(file, options)
+    return file if Pathname.new(file).absolute?
+    File.join(options[:configs], file)
+  end
+
+  def config_path(file, options)
+    self.class.config_path(file, options)
+  end
+
   def self.update(options)
     options = config_defaults.merge(options)
-    defaults = Util.parse_config("#{options[:configs]}/#{CONF_FILE}")
+    defaults = Util.parse_config(config_path(CONF_FILE, options))
+    if options[:pr]
+      unless options[:branch]
+        $stderr.puts 'A branch must be specified with --branch to use --pr!'
+        raise
+      end
 
-    path = "#{options[:configs]}/#{MODULE_FILES_DIR}"
-    local_files = self.local_files(path)
-    module_files = self.module_files(local_files, path)
+      @pr = create_pr_manager if options[:pr]
+    end
+
+    local_template_dir = config_path(MODULE_FILES_DIR, options)
+    local_files = find_template_files(local_template_dir)
+    module_files = relative_names(local_files, local_template_dir)
 
-    managed_modules = self.managed_modules("#{options[:configs]}/#{options[:managed_modules_conf]}", options[:filter], options[:negative_filter])
+    managed_modules = self.managed_modules(config_path(options[:managed_modules_conf], options),
+                                           options[:filter],
+                                           options[:negative_filter])
 
     errors = false
     # managed_modules is either an array or a hash
     managed_modules.each do |puppet_module, module_options|
       begin
-        manage_module(puppet_module, module_files, module_options, defaults, options)
+        mod_options = module_options.nil? ? nil : Util.symbolize_keys(module_options)
+        manage_module(puppet_module, module_files, mod_options, defaults, options)
       rescue # rubocop:disable Lint/RescueWithoutErrorClass
-        STDERR.puts "Error while updating #{puppet_module}"
+        $stderr.puts "Error while updating #{puppet_module}"
         raise unless options[:skip_broken]
         errors = true
-        puts "Skipping #{puppet_module} as update process failed"
+        $stdout.puts "Skipping #{puppet_module} as update process failed"
       end
     end
     exit 1 if errors && options[:fail_on_warnings]
   end
+
+  def self.pr(module_options)
+    module_options ||= {}
+    github_conf = module_options[:github]
+    gitlab_conf = module_options[:gitlab]
+
+    if !github_conf.nil?
+      base_url = github_conf[:base_url] || ENV.fetch('GITHUB_BASE_URL', 'https://api.github.com')
+      require 'modulesync/pr/github'
+      ModuleSync::PR::GitHub.new(github_conf[:token], base_url)
+    elsif !gitlab_conf.nil?
+      base_url = gitlab_conf[:base_url] || ENV.fetch('GITLAB_BASE_URL', 'https://gitlab.com/api/v4')
+      require 'modulesync/pr/gitlab'
+      ModuleSync::PR::GitLab.new(gitlab_conf[:token], base_url)
+    elsif @pr.nil?
+      $stderr.puts 'No GitHub or GitLab token specified for --pr!'
+      raise
+    else
+      @pr
+    end
+  end
+
+  def self.create_pr_manager
+    github_token = ENV.fetch('GITHUB_TOKEN', '')
+    gitlab_token = ENV.fetch('GITLAB_TOKEN', '')
+
+    if !github_token.empty?
+      require 'modulesync/pr/github'
+      ModuleSync::PR::GitHub.new(github_token, ENV.fetch('GITHUB_BASE_URL', 'https://api.github.com'))
+    elsif !gitlab_token.empty?
+      require 'modulesync/pr/gitlab'
+      ModuleSync::PR::GitLab.new(gitlab_token, ENV.fetch('GITLAB_BASE_URL', 'https://gitlab.com/api/v4'))
+    else
+      warn '--pr specified without environment variables GITHUB_TOKEN or GITLAB_TOKEN'
+    end
+  end
 end