modulesync 2.2.0 → 2.3.0

data/lib/modulesync/cli.rb

@@ -7,31 +7,34 @@ require 'modulesync/util'
 
 module ModuleSync
   module CLI
+    def self.prepare_options(cli_options, **more_options)
+      options = CLI.defaults
+      options.merge! Util.symbolize_keys(cli_options)
+      options.merge! more_options
+
+      Util.symbolize_keys options
+    end
+
     def self.defaults
       @defaults ||= Util.symbolize_keys(Util.parse_config(Constants::MODULESYNC_CONF_FILE))
     end
 
     class Hook < Thor
-      class_option :project_root,
-                   :aliases => '-c',
-                   :desc => 'Path used by git to clone modules into. Defaults to "modules"',
-                   :default => CLI.defaults[:project_root] || 'modules'
-      class_option :hook_args,
-                   :aliases => '-a',
-                   :desc => 'Arguments to pass to msync in the git hook'
-
+      option :hook_args,
+             :aliases => '-a',
+             :desc => 'Arguments to pass to msync in the git hook'
+      option :branch,
+             :aliases => '-b',
+             :desc => 'Branch name to pass to msync in the git hook',
+             :default => CLI.defaults[:branch]
       desc 'activate', 'Activate the git hook.'
       def activate
-        config = { :command => 'hook' }.merge(options)
-        config[:hook] = 'activate'
-        ModuleSync.hook(config)
+        ModuleSync.hook CLI.prepare_options(options, hook: 'activate')
       end
 
       desc 'deactivate', 'Deactivate the git hook.'
       def deactivate
-        config = { :command => 'hook' }.merge(options)
-        config[:hook] = 'deactivate'
-        ModuleSync.hook(config)
+        ModuleSync.hook CLI.prepare_options(options, hook: 'deactivate')
       end
     end
 
@@ -39,7 +42,7 @@ module ModuleSync
       class_option :project_root,
                    :aliases => '-c',
                    :desc => 'Path used by git to clone modules into.',
-                   :default => CLI.defaults[:project_root] || 'modules'
+                   :default => CLI.defaults[:project_root]
       class_option :git_base,
                    :desc => 'Specify the base part of a git URL to pull from',
                    :default => CLI.defaults[:git_base] || 'git@github.com:'
@@ -53,11 +56,11 @@ module ModuleSync
       class_option :negative_filter,
                    :aliases => '-x',
                    :desc => 'A regular expression to skip repositories.'
-      class_option :branch,
-                   :aliases => '-b',
-                   :desc => 'Branch name to make the changes in.' \
-                              ' Defaults to the default branch of the upstream repository, but falls back to "master".',
-                   :default => CLI.defaults[:branch]
+      class_option :verbose,
+                   :aliases => '-v',
+                   :desc => 'Verbose mode',
+                   :type => :boolean,
+                   :default => false
 
       desc 'update', 'Update the modules in managed_modules.yml'
       option :message,
@@ -67,7 +70,7 @@ module ModuleSync
       option :configs,
              :aliases => '-c',
              :desc => 'The local directory or remote repository to define the list of managed modules,' \
-                        ' the file templates, and the default values for template variables.'
+                      ' the file templates, and the default values for template variables.'
       option :managed_modules_conf,
              :desc => 'The file name to define the list of managed modules'
       option :remote_branch,
@@ -104,7 +107,7 @@ module ModuleSync
              :default => CLI.defaults[:pr_labels] || []
       option :pr_target_branch,
              :desc => 'Target branch for the pull/merge request',
-             :default => CLI.defaults[:pr_target_branch] || 'master'
+             :default => CLI.defaults[:pr_target_branch]
       option :offline,
              :type => :boolean,
              :desc => 'Do not run any Git commands. Allows the user to manage Git outside of ModuleSync.',
@@ -130,17 +133,108 @@ module ModuleSync
              :type => :boolean,
              :aliases => '-F',
              :desc => 'Produce a failure exit code when there are warnings' \
-                        ' (only has effect when --skip_broken is enabled)',
+                      ' (only has effect when --skip_broken is enabled)',
              :default => false
-
+      option :branch,
+             :aliases => '-b',
+             :desc => 'Branch name to make the changes in.' \
+                      ' Defaults to the default branch of the upstream repository, but falls back to "master".',
+             :default => CLI.defaults[:branch]
       def update
-        config = { :command => 'update' }.merge(options)
-        config = Util.symbolize_keys(config)
+        config = CLI.prepare_options(options)
         raise Thor::Error, 'No value provided for required option "--message"' unless config[:noop] \
                                                                                       || config[:message] \
                                                                                       || config[:offline]
-        config[:git_opts] = { 'amend' => config[:amend], 'force' => config[:force] }
-        ModuleSync.update(config)
+
+        ModuleSync.update config
+      end
+
+      desc 'execute [OPTIONS] -- COMMAND..', 'Execute the command in each managed modules'
+      long_desc <<~DESC
+        Execute the command in each managed modules.
+
+        COMMAND can be an absolute or a relative path.
+
+        To ease running local commands, a relative path is expanded with the current user directory but only if the target file exists.
+
+        Example: `msync exec custom-scripts/true` will run "$PWD/custom-scripts/true" in each repository.
+
+        As side effect, you can shadow system binary if a local file is present:
+        \x5  `msync exec true` will run "$PWD/true", not `/bin/true` if "$PWD/true" exists.
+      DESC
+
+      option :configs,
+             :aliases => '-c',
+             :desc => 'The local directory or remote repository to define the list of managed modules,' \
+                      ' the file templates, and the default values for template variables.'
+      option :managed_modules_conf,
+             :desc => 'The file name to define the list of managed modules'
+      option :branch,
+             :aliases => '-b',
+             :desc => 'Branch name to make the changes in.',
+             :default => CLI.defaults[:branch]
+      option :fail_fast,
+             :type => :boolean,
+             :desc => 'Abort the run after a command execution failure',
+             :default => CLI.defaults[:fail_fast].nil? ? true : CLI.defaults[:fail_fast]
+      def execute(*command_args)
+        raise Thor::Error, 'COMMAND is a required argument' if command_args.empty?
+
+        ModuleSync.execute CLI.prepare_options(options, command_args: command_args)
+      end
+
+      desc 'reset', 'Reset local repositories to a well-known state'
+      long_desc <<~DESC
+        Reset local repository to a well-known state:
+        \x5  * Switch local repositories to specified branch
+        \x5  * Fetch and prune repositories unless running with `--offline` option
+        \x5  * Hard-reset any changes to specified source branch, technically any git refs, e.g. `main`, `origin/wip`
+        \x5  * Clean all extra local files
+
+        Note: If a repository is not already cloned, it will operate the following to reach to well-known state:
+        \x5  * Clone the repository
+        \x5  * Switch to specified branch
+      DESC
+      option :configs,
+             :aliases => '-c',
+             :desc => 'The local directory or remote repository to define the list of managed modules,' \
+                      ' the file templates, and the default values for template variables.'
+      option :managed_modules_conf,
+             :desc => 'The file name to define the list of managed modules'
+      option :branch,
+             :aliases => '-b',
+             :desc => 'Branch name to make the changes in.',
+             :default => CLI.defaults[:branch]
+      option :offline,
+             :type => :boolean,
+             :desc => 'Only proceed local operations',
+             :default => false
+      option :source_branch,
+             :desc => 'Branch to reset from (e.g. origin/wip)'
+      def reset
+        ModuleSync.reset CLI.prepare_options(options)
+      end
+
+      desc 'push', 'Push all available commits from branch to remote'
+      option :configs,
+             :aliases => '-c',
+             :desc => 'The local directory or remote repository to define the list of managed modules,' \
+                      ' the file templates, and the default values for template variables.'
+      option :managed_modules_conf,
+             :desc => 'The file name to define the list of managed modules'
+      option :branch,
+             :aliases => '-b',
+             :desc => 'Branch name to push',
+             :default => CLI.defaults[:branch]
+      option :remote_branch,
+             :desc => 'Remote branch to push to (e.g. maintenance)'
+      def push
+        ModuleSync.push CLI.prepare_options(options)
+      end
+
+      desc 'clone', 'Clone repositories that need to'
+      def clone
+        ModuleSync.clone CLI.prepare_options(options)
       end
 
       desc 'hook', 'Activate or deactivate a git hook.'

data/lib/modulesync/git_service/base.rb

@@ -0,0 +1,63 @@
+module ModuleSync
+  module GitService
+    # Generic class for git services
+    class Base
+      def open_pull_request(repo_path:, namespace:, title:, message:, source_branch:, target_branch:, labels:, noop:) # rubocop:disable Metrics/ParameterLists
+        unless source_branch != target_branch
+          raise ModuleSync::Error,
+                "Unable to open a pull request with the same source and target branch: '#{source_branch}'"
+        end
+
+        _open_pull_request(
+          repo_path: repo_path,
+          namespace: namespace,
+          title: title,
+          message: message,
+          source_branch: source_branch,
+          target_branch: target_branch,
+          labels: labels,
+          noop: noop,
+        )
+      end
+
+      # This method attempts to guess the git service endpoint based on remote
+      def self.guess_endpoint_from(remote:)
+        hostname = extract_hostname(remote)
+        return nil if hostname.nil?
+
+        "https://#{hostname}"
+      end
+
+      # This method extracts hostname from URL like:
+      #
+      # - ssh://[user@]host.xz[:port]/path/to/repo.git/
+      # - git://host.xz[:port]/path/to/repo.git/
+      # - [user@]host.xz:path/to/repo.git/
+      # - http[s]://host.xz[:port]/path/to/repo.git/
+      # - ftp[s]://host.xz[:port]/path/to/repo.git/
+      #
+      # Returns nil if
+      # - /path/to/repo.git/
+      # - file:///path/to/repo.git/
+      # - any invalid URL
+      def self.extract_hostname(url)
+        return nil if url.start_with?('/') || url.start_with?('file://') # local path (e.g. file:///path/to/repo)
+
+        unless url.start_with?(%r{[a-z]+://}) # SSH notation does not contain protocol (e.g. user@server:path/to/repo/)
+          pattern = /^(?<user>.*@)?(?<hostname>[\w|.]*):(?<repo>.*)$/ # SSH path (e.g. user@server:repo)
+          return url.match(pattern)[:hostname] if url.match?(pattern)
+        end
+
+        URI.parse(url).host
+      rescue URI::InvalidURIError
+        nil
+      end
+
+      protected
+
+      def _open_pull_request(repo_path:, namespace:, title:, message:, source_branch:, target_branch:, labels:, noop:) # rubocop:disable Metrics/ParameterLists
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/modulesync/git_service/factory.rb

@@ -0,0 +1,28 @@
+module ModuleSync
+  module GitService
+    # Git service's factory
+    module Factory
+      def self.instantiate(type:, endpoint:, token:)
+        raise MissingCredentialsError, <<~MESSAGE if token.nil?
+          A token is required to use services from #{type}:
+            Please set environment variable: "#{type.upcase}_TOKEN" or set the token entry in module options.
+        MESSAGE
+
+        klass(type: type).new token, endpoint
+      end
+
+      def self.klass(type:)
+        case type
+        when :github
+          require 'modulesync/git_service/github'
+          ModuleSync::GitService::GitHub
+        when :gitlab
+          require 'modulesync/git_service/gitlab'
+          ModuleSync::GitService::GitLab
+        else
+          raise NotImplementedError, "Unknown git service: '#{type}'"
+        end
+      end
+    end
+  end
+end

data/lib/modulesync/{pr → git_service}/github.rb

@@ -1,28 +1,30 @@
+require 'modulesync/git_service'
+require 'modulesync/git_service/base'
 require 'octokit'
-require 'modulesync/util'
 
 module ModuleSync
-  module PR
+  module GitService
     # GitHub creates and manages pull requests on github.com or GitHub
     # Enterprise installations.
-    class GitHub
+    class GitHub < Base
       def initialize(token, endpoint)
+        super()
+
         Octokit.configure do |c|
           c.api_endpoint = endpoint
         end
         @api = Octokit::Client.new(:access_token => token)
       end
 
-      def manage(namespace, module_name, options)
-        repo_path = File.join(namespace, module_name)
-        branch = options[:remote_branch] || options[:branch]
-        head = "#{namespace}:#{branch}"
-        target_branch = options[:pr_target_branch] || 'master'
+      private
+
+      def _open_pull_request(repo_path:, namespace:, title:, message:, source_branch:, target_branch:, labels:, noop:) # rubocop:disable Metrics/ParameterLists
+        head = "#{namespace}:#{source_branch}"
 
-        if options[:noop]
+        if noop
           $stdout.puts \
-            "Using no-op. Would submit PR '#{options[:pr_title]}' to #{repo_path} " \
-            "- merges #{branch} into #{target_branch}"
+            "Using no-op. Would submit PR '#{title}' to '#{repo_path}' " \
+            "- merges '#{source_branch}' into '#{target_branch}'"
           return
         end
 
@@ -32,25 +34,25 @@ module ModuleSync
                                            :head => head)
         unless pull_requests.empty?
           # Skip creating the PR if it exists already.
-          $stdout.puts "Skipped! #{pull_requests.length} PRs found for branch #{branch}"
+          $stdout.puts "Skipped! #{pull_requests.length} PRs found for branch '#{source_branch}'"
           return
         end
 
-        pr_labels = ModuleSync::Util.parse_list(options[:pr_labels])
         pr = @api.create_pull_request(repo_path,
                                       target_branch,
-                                      branch,
-                                      options[:pr_title],
-                                      options[:message])
+                                      source_branch,
+                                      title,
+                                      message)
         $stdout.puts \
-          "Submitted PR '#{options[:pr_title]}' to #{repo_path} " \
-          "- merges #{branch} into #{target_branch}"
+          "Submitted PR '#{title}' to '#{repo_path}' " \
+          "- merges #{source_branch} into #{target_branch}"
 
         # We only assign labels to the PR if we've discovered a list > 1. The labels MUST
         # already exist. We DO NOT create missing labels.
-        return if pr_labels.empty?
-        $stdout.puts "Attaching the following labels to PR #{pr['number']}: #{pr_labels.join(', ')}"
-        @api.add_labels_to_an_issue(repo_path, pr['number'], pr_labels)
+        return if labels.empty?
+
+        $stdout.puts "Attaching the following labels to PR #{pr['number']}: #{labels.join(', ')}"
+        @api.add_labels_to_an_issue(repo_path, pr['number'], labels)
       end
     end
   end

data/lib/modulesync/git_service/gitlab.rb

@@ -0,0 +1,62 @@
+require 'gitlab'
+require 'modulesync/git_service'
+require 'modulesync/git_service/base'
+
+module ModuleSync
+  module GitService
+    # GitLab creates and manages merge requests on gitlab.com or private GitLab
+    # installations.
+    class GitLab < Base
+      def initialize(token, endpoint)
+        super()
+
+        @api = Gitlab::Client.new(
+          :endpoint => endpoint,
+          :private_token => token,
+        )
+      end
+
+      def self.guess_endpoint_from(remote:)
+        endpoint = super
+        return nil if endpoint.nil?
+
+        endpoint += '/api/v4'
+        endpoint
+      end
+
+      private
+
+      def _open_pull_request(repo_path:, namespace:, title:, message:, source_branch:, target_branch:, labels:, noop:) # rubocop:disable Metrics/ParameterLists, Lint/UnusedMethodArgument
+        if noop
+          $stdout.puts \
+            "Using no-op. Would submit MR '#{title}' to '#{repo_path}' " \
+            "- merges #{source_branch} into #{target_branch}"
+          return
+        end
+
+        merge_requests = @api.merge_requests(repo_path,
+                                             :state => 'opened',
+                                             :source_branch => source_branch,
+                                             :target_branch => target_branch)
+        unless merge_requests.empty?
+          # Skip creating the MR if it exists already.
+          $stdout.puts "Skipped! #{merge_requests.length} MRs found for branch '#{source_branch}'"
+          return
+        end
+
+        mr = @api.create_merge_request(repo_path,
+                                       title,
+                                       :source_branch => source_branch,
+                                       :target_branch => target_branch,
+                                       :labels => labels)
+        $stdout.puts \
+          "Submitted MR '#{title}' to '#{repo_path}' " \
+          "- merges '#{source_branch}' into '#{target_branch}'"
+
+        return if labels.empty?
+
+        $stdout.puts "Attached the following labels to MR #{mr.iid}: #{labels.join(', ')}"
+      end
+    end
+  end
+end

data/lib/modulesync/git_service.rb

@@ -0,0 +1,96 @@
+module ModuleSync
+  class Error < StandardError; end
+
+  # Namespace for Git service classes (ie. GitHub, GitLab)
+  module GitService
+    class MissingCredentialsError < Error; end
+
+    class UnguessableTypeError < Error; end
+
+    def self.configuration_for(sourcecode:)
+      type = type_for(sourcecode: sourcecode)
+
+      {
+        type: type,
+        endpoint: endpoint_for(sourcecode: sourcecode, type: type),
+        token: token_for(sourcecode: sourcecode, type: type),
+      }
+    end
+
+    # This method attempts to guess git service's type (ie. gitlab or github)
+    # It process in this order
+    #   1. use module specific configuration entry (ie. a specific entry named `gitlab` or `github`)
+    #   2. guess using remote url (ie. looking for `github` or `gitlab` string)
+    #   3. use environment variables (ie. check if GITHUB_TOKEN or GITLAB_TOKEN is set)
+    #   4. fail
+    def self.type_for(sourcecode:)
+      return :github unless sourcecode.options[:github].nil?
+      return :gitlab unless sourcecode.options[:gitlab].nil?
+      return :github if sourcecode.repository_remote.include? 'github'
+      return :gitlab if sourcecode.repository_remote.include? 'gitlab'
+
+      if ENV['GITLAB_TOKEN'].nil? && ENV['GITHUB_TOKEN'].nil?
+        raise UnguessableTypeError, <<~MESSAGE
+          Unable to guess Git service type without GITLAB_TOKEN or GITHUB_TOKEN sets.
+        MESSAGE
+      end
+
+      unless ENV['GITLAB_TOKEN'].nil? || ENV['GITHUB_TOKEN'].nil?
+        raise UnguessableTypeError, <<~MESSAGE
+          Unable to guess Git service type with both GITLAB_TOKEN and GITHUB_TOKEN sets.
+
+          Please set the wanted one in configuration (ie. add `gitlab:` or `github:` key)
+        MESSAGE
+      end
+
+      return :github unless ENV['GITHUB_TOKEN'].nil?
+      return :gitlab unless ENV['GITLAB_TOKEN'].nil?
+
+      raise NotImplementedError
+    end
+
+    # This method attempts to find git service's endpoint based on sourcecode and type
+    # It process in this order
+    #   1. use module specific configuration (ie. `base_url`)
+    #   2. use environment variable dependending on type (e.g. GITLAB_BASE_URL)
+    #   3. guess using the git remote url
+    #   4. fail
+    def self.endpoint_for(sourcecode:, type:)
+      endpoint = sourcecode.options.dig(type, :base_url)
+
+      endpoint ||= case type
+                   when :github
+                     ENV['GITHUB_BASE_URL']
+                   when :gitlab
+                     ENV['GITLAB_BASE_URL']
+                   end
+
+      endpoint ||= GitService::Factory.klass(type: type).guess_endpoint_from(remote: sourcecode.repository_remote)
+
+      raise NotImplementedError, <<~MESSAGE if endpoint.nil?
+        Unable to guess endpoint for remote: '#{sourcecode.repository_remote}'
+        Please provide `base_url` option in configuration file
+      MESSAGE
+
+      endpoint
+    end
+
+    # This method attempts to find the token associated to provided sourcecode and type
+    # It process in this order:
+    #   1. use module specific configuration (ie. `token`)
+    #   2. use environment variable depending on type (e.g. GITLAB_TOKEN)
+    #   3. fail
+    def self.token_for(sourcecode:, type:)
+      token = sourcecode.options.dig(type, :token)
+
+      token ||= case type
+                when :github
+                  ENV['GITHUB_TOKEN']
+                when :gitlab
+                  ENV['GITLAB_TOKEN']
+                end
+
+      token
+    end
+  end
+end

data/lib/modulesync/hook.rb

@@ -6,20 +6,20 @@ module ModuleSync
 
     def initialize(hook_file, options = [])
       @hook_file = hook_file
-      @namespace = options['namespace']
-      @branch = options['branch']
-      @args = options['hook_args']
+      @namespace = options[:namespace]
+      @branch = options[:branch]
+      @args = options[:hook_args]
     end
 
     def content(arguments)
-      <<-CONTENT
-#!/usr/bin/env bash
+      <<~CONTENT
+        #!/usr/bin/env bash
 
-current_branch=\`git symbolic-ref HEAD | sed -e 's,.*/\(.*\),\1,'\`
-git_dir=\`git rev-parse --show-toplevel\`
-message=\`git log -1 --format=%B\`
-msync -m "\$message" #{arguments}
-CONTENT
+        current_branch=\`git symbolic-ref HEAD | sed -e 's,.*/\(.*\),\1,'\`
+        git_dir=\`git rev-parse --show-toplevel\`
+        message=\`git log -1 --format=%B\`
+        msync -m "\$message" #{arguments}
+      CONTENT
     end
 
     def activate
@@ -28,9 +28,7 @@ CONTENT
       hook_args << "-b #{branch}" if branch
       hook_args << args if args
 
-      File.open(hook_file, 'w') do |file|
-        file.write(content(hook_args.join(' ')))
-      end
+      File.write(hook_file, content(hook_args.join(' ')))
     end
 
     def deactivate

data/lib/modulesync/renderer.rb

@@ -12,7 +12,7 @@ module ModuleSync
 
     def self.build(target_name)
       template_file = if !File.exist?("#{target_name}.erb") && File.exist?(target_name)
-                        STDERR.puts "Warning: using '#{target_name}' as template without '.erb' suffix"
+                        $stderr.puts "Warning: using '#{target_name}' as template without '.erb' suffix"
                         target_name
                       else
                         "#{target_name}.erb"
@@ -32,11 +32,8 @@ module ModuleSync
     end
 
     def self.sync(template, target_name)
-      path = target_name.rpartition('/').first
-      FileUtils.mkdir_p(path) unless path.empty?
-      File.open(target_name, 'w') do |file|
-        file.write(template)
-      end
+      FileUtils.mkdir_p(File.dirname(target_name))
+      File.write(target_name, template)
     end
   end
 end