multi_repo 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7e5efacc85571dad2a80c845667ec2fa40e253fd13547da6df08725d1ee18daa
+  data.tar.gz: 9764b15530f6e62de724fa57152b6a6b3aa1af5c7bc2620a30c52de8b2314692
+SHA512:
+  metadata.gz: 45b956c9f7d52529f5c278eca077c0ce3acdbd573d80a73eac286832b8d7f99adc1efc10cee545a32decfdddb679585fac97529217104b5bb2069ee55e31dd01
+  data.tar.gz: 66f89c406e312e39aac877141be238633535b35f2dcdf22802539c5de2433f70d46fa5e0c00ce596dd235248800a571a2a608a9b66fc03d8cb520e3232c20227

data/.codeclimate.yml

@@ -0,0 +1,16 @@
+prepare:
+  fetch:
+  - url: https://raw.githubusercontent.com/ManageIQ/manageiq-style/master/.rubocop_base.yml
+    path: ".rubocop_base.yml"
+  - url: https://raw.githubusercontent.com/ManageIQ/manageiq-style/master/.rubocop_cc_base.yml
+    path: ".rubocop_cc_base.yml"
+  - url: https://raw.githubusercontent.com/ManageIQ/manageiq-style/master/styles/base.yml
+    path: styles/base.yml
+  - url: https://raw.githubusercontent.com/ManageIQ/manageiq-style/master/styles/cc_base.yml
+    path: styles/cc_base.yml
+plugins:
+  rubocop:
+    enabled: true
+    config: ".rubocop_cc.yml"
+    channel: rubocop-0-82
+version: '2'

data/.github/workflows/ci.yaml

@@ -0,0 +1,32 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+  schedule:
+  - cron: '0 0 * * 0'
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version:
+        - '2.7'
+        - '3.0'
+    env:
+      CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true
+      timeout-minutes: 30
+    - name: Run tests
+      run: bundle exec rake
+    - name: Report code coverage
+      if: ${{ github.ref == 'refs/heads/master' && matrix.ruby-version == '3.0' }}
+      continue-on-error: true
+      uses: paambaati/codeclimate-action@v3.0.0

data/.gitignore

@@ -0,0 +1,6 @@
+/.bundle/
+/config/
+/mirrors/
+/repos/
+/spec/root/
+Gemfile.lock

data/.rspec

@@ -0,0 +1,2 @@
+--require spec_helper
+--order random

data/.rubocop.yml

@@ -0,0 +1,4 @@
+inherit_gem:
+  manageiq-style: ".rubocop_base.yml"
+inherit_from:
+- ".rubocop_local.yml"

data/.rubocop_cc.yml

@@ -0,0 +1,4 @@
+inherit_from:
+- ".rubocop_base.yml"
+- ".rubocop_cc_base.yml"
+- ".rubocop_local.yml"

data/.whitesource

@@ -0,0 +1,3 @@
+{
+  "settingsInheritedFrom": "ManageIQ/whitesource-config@master"
+}

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+plugin 'bundler-inject'
+require File.join(Bundler::Plugin.index.load_paths("bundler-inject")[0], "bundler-inject") rescue nil
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 ManageIQ Authors.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,90 @@
+# MultiRepo
+
+MultiRepo is a tool for managing multiple git repositories.
+
+[![Gem Version](https://badge.fury.io/rb/multi_repo.svg)](http://badge.fury.io/rb/multi_repo)
+[![CI](https://github.com/ManageIQ/multi_repo/actions/workflows/ci.yaml/badge.svg)](https://github.com/ManageIQ/multi_repo/actions/workflows/ci.yaml)
+[![Code Climate](http://img.shields.io/codeclimate/github/ManageIQ/multi_repo.svg)](https://codeclimate.com/github/ManageIQ/multi_repo)
+
+## Installation
+
+```sh
+gem install multi_repo
+```
+
+## Configuration
+
+## Usage
+
+Typical usage will be from single scripts. In order to keep each script manageable, it can be preferable to use bundler/inline to define the gems needed by that script. To do this, add the following to the top of the script:
+
+```ruby
+#/usr/bin/env ruby
+
+require "bundler/inline"
+gemfile do
+  source "https://rubygems.org"
+  gem "multi_repo", require: "multi_repo/cli"
+end
+```
+
+Then, you would set up options for your script. A `MultiRepo::CLI` helper is provided to make this easier. It has the [optimist](https://github.com/ManageIQ/optimist) gem already prepared and comes with a `.common_options` helper method to set up options for `--repo-set`, `--repo`, and `--dry-run`. For example,
+
+```ruby
+opts = Optimist.options do
+  opt :some_opt, "An option your script needs", :type => :string, :required => true
+
+  MultiRepo::CLI.common_options(self)
+end
+```
+
+would produce the following help output:
+
+```
+Options:
+  -o, --some-opt=<s>    An option your script needs
+
+Common Options:
+  -s, --repo-set=<s>    The repo set to work with (default: master)
+  -r, --repo=<s+>       Individual repo(s) to work with; Overrides --repo-set
+  -d, --dry-run         Execute without making changes
+  -h, --help            Show this message
+```
+
+After you have set up the options, you can write your script. `MultiRepo::Service` classes are provided to help interface with common third-party services, such as GitHub. `MultiRepo::Helper` classes are provided to do relatively common operations, such as renaming labels. The `MultiRepo::CLI` class also has helpers for looping over the repo set.  For example, to loop over each repo in the repo set and show the file contents one can do:
+
+```ruby
+MultiRepo::CLI.each_repo(**opts) do |repo|
+  system("ls")
+end
+```
+
+## GitHub interactions
+
+Certain commands interact with GitHub and expect a GitHub API Token set in the
+ENV variable GITHUB_API_TOKEN.
+
+If you don't already have a token, or want to create one specific to these
+purposes
+- Go to https://github.com/settings/tokens
+- Choose "Generate New Token"
+- Give the token a description
+- At a mimimum, choose "repo" for the permissions.
+- Click "Generate Token"
+- Copy the token given to you, and keep it in a safe location, as once you leave
+  the page, the token is no longer accessible
+
+Then, in order to use it, export the ENV variable permanently, or pass it to the
+program as part of the call.
+
+```sh
+GITHUB_API_TOKEN=<token> bin/update_labels
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/ManageIQ/multi_repo.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new('spec')
+task :test => :spec
+task :default => :spec

data/bin/console

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH << File.expand_path("../lib", __dir__)
+
+require "bundler/setup"
+require "multi_repo/cli"
+require "irb"
+IRB.start

data/exe/multi_repo

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." &>/dev/null && pwd)/scripts"
+
+usage() {
+  echo "Usage: multi_repo <script> [args]"
+  echo "  script      Script to run"
+  echo "  args        Arguments to pass to the script"
+  echo "  -h, --help  Show this help message"
+  echo
+  echo "Available scripts:"
+  for f in $(ls -1 "$SCRIPT_DIR" | sort); do
+    echo "  $f"
+  done
+}
+
+if [ -z "$1" -o "$1" = "--help" -o "$1" = "-h" ]; then
+  usage
+  exit
+fi
+
+if [ ! -f "$SCRIPT_DIR/$1" ]; then
+  echo "ERROR: script '$1' not found"
+  echo
+  usage
+  exit 1
+fi
+
+exec "$SCRIPT_DIR/$1" "${@:2}"

data/lib/multi_repo/cli.rb

@@ -0,0 +1,92 @@
+require "multi_repo"
+require "optimist"
+require "colorize"
+
+module MultiRepo
+  module CLI
+    def self.each_repo(**kwargs)
+      raise "no block given" unless block_given?
+
+      repos_for(**kwargs).each do |repo|
+        puts header(repo.name)
+        yield repo
+        puts
+      end
+    end
+
+    def self.repos_for(repo: nil, repo_set: nil, dry_run: false, **_)
+      Optimist.die("options --repo or --repo_set must be specified") unless repo || repo_set
+
+      if repo_set
+        repos = MultiRepo::RepoSet[repo_set]&.deep_dup
+        Optimist.die(:repo_set, "#{repo_set.inspect} was not found in the config") if repos.nil?
+
+        if repo
+          repo_names = Set.new(Array(repo))
+          repos.select! { |r| repo_names.include?(r.name) }
+        end
+
+        repos.each { |r| r.dry_run = dry_run }
+
+        repos
+      else
+        Array(repo).map { |n| MultiRepo::Repo.new(n, dry_run: dry_run) }
+      end
+    end
+
+    def self.repo_for(repo_name, repo_set: nil, dry_run: false)
+      Optimist.die(:repo, "must be specified") if repo_name.nil?
+
+      repos_for(repo: repo_name, repo_set: repo_set, dry_run: dry_run).first
+    end
+
+    def self.common_options(optimist, only: %i[repo repo_set dry_run], except: nil, repo_set_default: "master")
+      optimist.banner("")
+      optimist.banner("Common Options:")
+
+      subset = Array(only).map(&:to_sym) - Array(except).map(&:to_sym)
+
+      if subset.include?(:repo_set)
+        optimist.opt :repo_set, "The repo set to work with", :type => :string, :default => repo_set_default, :short => "s"
+      end
+      if subset.include?(:repo)
+        msg = "Individual repo(s) to work with"
+        if subset.include?(:repo_set)
+          sub_opts = {}
+          msg << "; Overrides --repo-set"
+        else
+          sub_opts = {:required => true}
+        end
+        optimist.opt :repo, msg, sub_opts.merge(:type => :strings)
+      end
+      if subset.include?(:dry_run)
+        optimist.opt :dry_run, "Execute without making changes", :default => false
+      end
+    end
+
+    #
+    # Logging helpers
+    #
+
+    HEADER_SIZE = 80
+
+    def self.header(title, char = "=")
+      title = " #{title} "
+      start = (HEADER_SIZE / 2) - (title.length / 2)
+      separator(char).tap { |h| h[start, title.length] = title }
+    end
+
+    def self.separator(char = "*")
+      char * HEADER_SIZE
+    end
+
+    def self.progress_bar(total = 100)
+      require "progressbar"
+      ProgressBar.create(
+        :format => "%j%% |%B| %E",
+        :length => HEADER_SIZE,
+        :total  => total
+      )
+    end
+  end
+end

data/lib/multi_repo/helpers/git_mirror.rb

@@ -0,0 +1,198 @@
+module MultiRepo::Helpers
+  class GitMirror
+    def initialize
+      require "colorize"
+      require "config"
+
+      @errors_occurred = false
+    end
+
+    def settings
+      @settings ||= Config.load_files(MultiRepo.config_dir.join("settings.yml").to_s, MultiRepo.config_dir.join("settings.local.yml").to_s)
+    end
+
+    def mirror_all
+      settings.git_mirror.repos_to_mirror.keys.each { |repo| mirror(repo) }
+      !@errors_occurred
+    end
+
+    def mirror(repo)
+      repo = repo.to_s
+      options = default_repo_options.dup.merge!(settings.git_mirror.repos_to_mirror[repo].to_h)
+      with_repo(repo, options) do
+        send("mirror_#{options.remote_source}_repo", repo)
+      end
+      !@errors_occurred
+    end
+
+    private
+
+    def default_repo_options
+      Config::Options.new(:remote_source => :upstream)
+    end
+
+    def backup_remote_defined?
+      !!settings.git_mirror.remotes.backup
+    end
+
+    def mirror_branches_for(repo)
+      settings.git_mirror.branch_mirror_defaults.to_h.merge(settings.git_mirror.branch_mirror_overrides[repo].to_h || {}).each_with_object({}) { |(k, v), h| h[k.to_s] = v }
+    end
+
+    def mirror_branches(repo, source_remote, dest_remote)
+      mirror_branches_for(repo).each do |source_name, dest_name|
+        sync_branch(source_remote, source_name, dest_remote, dest_name)
+      end
+    end
+
+    def mirror_upstream_repo(repo)
+      mirror_remote_refs(repo, "upstream", "downstream")
+      mirror_branches(repo, "upstream", "downstream")
+      mirror_remote_refs(repo, "downstream", "backup") if backup_remote_defined?
+    end
+
+    def mirror_downstream_repo(repo)
+      mirror_branches(repo, "downstream", "downstream")
+      mirror_remote_refs(repo, "downstream", "backup") if backup_remote_defined?
+    end
+
+    def dry_run?
+      return @dry_run if defined?(@dry_run)
+      @dry_run = ARGV.include?("--dry-run")
+    end
+
+    def downstream_repo_name(repo, options)
+      options.downstream_repo_name || repo.sub(/^manageiq/, settings.git_mirror.productization_name)
+    end
+
+    def system(*args)
+      puts "+ #{"dry_run: " if dry_run?}#{args.join(" ")}"
+      return true if dry_run?
+
+      args << {} unless args.last.is_a?(Hash)
+      args.last[[:out, :err]] = ["/tmp/mirror_helper_out", "w"]
+
+      super.tap do |result|
+        unless result
+          @errors_occurred = true
+          STDERR.puts "!!! An error has occurred:\n#{File.read("/tmp/mirror_helper_out")}".bold.red
+        end
+      end
+    end
+
+    def with_repo(repo, options)
+      repo_name = downstream_repo_name(repo, options)
+      puts "\n==== Mirroring #{repo_name} ====".bold.cyan
+
+      working_dir = settings.git_mirror.working_directory
+      FileUtils.mkdir_p(working_dir)
+
+      path = "#{working_dir}/#{repo_name}"
+      clone_repo(repo, repo_name, path, options.remote_source) unless File.directory?(path)
+
+      Dir.chdir(path) do
+        puts "\n==== Fetching for #{repo_name} ====".bold.green
+        # Enforce an order for remote fetching to ensure that moved
+        #   tags prefer what is on upstream
+        system("git fetch backup --prune --tags") if backup_remote_defined? && remote_exists?("backup")
+        system("git fetch downstream --prune --tags")
+        system("git fetch upstream --prune --tags") if [:red_hat_cloudforms, :upstream].include?(options.remote_source)
+
+        yield
+      end
+
+      puts
+    end
+
+    def clone_repo(upstream_repo, downstream_repo, path, remote_source)
+      upstream_remote = settings.git_mirror.remotes[remote_source]
+      raise "remote '#{remote_source}'' not found in settings" if upstream_remote.nil?
+
+      system("git clone #{upstream_remote}/#{upstream_repo}.git #{path} -o upstream")
+      Dir.chdir(path) do
+        unless remote_exists?("downstream")
+          downstream_remote = settings.git_mirror.remotes.downstream
+          raise "remote 'downstream' not found in settings" if downstream_remote.nil?
+
+          system("git remote add downstream #{downstream_remote}/#{downstream_repo}.git")
+        end
+        if backup_remote_defined? && !remote_exists?("backup")
+          backup_remote = settings.git_mirror.remotes.backup
+          system("git remote add backup #{backup_remote}/#{downstream_repo}.git")
+        end
+      end
+    end
+
+    def remote_refs(repo, remote)
+      return unless remote_exists?(remote)
+
+      `git ls-remote #{remote} | grep "heads"`.split("\n").collect do |line|
+        branch = line.split("/").last
+        next if remote == "upstream" && !upstream_branch?(repo, branch)
+        "#{remote}/#{branch}:refs/heads/#{branch}"
+      end.compact.join(" ")
+    end
+
+    def remote_exists?(remote)
+      `git ls-remote #{remote} --exit-code 2>/dev/null`
+      $? == 0
+    end
+
+    def upstream_branch?(repo, branch)
+      (mirror_branches_for(repo).keys.collect(&:to_s) + ["master"]).include?(branch)
+    end
+
+    def remote_branch?(branch)
+      !`git branch -r | grep "\\b#{branch}\\b"`.strip.empty?
+    end
+
+    def sync_branch(source_remote, source_name, dest_remote, dest_name)
+      return unless dest_remote && dest_name
+
+      source_fq_name = "#{source_remote}/#{source_name}"
+      dest_fq_name   = "#{dest_remote}/#{dest_name}"
+
+      puts "\n==== Syncing #{source_name} to #{dest_name} ====".bold.green
+      unless remote_branch?(source_fq_name)
+        puts "! Skipping sync of #{source_name} to #{dest_name} since #{source_fq_name} branch does not exist".yellow
+        return
+      end
+
+      start_point = remote_branch?(dest_fq_name) ? dest_fq_name : source_fq_name
+      system("git rebase --abort || true") # `git rebase --abort` will exit non-zero if there's nothing to abort
+      system("git reset --hard")
+
+      success =
+        system("git checkout -B #{dest_name} #{start_point}") &&
+        system("git pull --rebase=merges #{source_remote} #{source_name}") &&
+        system("git push -f #{dest_remote} #{dest_name}")
+
+      if backup_remote_defined?
+        if success && remote_exists?("backup")
+          success = system("git push -f backup #{dest_name}")
+        else
+          puts "! Skipping sync of #{source_name} to backup/#{dest_name} since backup remote does not exist".yellow
+        end
+      end
+
+      success
+    end
+
+    def mirror_remote_refs(repo, source_remote, dest_remote)
+      puts "\n==== Mirroring #{source_remote} to #{dest_remote} ====".bold.green
+      unless remote_exists?(dest_remote)
+        puts "! Skipping mirror of #{source_remote} to #{dest_remote} since #{dest_remote} does not exist".yellow
+        return
+      end
+
+      refs = remote_refs(repo, source_remote)
+      if refs.to_s.strip.empty?
+        puts "! Skipping mirror of #{source_remote} to #{dest_remote} since there are no refs to mirror".yellow
+        return
+      end
+
+      system("git push #{dest_remote} #{refs}") &&
+        system("git push -f #{dest_remote} --tags")
+    end
+  end
+end

data/lib/multi_repo/helpers/license.rb

@@ -0,0 +1,106 @@
+module MultiRepo::Helpers
+  class License
+    attr_reader :repo, :dry_run
+
+    def initialize(repo, dry_run: false, **)
+      @repo    = repo
+      @dry_run = dry_run
+      reload
+    end
+
+    def save!
+      save_license!
+      save_readme_license!
+      reload unless dry_run
+      true
+    end
+
+    def content
+      @license.text
+    end
+
+    def license
+      @license.key
+    end
+
+    def license=(value)
+      @license = Licensee::License.new(value)
+    end
+
+    private
+
+    def reload
+      require 'licensee'
+      @license = Licensee.license(repo.path.to_s)
+    end
+
+    def save_license!
+      repo.rm_file("LICENSE.md", dry_run: dry_run)
+      repo.rm_file("LICENSE", dry_run: dry_run)
+      repo.write_file("LICENSE.txt", content, dry_run: dry_run)
+    end
+
+    def save_readme_license!
+      readme_file = repo.detect_readme_file
+      lines = readme_file ? File.read(repo.path.join(readme_file)).lines : []
+      readme_file ||= "README.md"
+
+      apply_readme_license!(lines)
+      repo.write_file(readme_file, lines.join, dry_run: dry_run)
+    end
+
+    def extract_readme_license(lines)
+      section = lines.each.with_index.drop_while do |l, _i|
+        !l.downcase.include?("## license")
+      end.take_while.with_index do |(l, _i), i2|
+        i2 == 0 || !l.start_with?("## ")
+      end
+
+      section.each { |l, _i| l =~ /(mit|apache)/i && break }
+      type =
+        case $1.presence&.downcase
+        when "mit"    then "mit"
+        when "apache" then "apache-2.0"
+        end
+
+      return type, section.map(&:last)
+    end
+
+    def license_details
+      case license
+      when "mit"
+        {
+          :name => "MIT License",
+          :url  => "https://opensource.org/licenses/MIT"
+        }
+      when "apache-2.0"
+        {
+          :name => "Apache License 2.0",
+          :url  => "http://www.apache.org/licenses/LICENSE-2.0"
+        }
+      end
+    end
+
+    def apply_readme_license!(lines)
+      readme_license, readme_license_indexes = extract_readme_license(lines)
+      return if readme_license == license
+
+      details = license_details
+      return unless details
+
+      lines.reject!.with_index { |_l, i| readme_license_indexes.include?(i) }
+
+      start_index = readme_license_indexes[0] || lines.size
+      new_lines = <<~EOF.lines
+## License
+
+This project is available as open source under the terms of the [#{details[:name]}](#{details[:url]}).
+
+EOF
+
+      new_lines.reverse_each do |l|
+        lines.insert(start_index, l)
+      end
+    end
+  end
+end