repository_merger 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 1490218b0506f8561f3c7e6044783d6b583ceff8ff0fbb2f163fbcb3e793b323
+  data.tar.gz: 9766418d0125e6b70670bbf7e132bd07ab322b6871ccdda2c7c71b07d31d8b5e
+SHA512:
+  metadata.gz: cbe57c8d914f2669b1dd39e8a70fabff6db161a544d354d56895a221dc641024fe1b3e19143822aa59c186fdd8dd583463afeb427f49f49a95bd1b96588d7490
+  data.tar.gz: 498183071861f94ded94f03c398391521a51fcf32e9f45437e8568be9c7dcf9f7df059761c36a7eb24e2473c19ba8d5de0f78c86b2ef66d86e03bd2a8fbaf895

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,30 @@
+inherit_from: .rubocop_todo.yml
+
+AllCops:
+  Exclude:
+    - 'dest/**/*'
+    - 'tmp/**/*'
+    - 'vendor/**/*'
+  NewCops: enable
+  SuggestExtensions: false
+
+Layout/FirstArrayElementIndentation:
+  EnforcedStyle: consistent
+
+Layout/FirstHashElementIndentation:
+  EnforcedStyle: consistent
+
+Layout/HashAlignment:
+  EnforcedHashRocketStyle: table
+
+Naming/HeredocDelimiterNaming:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/EmptyElse:
+  Enabled: false
+
+Style/IfUnlessModifier:
+  Enabled: false

data/.rubocop_todo.yml

@@ -0,0 +1,91 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2021-10-03 08:53:27 UTC using RuboCop version 1.22.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 10
+# Cop supports --auto-correct.
+Layout/EmptyLineAfterGuardClause:
+  Exclude:
+    - 'lib/repository_merger/branch_local_commit_map.rb'
+    - 'lib/repository_merger/github_issue_reference.rb'
+    - 'lib/repository_merger/logger.rb'
+    - 'lib/repository_merger/mono_repository.rb'
+    - 'lib/repository_merger/repository.rb'
+    - 'lib/repository_merger/repository_commit_map.rb'
+    - 'spec/support/fixture_helper.rb'
+    - 'spec/support/git_helper.rb'
+
+# Offense count: 8
+# Cop supports --auto-correct.
+# Configuration parameters: AllowMultipleStyles, EnforcedHashRocketStyle, EnforcedColonStyle, EnforcedLastArgumentHashStyle.
+# SupportedHashRocketStyles: key, separator, table
+# SupportedColonStyles: key, separator, table
+# SupportedLastArgumentHashStyles: always_inspect, always_ignore, ignore_implicit, ignore_explicit
+Layout/HashAlignment:
+  Exclude:
+    - 'spec/rspec/monorepo_spec.rb'
+    - 'spec/rspec/original_repos_spec.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, IndentationWidth.
+# SupportedStyles: aligned, indented, indented_relative_to_receiver
+Layout/MultilineMethodCallIndentation:
+  Exclude:
+    - 'spec/repository_merger/mono_repository_spec.rb'
+
+# Offense count: 4
+# Configuration parameters: IgnoredMethods, CountRepeatedAttributes.
+Metrics/AbcSize:
+  Max: 42
+
+# Offense count: 36
+# Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.
+# IgnoredMethods: refine
+Metrics/BlockLength:
+  Max: 686
+
+# Offense count: 4
+# Configuration parameters: CountComments, CountAsOne.
+Metrics/ClassLength:
+  Max: 442
+
+# Offense count: 10
+# Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.
+Metrics/MethodLength:
+  Max: 40
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, AllowInnerSlashes.
+# SupportedStyles: slashes, percent_r, mixed
+Style/RegexpLiteral:
+  Exclude:
+    - 'exe/merge_rspec_repos'
+    - 'lib/repository_merger/github_issue_reference.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: MinSize.
+# SupportedStyles: percent, brackets
+Style/SymbolArray:
+  EnforcedStyle: brackets
+
+# Offense count: 5
+# Cop supports --auto-correct.
+# Configuration parameters: WordRegex.
+# SupportedStyles: percent, brackets
+Style/WordArray:
+  EnforcedStyle: percent
+  MinSize: 4
+
+# Offense count: 14
+# Cop supports --auto-correct.
+# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
+# URISchemes: http, https
+Layout/LineLength:
+  Max: 199

data/Gemfile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gemspec
+
+group :development do
+  gem 'pry-byebug', '~> 3.9'
+  gem 'rake', '~> 13.0'
+  gem 'rspec', '~> 3.10'
+  gem 'rubocop', '~> 1.22'
+end

data/Gemfile.lock

@@ -0,0 +1,68 @@
+PATH
+  remote: .
+  specs:
+    repository_merger (0.1.0)
+      ruby-progressbar (~> 1.11)
+      rugged (~> 1.2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.2)
+    byebug (11.1.3)
+    coderay (1.1.3)
+    diff-lcs (1.4.4)
+    method_source (1.0.0)
+    parallel (1.21.0)
+    parser (3.0.2.0)
+      ast (~> 2.4.1)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry-byebug (3.9.0)
+      byebug (~> 11.0)
+      pry (~> 0.13.0)
+    rainbow (3.0.0)
+    rake (13.0.6)
+    regexp_parser (2.1.1)
+    rexml (3.2.5)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
+    rubocop (1.22.0)
+      parallel (~> 1.10)
+      parser (>= 3.0.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.12.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.12.0)
+      parser (>= 3.0.1.1)
+    ruby-progressbar (1.11.0)
+    rugged (1.2.0)
+    unicode-display_width (2.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  pry-byebug (~> 3.9)
+  rake (~> 13.0)
+  repository_merger!
+  rspec (~> 3.10)
+  rubocop (~> 1.22)
+
+BUNDLED WITH
+   2.2.28

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 Yuji Nakayama
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,5 @@
+[![Test](https://github.com/yujinakayama/repository_merger/workflows/Test/badge.svg)](https://github.com/yujinakayama/repository_merger/actions?query=workflow%3ATest)
+
+# RepositoryMerger
+
+For https://github.com/rspec/rspec-core/issues/2509

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+
+desc 'Remove all generated files including monorepo'
+task :clean do
+  require 'fileutils'
+
+  Dir.chdir(__dir__) do
+    FileUtils.rm_rf(['dest', 'tmp'])
+  end
+end

data/exe/merge_rspec_repos

@@ -0,0 +1,165 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require_relative '../lib/repository_merger'
+require_relative '../lib/repository_merger/github_issue_reference'
+require 'pry'
+
+def main
+  Signal.trap('INT') do
+    puts 'Aborting...'
+    exit(1)
+  end
+
+  Dir.chdir(dest_dir)
+
+  total_import_phase_count = tag_names_unreachable_from_target_branches.size + target_branch_names.size
+  current_import_phase_number = 1
+
+  tag_names_unreachable_from_target_branches.each do |tag_name|
+    tags = configuration.original_repos.map { |repo| repo.tag_for(tag_name) }.compact
+
+    repo_merger.merge_commit_history_of(
+      tags,
+      commit_message_conversion: method(:convert_commit_message),
+      progress_title: "[#{current_import_phase_number}/#{total_import_phase_count}: #{tag_name}]"
+    )
+
+    # We need to import the tags here before importing target branches
+    # so that the tags will reference right commits.
+    repo_merger.import_tags(tags, tag_name_conversion: method(:convert_tag_name))
+
+    current_import_phase_number += 1
+  end
+
+  target_branch_names.each do |branch_name|
+    repo_merger.merge_commit_history_of_branches_named(
+      branch_name,
+      commit_message_conversion: method(:convert_commit_message),
+      progress_title: "[#{current_import_phase_number}/#{total_import_phase_count}: #{branch_name}]"
+    )
+
+    current_import_phase_number += 1
+  end
+
+  repo_merger.import_all_tags(tag_name_conversion: method(:convert_tag_name))
+
+  Dir.chdir(configuration.monorepo_path) do
+    # Clear index and working tree since they're cluttered after the merge
+    `git reset --hard`
+
+    # Merged repos without GC tend to have large volume
+    puts 'Running `git gc`...'
+    system('git gc')
+  end
+end
+
+def repo_merger
+  @repo_merger ||= RepositoryMerger.new(configuration)
+end
+
+def configuration
+  @configuration ||= RepositoryMerger::Configuration.new(
+    original_repo_paths: fetch_original_repos_if_needed,
+    monorepo_path: create_monorepo_if_needed,
+    verbose_logging: ARGV.include?('-v')
+  )
+end
+
+def target_branch_names
+  @target_branch_names ||= begin
+    all_branch_names = configuration.original_repos.flat_map { |repo| repo.branches.map(&:name) }.uniq.sort
+    ['origin/main'] + all_branch_names.grep(/\Aorigin\/\d+-\d+-(maintenance|stable)\z/)
+  end
+end
+
+def tag_names_unreachable_from_target_branches
+  %w[
+    v2.0.0.beta.9
+    v3.5.0.beta2
+  ]
+end
+
+def convert_commit_message(original_commit)
+  message = RepositoryMerger::GitHubIssueReference.convert_repo_local_references_to_absolute_ones_in(
+    original_commit.message,
+    username: 'rspec',
+    repo_name: original_commit.repo.name
+  )
+
+  lines = message.split(/\r?\n/)
+
+  scope = original_commit.repo.name.sub(/\Arspec-/, '')
+  lines.first.prepend("[#{scope}] ")
+
+  if lines.size == 1
+    lines << ''
+  else
+    lines.concat([
+      '',
+      '---'
+    ])
+  end
+
+  original_commit_url = "https://github.com/rspec/#{original_commit.repo.name}/commit/#{original_commit.id}"
+  lines << "This commit was imported from #{original_commit_url}."
+
+  lines.join("\n")
+end
+
+def convert_tag_name(original_tag)
+  tag_name = original_tag.name
+  tag_name = "v#{tag_name}" if tag_name.match?(/\A\d+\.\d+\.\d+/)
+
+  scope = original_tag.repo.name.sub(/\Arspec-/, '')
+
+  "#{tag_name}-#{scope}"
+end
+
+def dest_dir
+  dest_dir = 'dest'
+  Dir.mkdir(dest_dir) unless Dir.exist?(dest_dir)
+  dest_dir
+end
+
+def fetch_original_repos_if_needed
+  repo_urls = %w[
+    https://github.com/yujinakayama/rspec.git
+    https://github.com/yujinakayama/rspec-core.git
+    https://github.com/yujinakayama/rspec-expectations.git
+    https://github.com/yujinakayama/rspec-mocks.git
+    https://github.com/yujinakayama/rspec-support.git
+  ]
+
+  original_repos_dir = 'original_repos'
+
+  Dir.mkdir(original_repos_dir) unless Dir.exist?(original_repos_dir)
+
+  Dir.chdir(original_repos_dir) do |current_directory|
+    repo_urls.map do |repo_url|
+      repo_name = File.basename(repo_url, '.git')
+
+      unless Dir.exist?(repo_name)
+        system("git clone #{repo_url}")
+      end
+
+      File.join(current_directory, repo_name)
+    end
+  end
+end
+
+def create_monorepo_if_needed
+  monorepo_dir = 'monorepo'
+
+  unless Dir.exist?(monorepo_dir)
+    Dir.mkdir(monorepo_dir)
+    Dir.chdir(monorepo_dir) do
+      system('git init')
+    end
+  end
+
+  monorepo_dir
+end
+
+main

data/lib/repository_merger/branch.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative 'reference'
+
+require 'rugged'
+require 'set'
+
+class RepositoryMerger
+  Branch = Struct.new(:rugged_branch, :repo) do
+    include Reference
+
+    def ==(other)
+      repo == other.repo && canonical_name == other.canonical_name
+    end
+
+    alias_method :eql?, :==
+
+    def hash
+      repo.hash ^ name.hash
+    end
+
+    def name
+      rugged_branch.name
+    end
+
+    def canonical_name
+      rugged_branch.canonical_name
+    end
+
+    def local_name
+      if rugged_branch.remote_name
+        name.delete_prefix("#{rugged_branch.remote_name}/")
+      else
+        name
+      end
+    end
+
+    def target_commit
+      repo.commit_for(rugged_branch.target_id)
+    end
+
+    def revision_id
+      name
+    end
+  end
+end

data/lib/repository_merger/branch_local_commit_map.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require_relative 'commit_map'
+
+class RepositoryMerger
+  class BranchLocalCommitMap
+    include CommitMap
+
+    attr_reader :monorepo
+
+    def initialize(monorepo:)
+      @monorepo = monorepo
+    end
+
+    def map
+      @map ||= {}
+    end
+
+    def register(monorepo_commit:, original_commit:)
+      key = original_commit_key(original_commit)
+      raise if map.key?(key)
+      map[key] = monorepo_commit.id
+    end
+
+    def monorepo_commit_for(original_commit)
+      monorepo_commit_id = monorepo_commit_id_for(original_commit)
+      monorepo.commit_for(monorepo_commit_id)
+    end
+
+    def monorepo_commit_id_for(original_commit)
+      map[original_commit_key(original_commit)]
+    end
+  end
+end

data/lib/repository_merger/commit.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+class RepositoryMerger
+  Commit = Struct.new(:rugged_commit, :repo) do
+    def ==(other)
+      id == other.id
+    end
+
+    alias_method :eql?, :==
+
+    def hash
+      id.hash
+    end
+
+    def id
+      rugged_commit.oid
+    end
+
+    def abbreviated_id
+      id[0, 7]
+    end
+
+    def message
+      rugged_commit.message
+    end
+
+    def commit_time
+      rugged_commit.committer[:time]
+    end
+
+    def root?
+      parents.empty?
+    end
+
+    def parents
+      @parents ||= rugged_commit.parents.map do |parent_rugged_commit|
+        create_parent(parent_rugged_commit)
+      end
+    end
+
+    def files
+      @files ||= begin
+        blob_entries = rugged_commit.tree.walk(:postorder).select { |_, entry| entry[:type] == :blob }
+        blob_entries.map { |directory, entry| "#{directory}#{entry[:name]}" }
+      end
+    end
+
+    def checkout_contents
+      repo.rugged_repo.checkout_tree(id, strategy: :force)
+    end
+
+    def extract_contents_into(directory_path)
+      # We need to specify an empty tree as a baseline tree
+      # to prevent libgit2 from skipping checkout of some contents
+      # by comparing the commit tree with the HEAD tree.
+      empty_tree = Rugged::Tree.empty(repo.rugged_repo)
+
+      repo.rugged_repo.checkout_tree(
+        id,
+        baseline: empty_tree,
+        strategy: [:dont_update_index, :force, :remove_ignored, :remove_untracked],
+        target_directory: directory_path
+      )
+    end
+
+    def revision_id
+      id
+    end
+
+    def inspect
+      "#<#{self.class} id=#{abbreviated_id} repo=#{repo.name} message=#{message.each_line.first.chomp.inspect}>"
+    end
+
+    def pretty_print(pp)
+      pp.text(inspect)
+    end
+
+    private
+
+    def create_parent(parent_rugged_commit)
+      self.class.new(parent_rugged_commit, repo)
+    end
+  end
+end