repository_merger 0.1.0

data/lib/repository_merger/commit_history_merger.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+require_relative 'branch_local_commit_map'
+
+class RepositoryMerger
+  class CommitHistoryMerger
+    attr_reader :configuration, :original_references, :commit_message_conversion, :progress_title
+    attr_accessor :wants_to_abort
+
+    def initialize(references, configuration:, commit_message_conversion: nil, progress_title: nil)
+      @original_references = references
+      @configuration = configuration
+      @commit_message_conversion = commit_message_conversion
+      @progress_title = progress_title
+    end
+
+    def run
+      logger.verbose("Importing commits from #{original_references.map { |ref| "#{ref.name} (#{ref.repo.name})" }.join(', ')}", title: true)
+      logger.start_tracking_progress_for('commits', total: unprocessed_original_commit_queue.size, title: progress_title)
+
+      while (original_commit = unprocessed_original_commit_queue.next)
+        process_commit(original_commit)
+        break if wants_to_abort
+      end
+
+      monorepo_head_commit
+    ensure
+      repo_commit_map.merge!(branch_local_commit_map)
+      repo_commit_map.save if repo_commit_map.path
+    end
+
+    private
+
+    attr_reader :monorepo_head_commit
+
+    def process_commit(original_commit)
+      logger.verbose "  #{original_commit.commit_time} [#{original_commit.repo.name}] #{original_commit.message.each_line.first}"
+
+      if (monorepo_commit = already_imported_monorepo_commit_for(original_commit))
+        logger.verbose "    Already imported as #{monorepo_commit.abbreviated_id}. Skipping."
+      else
+        monorepo_commit = import_commit_into_monorepo(original_commit)
+        logger.verbose "    Created commit #{monorepo_commit.abbreviated_id}."
+      end
+
+      branch_local_commit_map.register(
+        monorepo_commit: monorepo_commit,
+        original_commit: original_commit
+      )
+
+      if mainline?(original_commit)
+        @monorepo_head_commit = monorepo_commit
+      end
+
+      logger.increment_progress
+    end
+
+    def already_imported_monorepo_commit_for(original_commit)
+      monorepo_commits = repo_commit_map.monorepo_commits_for(original_commit)
+
+      monorepo_commits.find do |monorepo_commit|
+        monorepo_commit.parents == parent_commits_in_monorepo_for(original_commit)
+      end
+    end
+
+    def import_commit_into_monorepo(original_commit)
+      parent_commits_in_monorepo = parent_commits_in_monorepo_for(original_commit)
+
+      monorepo.import_commit(
+        original_commit,
+        new_parents: parent_commits_in_monorepo,
+        subdirectory: original_commit.repo.name,
+        message: commit_message_from(original_commit)
+      )
+    end
+
+    def parent_commits_in_monorepo_for(original_commit)
+      if original_commit.root?
+        return [monorepo_head_commit].compact
+      end
+
+      original_commit.parents.map do |original_parent_commit|
+        if mainline?(original_commit) && mainline?(original_parent_commit)
+          monorepo_head_commit
+        else
+          branch_local_commit_map.monorepo_commit_for(original_parent_commit)
+        end
+      end
+    end
+
+    def mainline?(original_commit)
+      original_reference = original_references_by_repo[original_commit.repo]
+      original_reference.mainline?(original_commit)
+    end
+
+    def commit_message_from(original_commit)
+      if commit_message_conversion
+        commit_message_conversion.call(original_commit)
+      else
+        original_commit.message
+      end
+    end
+
+    def unprocessed_original_commit_queue
+      @unprocessed_original_commit_queue ||= OriginalCommitQueue.new(original_references)
+    end
+
+    def original_references_by_repo
+      @original_references_by_repo ||= original_references.each_with_object({}) do |original_reference, hash|
+        hash[original_reference.repo] = original_reference
+      end
+    end
+
+    def monorepo
+      configuration.monorepo
+    end
+
+    def repo_commit_map
+      configuration.repo_commit_map
+    end
+
+    def branch_local_commit_map
+      @branch_local_commit_map ||= BranchLocalCommitMap.new(monorepo: monorepo)
+    end
+
+    def logger
+      configuration.logger
+    end
+
+    class OriginalCommitQueue
+      attr_reader :references
+
+      def initialize(references)
+        @references = references
+      end
+
+      def next
+        queue_having_earliest_commit = commit_queues.reject(&:empty?).min_by { |queue| queue.first.commit_time }
+        queue_having_earliest_commit&.shift
+      end
+
+      def size
+        commit_queues.sum(&:size)
+      end
+
+      def commit_queues
+        @commit_queues ||= references.map(&:topologically_ordered_commits_from_root).map(&:dup)
+      end
+    end
+  end
+end

data/lib/repository_merger/commit_map.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class RepositoryMerger
+  module CommitMap
+    def original_commit_key(commit)
+      "#{commit.id}@#{commit.repo.name}"
+    end
+  end
+end

data/lib/repository_merger/configuration.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_relative 'logger'
+require_relative 'mono_repository'
+require_relative 'repository'
+require_relative 'repository_commit_map'
+
+class RepositoryMerger
+  class Configuration
+    attr_reader :original_repo_paths, :monorepo_path, :commit_map_file_path, :log_output, :verbose_logging
+
+    def initialize(original_repo_paths:, monorepo_path:, commit_map_file_path: 'commit_map.json', log_output: $stdout, verbose_logging: false)
+      @original_repo_paths = original_repo_paths
+      @monorepo_path = monorepo_path
+      @commit_map_file_path = commit_map_file_path
+      @log_output = log_output
+      @verbose_logging = verbose_logging
+    end
+
+    def original_repos
+      @original_repos ||= original_repo_paths.map { |path| Repository.new(path) }
+    end
+
+    def monorepo
+      @monorepo ||= MonoRepository.new(monorepo_path)
+    end
+
+    def repo_commit_map
+      @repo_commit_map ||= RepositoryCommitMap.new(path: commit_map_file_path, monorepo: monorepo)
+    end
+
+    def logger
+      @logger ||= Logger.new(log_output, verbose: verbose_logging)
+    end
+  end
+end

data/lib/repository_merger/github_issue_reference.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+class RepositoryMerger
+  module GitHubIssueReference
+    RepositoryLocalReference = Struct.new(:issue_number, keyword_init: true)
+    AbsoluteReference = Struct.new(:username, :repo_name, :issue_number, keyword_init: true)
+
+    # https://github.com/isiahmeadows/github-limits
+    USERNAME_PATTERN = /(?<username>[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,38})/i.freeze
+    REPOSITORY_NAME_PATTERN = /(?<repo_name>[a-z0-9.\-_]{1,100})/i.freeze
+    ISSUE_NUMBER_PATTERN = /(?<issue_number>\d{1,5})/.freeze # Technically max is 1073741824 but it won't exist in real life
+
+    # https://docs.github.com/en/github/writing-on-github/working-with-advanced-formatting/autolinked-references-and-urls#issues-and-pull-requests
+    REPO_LOCAL_REFERENCE_PATTERN = /(?<!\w)(?<repo_local_reference>(?:#|GH-)#{ISSUE_NUMBER_PATTERN})(?!\w)/i.freeze
+    ABSOLUTE_REFERENCE_PATTERN = /(?<!\w)(?<absolute_reference>#{USERNAME_PATTERN}\/#{REPOSITORY_NAME_PATTERN}##{ISSUE_NUMBER_PATTERN})(?!\w)/.freeze
+    REFERENCE_PATTERN = /(?:#{REPO_LOCAL_REFERENCE_PATTERN}|#{ABSOLUTE_REFERENCE_PATTERN})/.freeze
+
+    def self.extract_references_from(message)
+      references = []
+
+      message.scan(REFERENCE_PATTERN) do
+        reference = create_referece_from(Regexp.last_match)
+        references << reference if reference
+      end
+
+      references
+    end
+
+    def self.convert_repo_local_references_to_absolute_ones_in(message, username:, repo_name:)
+      message.gsub(REPO_LOCAL_REFERENCE_PATTERN) do
+        reference = create_referece_from(Regexp.last_match)
+        raise unless reference
+        "#{username}/#{repo_name}##{reference.issue_number}"
+      end
+    end
+
+    def self.create_referece_from(match)
+      if match[:repo_local_reference]
+        RepositoryLocalReference.new(issue_number: Integer(match[:issue_number]))
+      elsif match[:absolute_reference]
+        AbsoluteReference.new(
+          username: match[:username],
+          repo_name: match[:repo_name],
+          issue_number: Integer(match[:issue_number])
+        )
+      end
+    end
+  end
+end

data/lib/repository_merger/logger.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'ruby-progressbar'
+
+class RepositoryMerger
+  class Logger
+    attr_reader :output
+
+    def initialize(output, verbose:)
+      @output = output
+      @verbose = verbose
+    end
+
+    def info(message, title: false)
+      log(message, title: title)
+    end
+
+    def verbose(message, title: false)
+      return unless verbose?
+      log(message, title: title)
+    end
+
+    def verbose?
+      @verbose
+    end
+
+    def start_tracking_progress_for(plural_noun, total:, title: nil)
+      format = " %c/%C #{plural_noun} |%w>%i| %e "
+      format = " %t#{format}" if title
+
+      @progressbar = ProgressBar.create(
+        format: format,
+        output: output,
+        title: title,
+        total: total
+      )
+    end
+
+    def increment_progress
+      progressbar.increment
+    end
+
+    private
+
+    def log(message, title:)
+      if title
+        message = "#{'=' * 10} #{message} #{'=' * 10}"
+      end
+
+      progressbar.log(message)
+    end
+
+    def progressbar
+      @progressbar ||= ProgressBar.create
+    end
+  end
+end

data/lib/repository_merger/mono_repository.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require_relative 'repository'
+require 'rugged'
+
+class RepositoryMerger
+  class MonoRepository < Repository
+    def initialize(_path)
+      super
+      configure
+    end
+
+    def import_commit(original_commit, new_parents:, subdirectory:, message: nil)
+      checkout_contents_if_needed(new_parents.first) unless new_parents.empty?
+
+      stage_contents_of(original_commit, subdirectory: subdirectory)
+
+      create_commit_with_metadata_of(
+        original_commit,
+        new_parent_ids: new_parents.map(&:id),
+        message: message
+      )
+    end
+
+    def import_tag(original_tag, new_commit_id:, new_tag_name:)
+      # This is to suppress warning messages
+      # `warning: Using the last argument as keyword parameters is deprecated`
+      # from rugged gem until a fixed version is released.
+      # https://github.com/libgit2/rugged/pull/840
+      if original_tag.annotation
+        rugged_repo.tags.create(new_tag_name, new_commit_id, **original_tag.annotation)
+      else
+        rugged_repo.tags.create(new_tag_name, new_commit_id)
+      end
+
+      tag_for(new_tag_name)
+    end
+
+    def create_or_update_branch(branch_name, commit_id:)
+      if branch_for(branch_name)
+        # `rugged_repo.branches.create` with master branch fails with error:
+        # cannot force update branch 'master' as it is the current HEAD of the repository. (Rugged::ReferenceError)
+        rugged_repo.references.update("refs/heads/#{branch_name}", commit_id)
+      else
+        rugged_repo.branches.create(branch_name, commit_id)
+      end
+    end
+
+    private
+
+    def configure
+      rugged_repo.config['core.ignorecase'] = false
+    end
+
+    attr_accessor :current_checked_out_commit_id
+
+    def checkout_contents_if_needed(commit)
+      return if empty_commit_for_debug?
+      return if commit.id == current_checked_out_commit_id
+      commit.checkout_contents
+      @current_checked_out_commit_id = commit.id
+    end
+
+    def stage_contents_of(original_commit, subdirectory:)
+      return if empty_commit_for_debug?
+
+      original_commit.extract_contents_into(File.join(path, subdirectory))
+
+      # First, track (almost) all files (including removed files) with .gitignore rules
+      rugged_repo.index.add_all(subdirectory)
+
+      # Then specify all the file paths explicily to track files
+      # that matches .gitignore but is tracked in the original commit.
+      # We cannot use Rugged::Index#add_all with :force option
+      # since it has a bug where it raises strange error when adding files in a ignored directory.
+      # https://github.com/libgit2/libgit2/issues/4377
+      original_commit.files.each do |original_file_path|
+        monorepo_file_path = File.join(subdirectory, original_file_path)
+        rugged_repo.index.add(monorepo_file_path)
+      end
+
+      rugged_repo.index.write
+    end
+
+    def create_commit_with_metadata_of(original_commit, new_parent_ids:, message:)
+      original_rugged_commit = original_commit.rugged_commit
+
+      new_commit_id = Rugged::Commit.create(rugged_repo, {
+        message: message || original_rugged_commit.message,
+        committer: original_rugged_commit.committer,
+        author: original_rugged_commit.author,
+        tree: empty_commit_for_debug? ? empty_tree.oid : rugged_repo.index.write_tree,
+        parents: new_parent_ids
+      })
+
+      @current_checked_out_commit_id = new_commit_id
+
+      commit_for(new_commit_id)
+    end
+
+    def empty_commit_for_debug?
+      ENV['REPO_MERGER_IMPORT_AS_EMPTY_COMMITS']
+    end
+
+    def empty_tree
+      @empty_tree ||= Rugged::Tree.empty(rugged_repo)
+    end
+  end
+end

data/lib/repository_merger/reference.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative 'commit'
+
+require 'rugged'
+require 'set'
+
+class RepositoryMerger
+  module Reference
+    def target_commit
+      raise NotImplementedError
+    end
+
+    def topologically_ordered_commits_from_root
+      @topologically_ordered_commits_from_root ||= begin
+        walker = Rugged::Walker.new(repo.rugged_repo)
+        walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE)
+        walker.push(target_commit.id)
+        walker.map { |rugged_commit| Commit.new(rugged_commit, repo) }.freeze
+      end
+    end
+
+    def mainline?(commit)
+      mainline_commit_ids.include?(commit.id)
+    end
+
+    def mainline_commit_ids
+      @mainline_commit_ids ||= Set.new.tap do |mainline_commit_ids|
+        commit = target_commit
+
+        while commit
+          mainline_commit_ids << commit.id
+          commit = commit.parents.first
+        end
+      end.freeze
+    end
+  end
+end

data/lib/repository_merger/repository.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require_relative 'branch'
+require_relative 'commit'
+require_relative 'tag'
+
+require 'rugged'
+
+class RepositoryMerger
+  class Repository
+    attr_reader :path
+
+    def initialize(path)
+      @path = File.expand_path(path)
+    end
+
+    def rugged_repo
+      @rugged_repo ||= Rugged::Repository.new(path)
+    end
+
+    def ==(other)
+      path == other.path
+    end
+
+    alias eql? ==
+
+    def hash
+      path.hash
+    end
+
+    def name
+      File.basename(path)
+    end
+
+    def branch_for(name)
+      rugged_branch = rugged_repo.branches[name]
+      return nil unless rugged_branch
+      Branch.new(rugged_branch, self)
+    end
+
+    def branches
+      rugged_repo.branches.map do |rugged_branch|
+        Branch.new(rugged_branch, self)
+      end
+    end
+
+    def tag_for(name)
+      rugged_tag = rugged_repo.tags[name]
+      return nil unless rugged_tag
+      Tag.new(rugged_tag, self)
+    end
+
+    def tags
+      rugged_repo.tags.map do |rugged_tag|
+        Tag.new(rugged_tag, self)
+      end
+    end
+
+    def commit_for(commit_id)
+      object = rugged_repo.lookup(commit_id)
+      return nil unless object.is_a?(Rugged::Commit)
+      Commit.new(object, self)
+    end
+  end
+end

data/lib/repository_merger/repository_commit_map.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative 'commit_map'
+require 'json'
+
+class RepositoryMerger
+  class RepositoryCommitMap
+    include CommitMap
+
+    CannotSaveWithoutPathError = Class.new(StandardError)
+
+    attr_reader :path, :monorepo
+
+    def initialize(monorepo:, path: nil)
+      @path = path
+      @monorepo = monorepo
+    end
+
+    def map
+      @map ||=
+        if path && File.exist?(path)
+          json = File.read(path)
+          JSON.parse(json)
+        else
+          {}
+        end
+    end
+
+    def merge!(branch_local_commit_map)
+      branch_local_commit_map.map.each do |original_commit_key, monorepo_commit_id|
+        map[original_commit_key] ||= []
+        next if map[original_commit_key].include?(monorepo_commit_id)
+        map[original_commit_key] << monorepo_commit_id
+      end
+    end
+
+    def save
+      raise CannotSaveWithoutPathError unless path
+      json = JSON.pretty_generate(map)
+      File.write(path, json)
+    end
+
+    def monorepo_commits_for(original_commit)
+      commit_ids = monorepo_commit_ids_for(original_commit)
+      commit_ids.map { |id| monorepo.commit_for(id) }
+    end
+
+    def monorepo_commit_ids_for(original_commit)
+      map[original_commit_key(original_commit)] || []
+    end
+  end
+end

data/lib/repository_merger/tag.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require_relative 'reference'
+
+class RepositoryMerger
+  Tag = Struct.new(:rugged_tag, :repo) do
+    include Reference
+
+    def name
+      rugged_tag.name
+    end
+
+    def target_commit
+      commit_id =
+        if rugged_tag.annotated?
+          rugged_tag.annotation.target_id
+        else
+          rugged_tag.target_id
+        end
+
+      repo.commit_for(commit_id)
+    end
+
+    def annotation
+      if rugged_tag.annotated?
+        { tagger: rugged_tag.annotation.tagger, message: rugged_tag.annotation.message }
+      else
+        nil
+      end
+    end
+
+    def id
+      name
+    end
+
+    def revision_id
+      name
+    end
+  end
+end

data/lib/repository_merger/tag_importer.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+class RepositoryMerger
+  class TagImporter
+    attr_reader :original_tags, :configuration, :tag_name_conversion
+
+    def initialize(tags, configuration:, tag_name_conversion:)
+      @original_tags = tags
+      @configuration = configuration
+      @tag_name_conversion = tag_name_conversion
+    end
+
+    def run
+      logger.verbose('Importing tags', title: true)
+      logger.start_tracking_progress_for('tags', total: original_tags.size)
+
+      original_tags.each do |original_tag|
+        process_tag(original_tag)
+      end
+    end
+
+    def process_tag(original_tag)
+      logger.verbose "  [#{original_tag.repo.name}] #{original_tag.name}"
+
+      new_tag_name = tag_name_conversion.call(original_tag)
+
+      if new_tag_name
+        if monorepo.tag_for(new_tag_name)
+          logger.verbose "    Already imported as #{new_tag_name.inspect}. Skipping."
+        else
+          new_tag = import_tag_into_monorepo(original_tag, new_tag_name: new_tag_name)
+          logger.verbose "    Imported as #{new_tag_name.inspect}." if new_tag
+        end
+      else
+        logger.verbose '    Not for import. Skipping.'
+      end
+
+      logger.increment_progress
+    end
+
+    def import_tag_into_monorepo(original_tag, new_tag_name:)
+      target_commit_id_in_monorepo = monorepo_commit_id_for(original_tag)
+
+      unless target_commit_id_in_monorepo
+        commit_description = "#{original_tag.target_commit.message.chomp.inspect} (#{original_tag.target_commit.abbreviated_id}) in #{original_tag.repo.name}"
+        logger.verbose "    The target commit #{commit_description} is not yet imported. Skipping."
+        return nil
+      end
+
+      monorepo.import_tag(
+        original_tag,
+        new_commit_id: target_commit_id_in_monorepo,
+        new_tag_name: new_tag_name
+      )
+    end
+
+    def monorepo_commit_id_for(original_tag)
+      # TODO: Choosing the first one might be wrong
+      configuration.repo_commit_map.monorepo_commit_ids_for(original_tag.target_commit).first
+    end
+
+    def monorepo
+      configuration.monorepo
+    end
+
+    def logger
+      configuration.logger
+    end
+  end
+end

data/lib/repository_merger/version.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class RepositoryMerger
+  module Version
+    MAJOR = 0
+    MINOR = 1
+    PATCH = 0
+
+    def self.to_s
+      [MAJOR, MINOR, PATCH].join('.')
+    end
+  end
+end