omnirepo 0.5.0.0.pre

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0de193035ffbc63293ac558163fa9bc6519e0597
+  data.tar.gz: 5597b8bfa5d0ac31564a58fc2444884ed1c24ff3
+SHA512:
+  metadata.gz: 98f40124f15c5be9e75017e11446788d1cc1882f396b228218065b1412e3ee57defdac1efb33f5ddd476707729ed923852d6a9fc11a90e4d5ab6d9188b050912
+  data.tar.gz: ebb92830746b0dbd4dceae34bcd51ef6a57abfcc637be28b349e2371a20f2eb7c66ecfdef4f1a84d3c9de731ad9dd3dafc61d016fa678be5d1ca8e2e634e28fc

data/.gitignore

@@ -0,0 +1,2 @@
+Gemfile.lock
+*pkg

data/CHANGES.md

@@ -0,0 +1,7 @@
+HEAD
+-----
+* Initial release, put together for [celluloid/core](https://github.com/celluloid/core).
+* Added `.omnirepo` configuration file support. YAML file with these options:
+  * [required, if using .omnirepo] `repositories` array, of individual repositories.
+  * [optional] `submodules` if `remove` will remove submodules from repositories.
+  * [optional] `merging` can be `skip` and it will just prepare but not merge.

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+gemspec
+

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011-2014 Donovan Keme
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,90 @@
+# omnirepo
+
+Create a new omnibus repository out of a collection of existing GitHub repositories.
+
+
+## Installation:
+
+```
+gem install omnirepo
+```
+
+## What exactly does it do with repositories?!
+
+0. Checks for the destination omnibus directory.
+0. Clones source repositories into a temporary directory.
+  0. It uses mirroring, to pull in all branches and tags.
+  0. It immediately removes any  remote `origin` detected.
+0. Preserves the git history for each repository.
+  0. (optional) If desired, it removes any submodules present first.
+  0. It also renames all branches and tags to be `<repository>/<name>`.
+0. Does garbage collection on the repository being migrated.
+0. Imports each source repository as a sub-directory of the omnibus repository.
+0. Does garbage collection on the new omnibus repository.
+0. **Does not push the new omnibus repository live.**
+
+### System Requirements:
+
+* Unix-based operating system, with `ruby` and `git` preinstalled.
+* Prefers `tmpfs` partition located at `/dev/shm` for temporary storage.
+* Read access to all remote repositories involed.
+
+### Usage:
+
+```sh
+omnirepo <username/organization> <destination-omnibus> <source-repository> [...]
+```
+
+### Example:
+
+If these self-contained repositories on GitHub need to be merged together into one repository:
+
+* `org9/repoA`
+* `org9/repoB`
+* `org9/repoC`
+
+And the directory housing the future location of the omnibus repository on GitHub is:
+
+* `org9/repo0`
+
+This would be your command:
+
+```sh
+omnirepo org9 repo0 repoA repoB repoC
+```
+# Use with a configuration file:
+
+For the scenario above, you can create a configuration file as follows:
+
+```yml
+repositories:
+  - repoA
+  - repoB
+  - repoC
+```
+
+Save that file as `repo0/.omnirepo`
+
+Then you can run this command:
+
+```sh
+omnirepo org9 repo0
+```
+
+
+## Future Features
+
+- [x] Read repositories from a configuration file.
+- [ ] Bring together source repositories from multiple possible organizations.
+- [ ] ...missing something? [Request it](https://github.com/digitalextremist/omnirepo/issues/new)...
+
+## Contributing
+
+* Fork this repository on GitHub.
+* Make your changes and send a pull request.
+
+## License
+
+Copyright (c) 2015 Donovan Keme.
+
+Distributed under the MIT License. See [LICENSE.txt](https://github.com/digitalextremist/omnirepo/LICENSE.txt) for further details.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/omnirepo

@@ -0,0 +1,253 @@
+#!/usr/bin/env ruby
+
+start = Time.now #de Not monotonic, but bah.
+puts "Omnibus repository unification tool..."
+
+def omnirepo?
+  if File.exists?(File.join("#{Dir.pwd}/#{ARGV[1]}", ".omnirepo"))
+    require 'psych'
+    return true
+  end
+  false
+end
+
+def urandom_id
+  `cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1`
+end
+
+def success?(results)
+  return false unless results.is_a?(Array)
+  results.uniq!
+  return false unless results.one?
+  results.first == true
+end
+
+begin
+
+  #de TODO: Validate owner.
+  #de TODO: Allow use of literal paths, vs. relative.
+  #de TODO: Check permissions/writeability, etc.
+
+  if ARGV.length == 2 && omnirepo?
+    OWNER = ARGV.shift
+    OMNIBUS = "#{Dir.pwd}/#{ARGV.shift}"
+    CONFIG = Psych.load_file(File.join(OMNIBUS, ".omnirepo"))
+    unless CONFIG.is_a?(Hash) && CONFIG["repositories"].is_a?(Array)
+      puts "Onmirepo configuration file invalid."
+      exit!
+    end
+    REPOSITORIES = CONFIG['repositories'].uniq
+    GIT = CONFIG["git"] if CONFIG["git"].is_a?(String)
+  elsif ARGV.length <= 3
+    puts "Usage: omnirepo <owner org/user> <destination-omnibus-dir> <source-repo1> <source-repo2> [...]"
+    puts "No owner organization/username supplied." unless ARGV.any?
+    puts "No destination omnibus directory supplied." unless ARGV.length >= 2
+    puts "No source repositories supplied." unless ARGV.length == 3
+    puts "Only one source repository supplied." if ARGV.length == 3
+    exit!
+  else
+    OWNER = ARGV.shift
+    OMNIBUS = "#{Dir.pwd}/#{ARGV.shift}"
+    REPOSITORIES = ARGV.uniq
+    CONFIG = {}
+  end
+
+  GIT = "git@github.com".freeze unless defined? GIT
+
+  tmp='/dev/shm'
+  tmp='/tmp' unless Dir.exists?(tmp)
+  TMP="#{tmp}/omnirepo.#{urandom_id}".chomp.strip
+
+  unless Dir.mkdir(TMP)
+    STDERR.puts "ERROR: Unable to make the temporary working directory: #{TMP}"
+    exit!
+  end
+
+  puts LINE = "-----------------------------------------------------------------------------------".freeze
+
+  unless Dir.exists?(OMNIBUS)
+    puts "Destination omnibus directory does not exist: #{OMNIBUS}"
+    exit!
+  end
+
+  puts "Owner organization/user: #{OWNER}"
+  puts "Destination omnibus: #{OMNIBUS}"
+  puts "Working directory: #{TMP}"
+  puts LINE
+
+  Dir.chdir(OMNIBUS)
+
+  if `git show 2> /dev/null | grep "commit" | wc -l`.to_i != 1
+    puts LINE
+    STDERR.puts "ERROR: Destination omnibus directory is not a git repository, or not ready."
+    exit!
+  end
+
+  #de TODO: Further validate omnibus repository.
+
+  failures = []
+
+  REPOSITORIES.each { |repo|
+    
+    puts "\n\n\n#{LINE}"
+    puts "Adding the #{repo} repository..."
+    puts "Source: #{GIT}:#{OWNER}/#{repo}.git"
+    puts LINE
+
+    unless Dir.chdir(TMP)
+      STDERR.puts "ERROR: Failure to change to the temporary working directory."
+      exit!
+    end
+
+    unless system("git clone --mirror '#{GIT}:#{OWNER}/#{repo}.git' #{repo}")
+      puts "FERROR: ailed to clone: #{repo}"
+      exit!
+    end
+
+    unless Dir.chdir(repo_bare = File.join(TMP, repo))
+      STDERR.puts "ERROR: Failed to mirror the #{repo} repository."
+      exit!
+    end
+
+    if `git remote | grep origin | wc -l`.to_i > 0
+      `git remote rm origin > /dev/null 2> /dev/null`
+    end
+
+    branches = []
+    `git branch | grep -v master | grep -v "refs/"`.split("\n").each { |branch|
+      begin
+        result = nil
+        branches << (branch = branch.strip.chomp)
+        result = system("git branch -m '#{branch}' '#{repo}/#{branch}")
+      rescue => ex
+        STDERR.puts "WARN: Could not migrate the #{branch} branch.\nReason: #{ex}"
+      ensure
+        failures << "#{repo} / branch: #{branch}" unless result == true
+      end
+    }
+
+    repo_clone = "../#{repo}.clone"
+    unless cloned = system("git clone . #{repo_clone} > /dev/null")
+      STDERR.puts "ERROR: Failed to make local clone of #{repo}."
+      failures << "#{repo} / locally clone"
+      next
+    end
+    
+    Dir.chdir(repo_clone)
+
+    begin
+      if CONFIG['submodules'] == 'remove'
+        if File.exists?(module_config = File.join(repo_bare, repo_clone, ".gitmodules"))
+          submodules = File.read(module_config)
+                        .split("\n")
+                        .map { |line|
+                          if line.include?("path =")
+                            submodule = line.sub("path =", "").chomp.strip
+                            unless submodule.empty?
+                              `git rm -rf ./#{submodule}`
+                               submodule
+                            end
+                          else
+                            nil
+                          end
+                        }.compact
+          result = []
+          result << system("git rm -rf .gitmodules")
+          result << system("git commit -am '[omnirepo] removing submodules: #{submodules.join(', ')}'")
+          result << system("git push")
+          failures << "#{repo} / submodules: remove .gitmodules, then commit/push" unless success?(result)
+          puts "#{LINE}\nRemoved submodules first: #{submodules.join(', ')}"
+        end
+      end
+    rescue => ex
+      STDERR.puts "WARN: Could not remove associated submodules.\nReason: #{ex}"
+      failures << "#{repo} / remove submodules"
+    ensure
+      Dir.chdir(repo_bare)
+    end
+
+    puts "#{LINE}\nPreserving change history and files, but making the repository a sub-directory..."
+    print "... please wait: "
+
+    unless system("git filter-branch --tree-filter " +
+            "'mkdir #{repo}; " +
+            "find -maxdepth 1 " +
+                "-not -name . " +
+                "-not -name .git " +
+                "-not -name #{repo} " +
+            "| xargs -I{} mv {} #{repo}' " +
+            "-d #{TMP}/#{urandom_id} -- --all")
+      failures << "#{repo} / repository tree-filter"
+    end
+
+    unless system("git commit -am '[omnirepo] merging the rearranged file structure'")
+      failures << "#{repo} / merge rearranged file structure"
+      STDERR.puts "ERROR: Could not merge-in rearranged file structure."
+      next
+    end
+
+    tags = []
+    `git tag`.split("\n").each { |tag|
+      begin
+        result = []
+        tags << (tag = tag.chomp.strip)
+        result << system("git tag '#{repo}/#{tag}' '#{tag}' > /dev/null")
+        result << system("git tag -d '#{tag}' > /dev/null")
+      rescue => ex
+        STDERR.puts "WARN: Could not migrate the #{tag} tag.\nReason: #{ex}"
+        result << false
+      ensure
+        failures << "#{repo} / tag: #{tag}" unless success?(result)
+      end
+    }
+
+    puts "#{LINE}\nGarbage collection of source repository...\n#{LINE}"
+    failures << "#{repo} / garbage collection" unless `git gc --aggressive`
+
+    if CONFIG['merging'] == 'skip'
+      puts "Skipping actual merge into omnibus..."
+      next
+    end
+
+    puts "#{LINE}\nMerging the prepared repository into the omnibus...\n#{LINE}"
+
+    result = []
+    Dir.chdir(OMNIBUS)
+    result << system("git remote add #{repo} #{repo_bare}")
+    result << system("git fetch #{repo}")
+    result << system("git merge --no-ff #{repo}/master --commit '[omnirepo] merging in the #{repo} repository.'")
+    result << system("git remote rm #{repo}")
+
+    unless success?(result)
+      failures << "#{repo} / merge into omnibus"
+      next
+    end
+
+    if branches.any?
+      puts "#{LINE}\nThese branches were migrated:\n#{LINE}"
+      branches.each { |branch| puts "#{branch.rjust(35)} is now #{repo}/#{branch}\n" }
+    end
+    
+    if tags.any?
+      puts "#{LINE}\nThese tags were migrated:\n#{LINE}"
+      tags.each { |tag| puts "#{tag.rjust(35)} is now #{repo}/#{tag}\n" }
+    end
+
+  }
+
+  puts "\n\n\n"
+  #de TODO: Delete working directory, or at least give config or command line option to do so.
+  puts "#{LINE}\nThere were the following failures:\n#{LINE}\n#{failures.join("\n")}" if failures.any?
+  puts "\n\n\n#{LINE}"
+  puts "Omnibus unification of repositories finished in #{"%0.4f" % (Time.now.to_f - start.to_f)} seconds."
+  puts "#{LINE}\nRemove your temporary working directory when you are ready:"
+  puts "         #{TMP}"
+  puts "#{LINE}\nWARNING: Be sure to check over the new omnibus repository before pushing it!"
+  puts "         Each repository has been committed into it."
+  puts LINE
+
+rescue => ex
+  puts "Exiting prematurely. Re-run to make an intact omnibus repository."
+  STDERR.puts "Error: #{ex}" unless ex.is_a?(Interrupt)
+  exit(false)
+end

data/omnirepo.gemspec

@@ -0,0 +1,18 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Donovan Keme"]
+  gem.email         = ["code@extremist.digital"]
+  gem.description   = "Create a new omnibus repository out of collections of existing GitHub repositories."
+  gem.summary       = "Unify GitHub repositories as one repository containing them all as sub-directories."
+  gem.homepage      = "https://github.com/digitalextremist/omnirepo"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "omnirepo"
+  gem.version       = '0.5.0.0.pre'
+  gem.licenses      = ['MIT']
+  gem.required_ruby_version = ">= 1.9.2"
+  gem.required_rubygems_version = ">= 1.3.6"
+end

metadata

@@ -0,0 +1,54 @@
+--- !ruby/object:Gem::Specification
+name: omnirepo
+version: !ruby/object:Gem::Version
+  version: 0.5.0.0.pre
+platform: ruby
+authors:
+- Donovan Keme
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-08-08 00:00:00.000000000 Z
+dependencies: []
+description: Create a new omnibus repository out of collections of existing GitHub
+  repositories.
+email:
+- code@extremist.digital
+executables:
+- omnirepo
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- CHANGES.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/omnirepo
+- omnirepo.gemspec
+homepage: https://github.com/digitalextremist/omnirepo
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.3.6
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Unify GitHub repositories as one repository containing them all as sub-directories.
+test_files: []