git-fastclone 0.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3feed5ccee58d01de0a1703664806f92309c3520
+  data.tar.gz: 9129d253e669fdbe2bd97d6ee11cfe832437b8a2
+SHA512:
+  metadata.gz: ce5daa467bffaf2a6c4b7118f7cb17adea720f130842603a1adb4050beb3110df4f6c9df64811f0edba82fd4a9f77895f7050a89980c120aed8af149c401feac
+  data.tar.gz: ba417ddff24e96b332b445c6dec39fb62c1363d9f5e8ec471ba59bf3ba0c6df0099f0cac7d2773c197d20bb616d19f4d679f278f873f1c9860e8d014db222203

data/bin/git-fastclone

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'git-fastclone'
+
+GitFastClone.new.run

data/lib/execution.rb

@@ -0,0 +1,71 @@
+require 'open3'
+
+# Wrapper around open3.popen2e which fails on error
+#
+# We emulate open3.capture2e with the following changes in behavior:
+# 1) The command is printed to stdout before execution.
+# 2) Attempts to use the shell implicitly are blocked.
+# 3) Nonzero return codes result in the process exiting.
+#
+# If you're looking for more process/stream control read the spawn documentation, and pass
+# options directly here
+def fail_on_error (*cmd, **opts)
+#  puts "Running Command: \n#{debug_print_cmd_list([cmd])}\n"
+  shell_safe(cmd)
+  output, status = Open3.capture2(*cmd, opts)
+  exit_on_status(output, status)
+end
+
+# Look at a cmd list intended for spawn.
+# determine if spawn will call the shell implicitly, fail in that case.
+def shell_safe (cmd)
+  # env and opts in the command spec both aren't of type string.
+  # If you're only passing one string, spawn is going to launch a shell.
+  if cmd.select{ |element| element.class == String }.length == 1
+    puts "You tried to use sqiosbuild to call the shell implicitly. Please don't."
+    puts "Think of the children."
+    puts "Think of shellshock."
+    puts "Please don't. Not ever."
+    exit 1
+  end
+end
+
+def debug_print_cmd_list(cmd_list)
+  # Take a list of command argument lists like you'd sent to open3.pipeline or fail_on_error_pipe and
+  # print out a string that would do the same thing when entered at the shell.
+  #
+  # This is a converter from our internal representation of commands to a subset of bash that
+  # can be executed directly.
+  #
+  # Note this has problems if you specify env or opts
+  # TODO: make this remove those command parts
+  "\"" +
+  cmd_list.map { |cmd|
+    cmd.map { |arg|
+      arg.gsub("\"", "\\\"") # Escape all double quotes in command arguments
+    }.join("\" \"") # Fully quote all command parts. We add quotes to the beginning and end too.
+  }.join("\" | \"") + # Pipe commands to one another.
+  "\""
+end
+
+# If any of the statuses are bad, exits with the
+# return code of the first one.
+#
+# Otherwise returns first argument (output)
+def exit_on_status (output, status)
+  # Do nothing for proper statuses
+  if status.exited? && status.exitstatus == 0
+    return output
+  end
+
+  # If we exited nonzero or abnormally, print debugging info
+  # and explode.
+  if status.exited?
+    puts "Return code was #{status.exitstatus}"
+    exit status.exitstatus
+  end
+  puts "This might be helpful:\nProcessStatus: #{status.inspect}\nRaw POSIX Status: #{status.to_i}\n"
+  exit 1
+end
+
+

data/lib/git-fastclone.rb

@@ -0,0 +1,181 @@
+require 'optparse'
+require 'fileutils'
+require_relative 'execution'
+
+class GitFastClone
+  def initialize()
+    # Prefetch reference repos for submodules we've seen before
+    # Keep our own reference accounting of module dependencies.
+    @prefetch_submodules = true
+
+    # Thread-level locking for reference repos
+    # TODO: Add flock-based locking if we want to do more than one build on a given slave
+    @reference_mutex = Hash.new { |hash, key| hash[key] = Mutex.new() }
+
+    # Only update each reference repo once per run.
+    # TODO: May want to update this if we're doing more than one build on a given slave.
+    #       Perhaps a last-updated-time and a timeout per reference repo.
+    @reference_updated = Hash.new { |hash, key| hash[key] = false }
+  end
+
+  def run()
+    @reference_dir = ENV['REFERENCE_REPO_DIR'] || "/var/tmp/git-fastclone/reference"
+
+    FileUtils.mkdir_p(@reference_dir)
+
+    # One option --branch=<branch>  We're not as brittle as clone. That branch can be a sha or tag and we're still okay.
+    @options = {}
+    OptionParser.new do |opts|
+      @options[:branch] = nil
+      opts.on("-b", "--branch BRANCH", "Checkout this branch rather than the default") do |branch|
+        @options[:branch] = branch
+      end
+      # TODO: add --verbose option that turns on and off printing of sub-commands
+      # TODO: Add help text.
+    end.parse!
+
+    puts ARGV
+
+    # Remaining two positional args are url and optional path
+    url = ARGV[0]
+    path = ARGV[1] || path_from_git_url(url)
+
+    # Do a checkout with reference repositories for main and submodules
+    clone(url, @options[:branch], File.join(Dir.pwd, path))
+  end
+
+  def path_from_git_url(url)
+    # Get the checkout path from tail-end of the url.
+    File.join(Dir.pwd, url.match(/([^\/]*)\.git$/)[1])
+  end
+
+  # Checkout to SOURCE_DIR. Update all submodules recursively. Use reference repos everywhere for speed.
+  def clone(url, rev, src_dir)
+    initial_time = Time.now()
+
+    with_git_mirror(url) do |mirror|
+      fail_on_error("git", "clone", "--reference", mirror, url, src_dir)
+    end
+
+    # Only checkout if we're changing branches to a non-default branch
+    unless rev.nil? then
+      fail_on_error("git", "checkout", rev, :chdir=>src_dir)
+    end
+
+    update_submodules(src_dir, url)
+
+    final_time = Time.now()
+    puts "Checkout of #{url} took #{final_time-initial_time}s"
+  end
+
+  # Update all submodules in current directory recursively
+  # Use a reference repository for speed.
+  # Use a separate thread for each submodule.
+  def update_submodules (pwd, url)
+    # Skip if there's no submodules defined
+    if File.exist?(File.join(pwd,".gitmodules")) then
+
+      # Update each submodule on a different thread.
+      threads = []
+      submodule_url_list = []
+
+      # Init outputs all the info we need to run the update commands.
+      # Parse its output directly to save time.
+      fail_on_error("git", "submodule", "init", :chdir=>pwd).split("\n").each do |line|
+        # Submodule path (not name) is in between single quotes '' at the end of the line
+        submodule_path = File.join(pwd, line.strip.match(/'([^']*)'$/)[1])
+        # URL is in between parentheses ()
+        submodule_url = line.strip.match(/\(([^)]*)\)/)[1]
+        submodule_url_list << submodule_url
+
+        # Each update happens on a separate thread for speed.
+        threads << Thread.new do
+          with_git_mirror(submodule_url) do |mirror|
+            fail_on_error("git", "submodule", "update", "--reference", mirror, submodule_path, :chdir=>pwd)
+          end
+          # Recurse into the submodule directory
+          update_submodules(submodule_path, submodule_url)
+        end
+      end
+      update_submodule_reference(url, submodule_url_list)
+      threads.each {|t| t.join}
+    end
+  end
+
+  def reference_repo_name(url)
+    # Derive a unique directory name from the git url.
+    url.gsub(/^.*:\/\//, "").gsub(/^[^@]*@/, "").gsub("/","-").gsub(":","-")
+  end
+
+  def reference_repo_dir(url)
+    File.join(@reference_dir, reference_repo_name(url))
+  end
+
+  def reference_repo_submodule_file(url)
+    # ':' is never a valid char in a reference repo dir, so this
+    # uniquely maps to a particular reference repo.
+    "#{reference_repo_dir(url)}:submodules.txt"
+  end
+
+  def with_reference_repo_lock(url)
+    @reference_mutex[reference_repo_name(url)].synchronize do
+      yield
+    end
+  end
+
+  def update_submodule_reference(url, submodule_url_list)
+    if submodule_url_list != [] and @prefetch_submodules then
+      with_reference_repo_lock(url) do
+
+        # Write the dependency file using submodule list
+        File.open(reference_repo_submodule_file(url), 'w') do |f|
+          submodule_url_list.each do |submodule_url|
+            f.write("#{submodule_url}\n")
+          end
+        end
+
+      end
+    end
+  end
+
+  def update_reference_repo(url)
+    repo_name = reference_repo_name(url)
+    mirror = reference_repo_dir(url)
+
+    with_reference_repo_lock(url) do
+      submodule_file = reference_repo_submodule_file(url)
+      if File.exist?(submodule_file) and @prefetch_submodules then
+        File.readlines(submodule_file).each do |line|
+          # We don't join these threads explicitly
+          Thread.new { update_reference_repo(line.strip) }
+        end
+      end
+
+      if !@reference_updated[repo_name] then
+        if !Dir.exist?(mirror)
+          fail_on_error("git", "clone", "--mirror", url, mirror)
+        end
+        fail_on_error("git", "remote", "update", :chdir=> mirror)
+        @reference_updated[repo_name] = true
+      end
+    end
+  end
+
+  # Executes a block passing in the directory of an up-to-date local git mirror
+  # for the given url. This will speed up most git commands that ask for data
+  # over the network after the mirror is cloned initially.
+  #
+  # This command will create and bring the mirror up-to-date on-demand,
+  # blocking any code passed in while the mirror is brought up-to-date
+  #
+  # In future we may need to synchronize with flock here if we run multiple builds
+  # at once against the same reference repos. One build per slave at the moment means
+  # we only need to synchronize our own threads in case a single submodule url is
+  # included twice via multiple dependency paths
+  def with_git_mirror(url)
+    update_reference_repo(url)
+
+    # May want to lock the reference repo for this, but don't need to for how we use this.
+    yield reference_repo_dir(url)
+  end
+end

metadata

@@ -0,0 +1,48 @@
+--- !ruby/object:Gem::Specification
+name: git-fastclone
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+platform: ruby
+authors:
+- Michael Tauraso
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-11-19 00:00:00.000000000 Z
+dependencies: []
+description: A git command that uses reference repositories and multithreading to
+  quickly and recursively clone repositories with many nested submodules
+email: mtauraso@gmail.com
+executables:
+- git-fastclone
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/git-fastclone
+- lib/execution.rb
+- lib/git-fastclone.rb
+homepage: https://rubygems.org/gems/git-fastclone
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: git-clone --recursive on steroids!
+test_files: []