git-fastclone 0.0.3 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 153c297d291e6ad8edcb66333438e70a4babfcfe
-  data.tar.gz: a3df048e6e1ab5e900e341513cb20248d996896d
+  metadata.gz: 4f2b4822387b6fa1c847455e8c09cf2bd48b711b
+  data.tar.gz: c30eba26437f7f4f4160af9ce0bf8e9e662d36b6
 SHA512:
-  metadata.gz: 1f31a2cdd9cb886b3af613a639dc5992eb5653c69b116e506e701053a037a0e25223cee0a6d7241220d9138c71b4351db54afc91215b5dce3ad3a9f364bb1acf
-  data.tar.gz: bf0c7a716e84db621438ee3b5ef58df7f46349a92c7a1d875d8aa3b32b5f0e2239c63220d3df8bed84b8ba03d7c2b26c2882496effc71006315be0e5ae8673aa
+  metadata.gz: ac6a82830270df7b8d1918905883cd9c8f9845f653b3a0886c3ca701f19fd84c5773e1179db7db0c3a83ab4469fe8abd013573fe19adf6babf9be28e10b2363a
+  data.tar.gz: fdf2fec80e7b8e7a13aef313c0c231159a363c49f6ef6dd25f954b4e1ca8fced18512bd6cac66f624c331e98a1548caa8043e9b9addd841f5f4312bd9f1e9dd2

data/LICENSE.txt

@@ -0,0 +1,177 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS

data/README.md

@@ -0,0 +1,104 @@
+git-fastclone
+=============
+[![Build Status](https://travis-ci.org/square/git-fastclone.svg?branch=master)](https://travis-ci.org/square/git-fastclone)
+
+git-fastclone is git clone --recursive on steroids.
+
+
+Why fastclone?
+--------------
+Doing lots of repeated checkouts on a specific machine?
+
+| Repository | 1st Fastclone | 2nd Fastclone | git clone | cp -R |
+| -----------|---------------|---------------|-----------|-------|
+| angular.js |    8s         |     3s        | 6s        | 0.5s  |
+| bootstrap  |    26s        |     3s        | 11s       | 0.2s  |
+| gradle     |    25s        |     9s        | 19s       | 6.2s  |
+| linux      |    4m 53s     |     1m 6s     | 3m 51s    | 29s   |
+| react.js   |    18s        |     3s        | 8s        | 0.5s  |
+| tensorflow |    19s        |     4s        | 8s        | 1.5s  |
+
+Above times captured using `time` without verbose mode.
+
+
+What does it do?
+----------------
+It creates a reference repo with `git clone --mirror` in `/var/tmp/git-fastclone/reference` for each
+repository and git submodule linked in the main repo. You can control where it puts these by
+changing the `REFERENCE_REPO_DIR` environment variable.
+
+It aggressively updates these mirrors from origin and then clones from the mirrors into the
+directory of your choosing. It always works recursively and multithreaded to get your checkout up as
+fast as possible.
+
+
+Usage
+-----
+    gem install git-fastclone
+    git fastclone [options] <git-repo-url>
+
+    -b, --branch <branch>   Clone a specific branch
+    -v, --verbose           Shows more info
+
+Change the default `REFERENCE_REPO_DIR` environment variable if necessary.
+
+Cygwin users need to add `~/bin` to PATH.
+
+
+How to test?
+------------
+Manual testing:
+
+    ruby -Ilib bin/git-fastclone <git url>
+
+Compatible with Travis and Kochiku.
+
+
+Contributing
+------------
+If you would like to contribute to git-fastclone, you can fork the repository and send us pull
+requests.
+
+When submitting code, please make every effort to follow existing conventions and style in order to
+keep the code as readable as possible.
+
+Before accepting any pull requests, we need you to sign an [Individual Contributor Agreement][1]
+(Google form).
+
+
+Acknowledgements
+----------------
+[thoughtbot/cocaine][2] - jyurek and collaborators
+
+[robolson][3]
+
+[ianchesal][4]
+
+[mtauraso][5]
+
+[chriseckhardt][6]
+
+
+License
+-------
+    Copyright 2015 Square Inc.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+
+[1]: https://docs.google.com/a/squareup.com/forms/d/13WR8m5uZ2nAkJH41k7GdVBXAAbzDk00vxtEYjd6Imzg/viewform?formkey=dDViT2xzUHAwRkI3X3k5Z0lQM091OGc6MQ&ndplr=1
+[2]: https://github.com/thoughtbot/cocaine
+[3]: https://github.com/robolson
+[4]: https://github.com/ianchesal
+[5]: https://github.com/mtauraso
+[6]: https://github.com/chriseckhardt

data/Rakefile

@@ -0,0 +1,11 @@
+# encoding: utf-8
+
+require 'bundler/setup'
+
+task default: %w(spec rubocop)
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new
+
+require 'rubocop/rake_task'
+RuboCop::RakeTask.new

data/bin/git-fastclone

@@ -1,5 +1,21 @@
 #!/usr/bin/env ruby
 
+# Copyright 2015 Square Inc.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+
 require 'git-fastclone'
 
-GitFastClone.new.run
+GitFastClone::Runner.new.run

data/lib/git-fastclone.rb

@@ -1,184 +1,261 @@
+# Copyright 2015 Square Inc.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 require 'optparse'
 require 'fileutils'
-require_relative 'execution'
-
-class GitFastClone
-  def initialize()
-    # Prefetch reference repos for submodules we've seen before
-    # Keep our own reference accounting of module dependencies.
-    @prefetch_submodules = true
-
-    # Thread-level locking for reference repos
-    # TODO: Add flock-based locking if we want to avoid conflicting with ourselves.
-    @reference_mutex = Hash.new { |hash, key| hash[key] = Mutex.new() }
-
-    # Only update each reference repo once per run.
-    # TODO: May want to update this so we don't duplicate work with other copies of ourself
-    #       Perhaps a last-updated-time and a timeout per reference repo.
-    @reference_updated = Hash.new { |hash, key| hash[key] = false }
-  end
+require 'logger'
+require 'cocaine'
 
-  def run()
-    @reference_dir = ENV['REFERENCE_REPO_DIR'] || "/var/tmp/git-fastclone/reference"
+# Contains helper module UrlHelper and execution class GitFastClone::Runner
+module GitFastClone
+  # Helper methods for fastclone url operations
+  module UrlHelper
+    def path_from_git_url(url)
+      File.basename(url, '.git')
+    end
+    module_function :path_from_git_url
+
+    def parse_update_info(line)
+      [line.strip.match(/'([^']*)'$/)[1], line.strip.match(/\(([^)]*)\)/)[1]]
+    end
+    module_function :parse_update_info
 
-    FileUtils.mkdir_p(@reference_dir)
+    def reference_repo_name(url)
+      "#{url.gsub(%r{^.*://}, '').gsub(/^[^@]*@/, '').tr('/', '-').tr(':', '-')}"
+    end
+    module_function :reference_repo_name
 
-    # One option --branch=<branch>  We're not as brittle as clone. That branch can be a sha or tag and we're still okay.
-    @options = {}
-    OptionParser.new do |opts|
-      opts.banner = "Usage: git fastclone [options] <git-url> [path]"
-      @options[:branch] = nil
-      opts.on("-b", "--branch BRANCH", "Checkout this branch rather than the default") do |branch|
-        @options[:branch] = branch
+    def reference_repo_dir(url, reference_dir, using_local_repo)
+      if using_local_repo
+        File.join(reference_dir, 'local' + reference_repo_name(url))
+      else
+        File.join(reference_dir, reference_repo_name(url))
       end
-    end.parse!
+    end
+    module_function :reference_repo_dir
 
-    # Remaining two positional args are url and optional path
-    url = ARGV[0]
-    path = ARGV[1] || path_from_git_url(url)
+    def reference_repo_submodule_file(url, reference_dir, using_local_repo)
+      "#{reference_repo_dir(url, reference_dir, using_local_repo)}:submodules.txt"
+    end
+    module_function :reference_repo_submodule_file
+  end
 
-    puts "Cloning #{url} to #{path}"
+  # Spawns one thread per submodule, and updates them in parallel. They will be
+  # cached in the reference directory (see DEFAULT_REFERENCE_REPO_DIR), and their
+  # index will be incrementally updated. This prevents a large amount of data
+  # copying.
+  class Runner
+    include GitFastClone::UrlHelper
 
-    # Do a checkout with reference repositories for main and submodules
-    clone(url, @options[:branch], path)
-  end
+    DEFAULT_REFERENCE_REPO_DIR = '/var/tmp/git-fastclone/reference'
 
-  def path_from_git_url(url)
-    # Get the checkout path from tail-end of the url.
-    url.match(/([^\/]*)\.git$/)[1]
-  end
+    attr_accessor :reference_dir, :prefetch_submodules, :reference_mutex, :reference_updated,
+                  :options, :logger, :abs_clone_path, :using_local_repo
+
+    def initialize
+      # Prefetch reference repos for submodules we've seen before
+      # Keep our own reference accounting of module dependencies.
+      self.prefetch_submodules = true
+
+      # Thread-level locking for reference repos
+      # TODO: Add flock-based locking if we want to avoid conflicting with
+      # ourselves.
+      self.reference_mutex = Hash.new { |hash, key| hash[key] = Mutex.new }
+
+      # Only update each reference repo once per run.
+      # TODO: May want to update this so we don't duplicate work with other copies
+      # of ourself. Perhaps a last-updated-time and a timeout per reference repo.
+      self.reference_updated = Hash.new { |hash, key| hash[key] = false }
+
+      self.options = {}
 
-  # Checkout to SOURCE_DIR. Update all submodules recursively. Use reference repos everywhere for speed.
-  def clone(url, rev, src_dir)
-    initial_time = Time.now()
+      self.logger = nil # Only set in verbose mode
 
-    with_git_mirror(url) do |mirror|
-      fail_on_error("git", "clone", "--quiet", "--reference", mirror, url, src_dir)
+      self.abs_clone_path = Dir.pwd
+
+      self.using_local_repo = false
     end
 
-    # Only checkout if we're changing branches to a non-default branch
-    unless rev.nil? then
-      fail_on_error("git", "checkout", "--quiet", rev, :chdir=>src_dir)
+    def run
+      url, path, options = parse_inputs
+      logger.info("Cloning #{url} to #{path}") if logger
+      clone(url, options[:branch], path)
     end
 
-    update_submodules(src_dir, url)
+    def parse_inputs
+      usage = 'Usage: git fastclone [options] <git-url> [path]'
 
-    final_time = Time.now()
-    puts "Checkout of #{url} took #{final_time-initial_time}s"
-  end
+      # One option --branch=<branch>  We're not as brittle as clone. That branch
+      # can be a sha or tag and we're still okay.
+      OptionParser.new do |opts|
+        opts.banner = usage
+        options[:branch] = nil
+        opts.on('-b', '--branch BRANCH', 'Checkout this branch rather than the default') do |branch|
+          options[:branch] = branch
+        end
+        opts.on('-v', '--verbose', 'Verbose mode') do
+          self.logger = Logger.new(STDOUT)
+          Cocaine::CommandLine.logger = logger
+        end
+      end.parse!
+
+      fail usage unless ARGV[0]
+
+      if Dir.exist?(ARGV[0])
+        url = File.expand_path ARGV[0]
+        self.using_local_repo = true
+      else
+        url = ARGV[0]
+      end
+
+      path = ARGV[1] || path_from_git_url(url)
+
+      fail "Clone destination #{path} already exists!" if Dir.exist?(path)
+
+      self.reference_dir = ENV['REFERENCE_REPO_DIR'] || DEFAULT_REFERENCE_REPO_DIR
+      FileUtils.mkdir_p(reference_dir)
+
+      [url, path, options]
+    end
+
+    # Checkout to SOURCE_DIR. Update all submodules recursively. Use reference
+    # repos everywhere for speed.
+    def clone(url, rev, src_dir)
+      initial_time = Time.now
+
+      with_git_mirror(url) do |mirror|
+        Cocaine::CommandLine.new("git clone --quiet --reference '#{mirror}' '#{url}'" \
+          " '#{File.join(abs_clone_path, src_dir)}'").run
+      end
+
+      # Only checkout if we're changing branches to a non-default branch
+      Dir.chdir(src_dir) { Cocaine::CommandLine.new("git checkout --quiet '#{rev}'").run } if rev
+
+      update_submodules(src_dir, url)
 
-  # Update all submodules in current directory recursively
-  # Use a reference repository for speed.
-  # Use a separate thread for each submodule.
-  def update_submodules (pwd, url)
-    # Skip if there's no submodules defined
-    if File.exist?(File.join(pwd,".gitmodules")) then
+      final_time = Time.now
+      logger.info("Checkout of #{url} took #{final_time - initial_time}s") if logger
+    end
+
+    def update_submodules(pwd, url)
+      return unless File.exist?(File.join(abs_clone_path, pwd, '.gitmodules'))
+
+      logger.info('Updating submodules') if logger
 
-      # Update each submodule on a different thread.
       threads = []
       submodule_url_list = []
 
-      # Init outputs all the info we need to run the update commands.
-      # Parse its output directly to save time.
-      fail_on_error("git", "submodule", "init", :chdir=>pwd).split("\n").each do |line|
-        # Submodule path (not name) is in between single quotes '' at the end of the line
-        submodule_path = line.strip.match(/'([^']*)'$/)[1]
-        # URL is in between parentheses ()
-        submodule_url = line.strip.match(/\(([^)]*)\)/)[1]
+      Cocaine::CommandLine.new("cd '#{File.join(abs_clone_path, pwd)}'; git submodule init").run
+        .split("\n").each do |line|
+        submodule_path, submodule_url = parse_update_info(line)
         submodule_url_list << submodule_url
 
-        # Each update happens on a separate thread for speed.
-        threads << Thread.new do
-          with_git_mirror(submodule_url) do |mirror|
-            fail_on_error("git", "submodule", "update", "--quiet", "--reference", mirror, submodule_path, :chdir=>pwd)
-          end
-          # Recurse into the submodule directory
-          update_submodules(File.join(pwd,submodule_path), submodule_url)
-        end
+        thread_update_submodule(submodule_url, submodule_path, threads, pwd)
       end
+
       update_submodule_reference(url, submodule_url_list)
-      threads.each {|t| t.join}
+      threads.each(&:join)
     end
-  end
-
-  def reference_repo_name(url)
-    # Derive a unique directory name from the git url.
-    url.gsub(/^.*:\/\//, "").gsub(/^[^@]*@/, "").gsub("/","-").gsub(":","-")
-  end
 
-  def reference_repo_dir(url)
-    File.join(@reference_dir, reference_repo_name(url))
-  end
+    def thread_update_submodule(submodule_url, submodule_path, threads, pwd)
+      threads << Thread.new do
+        with_git_mirror(submodule_url) do |mirror|
+          Cocaine::CommandLine
+            .new("cd '#{File.join(abs_clone_path, pwd)}'; git submodule update --quiet --reference"\
+              " '#{mirror}' '#{submodule_path}'").run
+        end
 
-  def reference_repo_submodule_file(url)
-    # ':' is never a valid char in a reference repo dir, so this
-    # uniquely maps to a particular reference repo.
-    "#{reference_repo_dir(url)}:submodules.txt"
-  end
+        update_submodules(File.join(pwd, submodule_path), submodule_url)
+      end
+    end
 
-  def with_reference_repo_lock(url)
-    @reference_mutex[reference_repo_name(url)].synchronize do
-      yield
+    def with_reference_repo_lock(url)
+      reference_mutex[reference_repo_name(url)].synchronize do
+        yield
+      end
     end
-  end
 
-  def update_submodule_reference(url, submodule_url_list)
-    if submodule_url_list != [] and @prefetch_submodules then
-      with_reference_repo_lock(url) do
+    def update_submodule_reference(url, submodule_url_list)
+      return if submodule_url_list.empty? || prefetch_submodules.nil?
 
+      with_reference_repo_lock(url) do
         # Write the dependency file using submodule list
-        File.open(reference_repo_submodule_file(url), 'w') do |f|
-          submodule_url_list.each do |submodule_url|
-            f.write("#{submodule_url}\n")
-          end
+        File.open(reference_repo_submodule_file(url, reference_dir, using_local_repo), 'w') do |f|
+          submodule_url_list.each { |submodule_url| f.write("#{submodule_url}\n") }
         end
-
       end
     end
-  end
 
-  def update_reference_repo(url)
-    repo_name = reference_repo_name(url)
-    mirror = reference_repo_dir(url)
+    # Fail_hard indicates whether the update is considered a failure of the
+    # overall checkout or not. When we pre-fetch based off of cached information,
+    # fail_hard is false. When we fetch based off info in a repository directly,
+    # fail_hard is true.
+    def update_reference_repo(url, fail_hard)
+      repo_name = reference_repo_name(url)
+      mirror = reference_repo_dir(url, reference_dir, using_local_repo)
 
-    with_reference_repo_lock(url) do
-      submodule_file = reference_repo_submodule_file(url)
-      if File.exist?(submodule_file) and @prefetch_submodules then
-        File.readlines(submodule_file).each do |line|
-          # We don't join these threads explicitly
-          Thread.new { update_reference_repo(line.strip) }
-        end
+      with_reference_repo_lock(url) do
+        # we've created this to track submodules' history
+        submodule_file = reference_repo_submodule_file(url, reference_dir, using_local_repo)
+
+        # if prefetch is on, then grab children immediately to frontload network requests
+        prefetch(submodule_file) if File.exist?(submodule_file) && prefetch_submodules
+
+        # Store the fact that our repo has been updated if necessary
+        store_updated_repo(url, mirror, repo_name, fail_hard) unless reference_updated[repo_name]
       end
+    end
 
-      if !@reference_updated[repo_name] then
-        if !Dir.exist?(mirror)
-          fail_on_error("git", "clone", "--mirror", url, mirror)
-        end
-        fail_on_error("git", "remote", "update", :chdir=> mirror)
-        @reference_updated[repo_name] = true
+    # Grab the children in the event of a prefetch
+    def prefetch(submodule_file)
+      File.readlines(submodule_file).each do |line|
+        # We don't join these threads explicitly
+        Thread.new { update_reference_repo(line.strip, false) }
       end
     end
-  end
 
-  # Executes a block passing in the directory of an up-to-date local git mirror
-  # for the given url. This will speed up most git commands that ask for data
-  # over the network after the mirror is cloned initially.
-  #
-  # This command will create and bring the mirror up-to-date on-demand,
-  # blocking any code passed in while the mirror is brought up-to-date
-  #
-  # In future we may need to synchronize with flock here if we run multiple builds
-  # at once against the same reference repos. One build per slave at the moment means
-  # we only need to synchronize our own threads in case a single submodule url is
-  # included twice via multiple dependency paths
-  def with_git_mirror(url)
-    update_reference_repo(url)
-
-    # Sometimes remote updates involve re-packing objects on a different thread
-    # We grab the reference repo lock here just to make sure whatever thread
-    # ended up doing the update is done with its housekeeping.
-    with_reference_repo_lock(url) do
-      yield reference_repo_dir(url)
+    # Stores the fact that our repo has been updated
+    def store_updated_repo(url, mirror, repo_name, fail_hard)
+      unless Dir.exist?(mirror)
+        Cocaine::CommandLine.new("git clone --mirror '#{url}' '#{mirror}'").run
+      end
+
+      Cocaine::CommandLine.new("cd '#{mirror}'; git remote update --prune").run
+
+      reference_updated[repo_name] = true
+
+    rescue Cocaine::ExitStatusError => e
+      raise e if fail_hard
+    end
+
+    # This command will create and bring the mirror up-to-date on-demand,
+    # blocking any code passed in while the mirror is brought up-to-date
+    #
+    # In future we may need to synchronize with flock here if we run multiple
+    # builds at once against the same reference repos. One build per slave at the
+    # moment means we only need to synchronize our own threads in case a single
+    # submodule url is included twice via multiple dependency paths
+    def with_git_mirror(url)
+      update_reference_repo(url, true)
+
+      # Sometimes remote updates involve re-packing objects on a different thread
+      # We grab the reference repo lock here just to make sure whatever thread
+      # ended up doing the update is done with its housekeeping.
+      # This makes sure we have control and unlock when the block returns:
+      with_reference_repo_lock(url) do
+        yield reference_repo_dir(url, reference_dir, using_local_repo)
+      end
     end
   end
 end

data/lib/git-fastclone/version.rb

@@ -0,0 +1,3 @@
+module GitFastCloneVersion
+  VERSION = '1.0.0'
+end

data/spec/git_fastclone_runner_spec.rb

@@ -0,0 +1,249 @@
+# Copyright 2015 Square Inc.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'spec_helper'
+require 'git-fastclone'
+
+describe GitFastClone::Runner do
+  let(:test_url_valid) { 'ssh://git@git.com/git-fastclone.git' }
+  let(:test_url_invalid) { 'ssh://git@git.com/git-fastclone' }
+  let(:test_reference_dir) { 'test_reference_dir' }
+  let(:test_reference_repo_dir) { '/var/tmp/git-fastclone/reference/test_reference_dir' }
+  let(:placeholder_arg) { 'PH' }
+
+  # Modified ARGV, watch out
+  ARGV = ['ssh://git@git.com/git-fastclone.git', 'test_reference_dir']
+
+  let(:yielded) { [] }
+
+  describe '.initialize' do
+    it 'should initialize properly' do
+      stub_const('GitFastClone::DEFAULT_REFERENCE_REPO_DIR', 'new_dir')
+
+      expect(Hash).to respond_to(:new).with(2).arguments
+      expect(GitFastClone::DEFAULT_REFERENCE_REPO_DIR).to eq('new_dir')
+      expect(subject.prefetch_submodules).to eq(true)
+      expect(subject.reference_mutex).to eq({})
+      expect(subject.reference_updated).to eq({})
+      expect(subject.options).to eq({})
+      expect(subject.logger).to eq(nil)
+    end
+  end
+
+  describe '.run' do
+    let(:options) { {:branch => placeholder_arg} }
+
+    it 'should run with the correct args' do
+      allow(subject).to receive(:parse_inputs) { [placeholder_arg, placeholder_arg, options] }
+      expect(subject).to receive(:clone).with(placeholder_arg, placeholder_arg, placeholder_arg)
+
+      subject.run
+    end
+  end
+
+  describe '.parse_inputs' do
+    it 'should print the proper inputs' do
+      subject.reference_dir = test_reference_dir
+      subject.options = {}
+      allow(FileUtils).to receive(:mkdir_p) {}
+
+      expect(subject.parse_inputs).to eq([test_url_valid, test_reference_dir, {:branch=>nil}])
+    end
+  end
+
+  describe '.clone' do
+    it 'should clone correctly' do
+      cocaine_commandline_double = double('new_cocaine_commandline')
+      allow(subject).to receive(:with_git_mirror) {}
+      allow(cocaine_commandline_double).to receive(:run) {}
+      allow(Cocaine::CommandLine).to receive(:new) { cocaine_commandline_double }
+
+      expect(Time).to receive(:now).twice
+      expect(Cocaine::CommandLine).to receive(:new)
+      expect(cocaine_commandline_double).to receive(:run)
+
+      subject.clone(placeholder_arg, placeholder_arg, '.')
+    end
+  end
+
+  describe '.update_submodules' do
+    it 'should return if no submodules' do
+      subject.update_submodules(placeholder_arg, placeholder_arg)
+      allow(File).to receive(:exist?) { false }
+
+      expect(Thread).not_to receive(:new)
+    end
+
+    it 'should correctly update submodules' do
+      expect(subject).to receive(:update_submodule_reference)
+
+      allow(File).to receive(:exist?) { true }
+      subject.update_submodules('.', placeholder_arg)
+    end
+  end
+
+  describe '.thread_update_submodule' do
+    it 'should update correctly' do
+      pending('need to figure out how to test this')
+      fail
+    end
+  end
+
+  describe '.with_reference_repo_lock' do
+    it 'should acquire a lock' do
+      allow(Mutex).to receive(:synchronize)
+      expect(Mutex).to respond_to(:synchronize)
+
+      subject.with_reference_repo_lock(test_url_valid) do
+        yielded << test_url_valid
+      end
+
+      expect(yielded).to eq([test_url_valid])
+    end
+  end
+
+  describe '.update_submodule_reference' do
+    context 'when we have an empty submodule list' do
+      it 'should return' do
+        expect(subject).not_to receive(:with_reference_repo_lock)
+
+        subject.prefetch_submodules = true
+        subject.update_submodule_reference(placeholder_arg, [])
+      end
+    end
+
+    context 'with a populated submodule list' do
+      it 'should write to a file' do
+        allow(File).to receive(:open) {}
+        allow(File).to receive(:write) {}
+        allow(subject).to receive(:reference_repo_name) {}
+        allow(subject).to receive(:reference_repo_submodule_file) {}
+        expect(File).to receive(:open)
+
+        subject.update_submodule_reference(placeholder_arg, [placeholder_arg, placeholder_arg])
+      end
+    end
+  end
+
+  describe '.update_reference_repo' do
+    context 'when prefetch is on' do
+      it 'should grab the children immediately and then store' do
+        expect(subject).to receive(:prefetch).once
+        expect(subject).to receive(:store_updated_repo).once
+
+        allow(File).to receive(:exist?) { true }
+        subject.prefetch_submodules = true
+        subject.reference_dir = placeholder_arg
+        subject.update_reference_repo(test_url_valid, false)
+      end
+    end
+
+    context 'when prefetch is off' do
+      it 'should store the updated repo' do
+        expect(subject).not_to receive(:prefetch)
+        expect(subject).to receive(:store_updated_repo).once
+
+        allow(File).to receive(:exist?) { true }
+        subject.prefetch_submodules = false
+        subject.reference_dir = placeholder_arg
+        subject.update_reference_repo(placeholder_arg, false)
+      end
+    end
+
+    let(:placeholder_hash) { Hash.new }
+
+    context 'when already have a hash' do
+      it 'should not store' do
+        placeholder_hash[placeholder_arg] = true
+        expect(subject).not_to receive(:store_updated_repo)
+
+        allow(subject).to receive(:reference_repo_name) { placeholder_arg }
+        allow(subject).to receive(:reference_repo_dir) { placeholder_arg }
+        subject.reference_updated = placeholder_hash
+        subject.prefetch_submodules = false
+        subject.update_reference_repo(placeholder_arg, false)
+      end
+    end
+
+    context 'when do not have a hash' do
+      it 'should store' do
+        placeholder_hash[placeholder_arg] = false
+        expect(subject).to receive(:store_updated_repo)
+
+        allow(subject).to receive(:reference_repo_name) { placeholder_arg }
+        subject.reference_updated = placeholder_hash
+        subject.reference_dir = placeholder_arg
+        subject.prefetch_submodules = false
+        subject.update_reference_repo(placeholder_arg, false)
+      end
+    end
+  end
+
+  describe '.prefetch' do
+    it 'should go through the submodule file properly' do
+      expect(Thread).to receive(:new).exactly(3).times
+
+      allow(File).to receive(:readlines) { %w(1 2 3) }
+      subject.prefetch_submodules = true
+      subject.prefetch(placeholder_arg)
+    end
+  end
+
+  describe '.store_updated_repo' do
+    context 'when fail_hard is true' do
+      it 'should raise a Cocaine error' do
+        expect do
+          subject.store_updated_repo(placeholder_arg, placeholder_arg, placeholder_arg, true)
+        end.to raise_error(Cocaine::ExitStatusError)
+      end
+    end
+
+    context 'when fail_hard is false' do
+      it 'should not raise a cocaine error' do
+        cocaine_commandline_double = double('new_cocaine_commandline')
+        allow(cocaine_commandline_double).to receive(:run) { fail Cocaine::ExitStatusError }
+        allow(Cocaine::CommandLine).to receive(:new) { cocaine_commandline_double }
+
+        expect do
+          subject.store_updated_repo(placeholder_arg, placeholder_arg, placeholder_arg, false)
+        end.not_to raise_error(Cocaine::ExitStatusError)
+      end
+    end
+
+    let(:placeholder_hash) { Hash.new }
+
+    it 'should correctly update the hash' do
+      cocaine_commandline_double = double('new_cocaine_commandline')
+      allow(cocaine_commandline_double).to receive(:run) {}
+      allow(Cocaine::CommandLine).to receive(:new) { cocaine_commandline_double }
+
+      subject.reference_updated = placeholder_hash
+      subject.store_updated_repo(placeholder_arg, placeholder_arg, placeholder_arg, false)
+      expect(subject.reference_updated).to eq(placeholder_arg => true)
+    end
+  end
+
+  describe '.with_git_mirror' do
+    it 'should yield properly' do
+      allow(subject).to receive(:update_reference_repo) {}
+      expect(subject).to receive(:reference_repo_dir)
+
+      subject.with_git_mirror(test_url_valid) do
+        yielded << test_url_valid
+      end
+
+      expect(yielded).to eq([test_url_valid])
+    end
+  end
+end

data/spec/git_fastclone_url_helper_spec.rb

@@ -0,0 +1,75 @@
+# Copyright 2015 Square Inc.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'spec_helper'
+require 'git-fastclone'
+
+describe GitFastClone::UrlHelper do
+  let(:test_url_valid) { 'ssh://git@git.com/git-fastclone.git' }
+  let(:test_url_invalid) { 'ssh://git@git.com/git-fastclone' }
+  let(:test_reference_dir) { 'test_reference_dir' }
+  let(:submodule_str) do
+    "Submodule 'TestModule' (https://github.com/TestModule1/TestModule2) registered for path
+      'TestModule'"
+  end
+
+  describe '.path_from_git_url' do
+    let(:tail) { 'git-fastclone' }
+
+    context 'with a valid path' do
+      it 'should get the tail' do
+        expect(subject.path_from_git_url(test_url_valid)).to eq(tail)
+      end
+    end
+
+    context 'with an invalid path' do
+      it 'should still get the tail' do
+        expect(subject.path_from_git_url(test_url_invalid)).to eq(tail)
+      end
+    end
+  end
+
+  describe '.parse_update_info' do
+    it 'should parse correctly' do
+      expect(subject.parse_update_info(submodule_str))
+        .to eq(['TestModule', 'https://github.com/TestModule1/TestModule2'])
+    end
+  end
+
+  describe '.reference_repo_name' do
+    let(:expected_result) { 'git.com-git-fastclone.git' }
+
+    it 'should come up with a unique repo name' do
+      expect(subject.reference_repo_name(test_url_valid)).to eq(expected_result)
+    end
+  end
+
+  describe '.reference_repo_dir' do
+    it 'should join correctly' do
+      allow(subject).to receive(:reference_repo_name) { test_reference_dir }
+
+      expect(subject.reference_repo_dir(test_url_valid, test_reference_dir, false))
+        .to eq(test_reference_dir + '/' + test_reference_dir)
+    end
+  end
+
+  describe '.reference_repo_submodule_file' do
+    it 'should return the right string' do
+      allow(subject).to receive(:reference_repo_dir) { test_reference_dir }
+
+      expect(subject.reference_repo_submodule_file(test_url_valid, test_reference_dir, false))
+        .to eq('test_reference_dir:submodules.txt')
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+require 'rspec/core'
+require 'rspec/mocks'

metadata

@@ -1,29 +1,52 @@
 --- !ruby/object:Gem::Specification
 name: git-fastclone
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 1.0.0
 platform: ruby
 authors:
 - Michael Tauraso
+- James Chang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-21 00:00:00.000000000 Z
-dependencies: []
+date: 2015-09-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: cocaine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A git command that uses reference repositories and threading to quickly
   and recursively clone repositories with many nested submodules
-email: mtauraso@gmail.com
+email:
+- mtauraso@squareup.com
+- jchang@squareup.com
 executables:
 - git-fastclone
 extensions: []
 extra_rdoc_files: []
 files:
+- LICENSE.txt
+- README.md
+- Rakefile
 - bin/git-fastclone
-- lib/execution.rb
 - lib/git-fastclone.rb
-homepage: https://github.com/mtauraso/git-fastclone
+- lib/git-fastclone/version.rb
+- spec/git_fastclone_runner_spec.rb
+- spec/git_fastclone_url_helper_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/square/git-fastclone
 licenses:
-- MIT
+- Apache
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -41,8 +64,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
 summary: git-clone --recursive on steroids!
-test_files: []
+test_files:
+- spec/git_fastclone_runner_spec.rb
+- spec/git_fastclone_url_helper_spec.rb
+- spec/spec_helper.rb

data/lib/execution.rb

@@ -1,71 +0,0 @@
-require 'open3'
-
-# Wrapper around open3.popen2e which fails on error
-#
-# We emulate open3.capture2e with the following changes in behavior:
-# 1) The command is printed to stdout before execution.
-# 2) Attempts to use the shell implicitly are blocked.
-# 3) Nonzero return codes result in the process exiting.
-#
-# If you're looking for more process/stream control read the spawn documentation, and pass
-# options directly here
-def fail_on_error (*cmd, **opts)
-#  puts "Running Command: \n#{debug_print_cmd_list([cmd])}\n"
-  shell_safe(cmd)
-  output, status = Open3.capture2(*cmd, opts)
-  exit_on_status(output, status)
-end
-
-# Look at a cmd list intended for spawn.
-# determine if spawn will call the shell implicitly, fail in that case.
-def shell_safe (cmd)
-  # env and opts in the command spec both aren't of type string.
-  # If you're only passing one string, spawn is going to launch a shell.
-  if cmd.select{ |element| element.class == String }.length == 1
-    puts "You tried to use sqiosbuild to call the shell implicitly. Please don't."
-    puts "Think of the children."
-    puts "Think of shellshock."
-    puts "Please don't. Not ever."
-    exit 1
-  end
-end
-
-def debug_print_cmd_list(cmd_list)
-  # Take a list of command argument lists like you'd sent to open3.pipeline or fail_on_error_pipe and
-  # print out a string that would do the same thing when entered at the shell.
-  #
-  # This is a converter from our internal representation of commands to a subset of bash that
-  # can be executed directly.
-  #
-  # Note this has problems if you specify env or opts
-  # TODO: make this remove those command parts
-  "\"" +
-  cmd_list.map { |cmd|
-    cmd.map { |arg|
-      arg.gsub("\"", "\\\"") # Escape all double quotes in command arguments
-    }.join("\" \"") # Fully quote all command parts. We add quotes to the beginning and end too.
-  }.join("\" | \"") + # Pipe commands to one another.
-  "\""
-end
-
-# If any of the statuses are bad, exits with the
-# return code of the first one.
-#
-# Otherwise returns first argument (output)
-def exit_on_status (output, status)
-  # Do nothing for proper statuses
-  if status.exited? && status.exitstatus == 0
-    return output
-  end
-
-  # If we exited nonzero or abnormally, print debugging info
-  # and explode.
-  if status.exited?
-    puts "Return code was #{status.exitstatus}"
-    exit status.exitstatus
-  end
-  puts "This might be helpful:\nProcessStatus: #{status.inspect}\nRaw POSIX Status: #{status.to_i}\n"
-  exit 1
-end
-
-