right_scraper 1.0.1 → 1.0.2

data/Rakefile

@@ -33,6 +33,9 @@ task :default => 'spec'
 desc "Run unit tests"
 Spec::Rake::SpecTask.new do |t|
   t.spec_files = Dir['**/*_spec.rb']
+  t.spec_opts = lambda do
+    IO.readlines(File.join(File.dirname(__FILE__), 'spec', 'spec.opts')).map {|l| l.chomp.split " "}.flatten
+  end
 end
 
 desc "Run unit tests with RCov"
@@ -40,7 +43,7 @@ Spec::Rake::SpecTask.new(:rcov) do |t|
   t.spec_files = Dir['**/*_spec.rb']
   t.rcov = true
   t.rcov_opts = lambda do
-    IO.readlines(File.join(File.dirname(__FILE__), 'rcov.opts')).map {|l| l.chomp.split " "}.flatten
+    IO.readlines(File.join(File.dirname(__FILE__), 'spec', 'rcov.opts')).map {|l| l.chomp.split " "}.flatten
   end
 end
 

data/lib/right_scraper/scraper_base.rb

@@ -80,7 +80,7 @@ module RightScale
       repo_dir = "#{dir_path}/repo"
     end
 
-    # Was last call to scrapesuccessful?
+    # Was last call to scrape successful?
     # Call errors to get error messages if false
     #
     # === Return

data/lib/right_scraper/scrapers/download_scraper.rb

@@ -34,14 +34,14 @@ module RightScale
       @callback.call(msg, is_step=true) if @callback
       filename = @repo.url.split('/').last
       user_opt = @repo.first_credential && @repo.second_credential ? "--user #{@repo.first_credential}:#{@repo.second_credential}" : ''
-      cmd = "curl --fail --silent --show-error --insecure --location #{user_opt} --output '#{@current_repo_dir}/#{filename}' '#{@repo.url}' 2>&1"
+      cmd = "curl --fail --silent --show-error --insecure --location #{user_opt} --output \"#{@current_repo_dir}/#{filename}\" '#{@repo.url}' 2>&1"
       FileUtils.mkdir_p(@current_repo_dir)
       res = `#{cmd}`
       @errors << res if $? != 0
       if succeeded?
         unzip_opt = case @repo.url[/\.(.*)$/]
-          when 'bzip', 'bzip2' then 'j'
-          when 'tgz', 'gzip' then 'z'
+          when 'bzip', 'bzip2', 'bz2' then 'j'
+          when 'tgz', 'gzip', 'gz' then 'z'
           else ''
         end
         Dir.chdir(@current_repo_dir) do

data/lib/right_scraper/scrapers/git_scraper.rb

@@ -44,41 +44,41 @@ module RightScale
     # === Return
     # true:: Always return true
     def scrape_imp
-      msg = @incremental ? "Pulling " : "Cloning "
+      msg = @incremental ? 'Pulling ' : 'Cloning '
       msg += "git repository '#{@repo.display_name}'"
       @callback.call(msg, is_step=true) if @callback
-      ssh_cmd = ssh_command
-      res = ""
-      is_tag = nil
+      ssh_cmd   = ssh_command
+      res       = ''
+      is_tag    = nil
       is_branch = nil
 
       if @incremental
         Dir.chdir(@current_repo_dir) do
-          is_tag, is_branch, res = git_tag_kind(ssh_cmd)
+          is_tag, is_branch, res = git_fetch(ssh_cmd)
           if !is_tag && !is_branch
-            @callback.call("Nothing to update: repo tag refers to neither a branch nor a tag", is_step=false)
+            @callback.call('Nothing to update: repo tag refers to neither a branch nor a tag', is_step=false)
             return true
           end
           if is_tag && is_branch
             @errors << 'Repository tag ambiguous: could be git tag or git branch'
           else
             tag = @repo.tag.nil? || @repo.tag.empty? ? 'master' : @repo.tag
-            res += `#{ssh_cmd} git pull --quiet --depth 1 origin #{tag} 2>&1`
+            res += `git checkout #{tag} 2>&1`
             if $? != 0
-              @callback.call("Failed to pull repo: #{res}, falling back to cloning", is_step=false) if @callback
+              @callback.call("Failed to update repo: #{res}, falling back to cloning", is_step=false) if @callback
               FileUtils.rm_rf(@current_repo_dir)
               @incremental = false
             end
           end
         end
       end
-      if !@incremental
-        res += `#{ssh_cmd} git clone --quiet --depth 1 #{@repo.url} #{@current_repo_dir} 2>&1`
+      if !@incremental && succeeded?
+        res += `#{ssh_cmd} git clone --quiet --depth 1 "#{@repo.url}" "#{@current_repo_dir}" 2>&1`
         @errors << res if $? != 0
         if !@repo.tag.nil? && !@repo.tag.empty? && @repo.tag != 'master' && succeeded?
           Dir.chdir(@current_repo_dir) do
             if is_tag.nil?
-              is_tag, is_branch, out = git_tag_kind(ssh_cmd)
+              is_tag, is_branch, out = git_fetch(ssh_cmd)
               res += out
             end
             if is_tag && is_branch
@@ -118,12 +118,13 @@ module RightScale
       options = opts.inject('') { |o, (k, v)| o << "#{k.to_s}=#{v}\n" }
     end
 
-    # Store public SSH key into temporary folder and create temporary script
+    # Store private SSH key into temporary folder and create temporary script
     # that wraps SSH and uses this key.
     #
     # === Return
     # ssh(String):: Code to initialize GIT_SSH environment variable with path to SSH wrapper script
     def ssh_command
+      return win32_ssh_command if RUBY_PLATFORM=~/mswin/
       ssh_dir = File.join(@scrape_dir_path, '.ssh')
       FileUtils.mkdir_p(ssh_dir)
       key_content = @repo.first_credential
@@ -131,7 +132,7 @@ module RightScale
         # Explicitely disable public key authentication so we don't endup using the system's key
         options = { :PubkeyAuthentication => 'no' }
       else    
-        ssh_key_path = File.join(ssh_dir, 'ssh.pub')
+        ssh_key_path = File.join(ssh_dir, 'id_rsa')
         File.open(ssh_key_path, 'w') { |f| f.puts(key_content) }
         File.chmod(0600, ssh_key_path)
         options = { :IdentityFile => ssh_key_path }
@@ -144,19 +145,55 @@ module RightScale
       "GIT_SSH=#{ssh}"
     end
 
+    # Prepare SSH for git on Windows
+    # The GIT_SSH trick doesn't seem to work on Windows, instead actually
+    # save the private key in the user ssh folder.
+    # Note: This will override any pre-existing SSH key that was on the system
+    #
+    # === Return
+    # '':: Always return an empty string
+    #
+    # === Raise
+    # Exception:: If the USERPROFILE environment variable is not set
+    def win32_ssh_command
+	  key_content = @repo.first_credential
+	  unless key_content.nil?
+		# resolve key file path.
+		raise 'Environment variable USERPROFILE is missing' unless ENV['USERPROFILE']
+		user_profile_dir_path = ENV['USERPROFILE']
+		ssh_keys_dir = File.join(user_profile_dir_path, '.ssh')
+		FileUtils.mkdir_p(ssh_keys_dir) unless File.directory?(ssh_keys_dir)
+		ssh_key_file_path = File.join(ssh_keys_dir, 'id_rsa')
+
+		# (re)create key file. must overwrite any existing credentials in case
+		# we are switching repositories and have different credentials for each.
+		File.open(ssh_key_file_path, 'w') { |f| f.puts(key_content) }
+
+		# we need to create the "known_hosts" file or else the process will
+		# halt in windows waiting for a yes/no response to the unknown
+		# git host. this is normally handled by specifying
+		# "-o StrictHostKeyChecking=no" in the GIT_SSH executable, but it is
+		# still a mystery why this doesn't work properly in windows.
+		# so make a ssh call which creates the proper "known_hosts" file.
+		system("ssh -o StrictHostKeyChecking=no #{repo.url.split(':').first} exit")
+	  end
+      return ''
+    end
+
+    # Shallow fetch
     # Resolves whehter repository tag is a git tag or a git branch
     # Return output of run commands too
-    # Note:: Assume that current working directory is a git directory
+    # Note: Assume that current working directory is a git directory
     #
     # === Parameters
-    # ssh_cmd<String>:: SSH command to be used with git if any
+    # ssh_cmd(String):: SSH command to be used with git if any
     #
     # === Return
-    # res<Array>::
+    # res(Array)::
     #   - res[0] is true if git repo has a tag with a name corresponding to the repository tag
     #   - res[1] is true if git repo has a branch with a name corresponding to the repository tag
     #   - res[2] contains the git output
-    def git_tag_kind(ssh_cmd)
+    def git_fetch(ssh_cmd)
       return [ false, true, "" ] if @repo.tag.nil? || @repo.tag.empty? || @repo.tag == 'master'
       output = `#{ssh_cmd} git fetch --tags --depth 1 2>&1`
       is_tag = `git tag`.split("\n").include?(@repo.tag)

data/lib/right_scraper/scrapers/svn_scraper.rb

@@ -62,7 +62,7 @@ module RightScale
         end
       end
       if !@incremental
-        svn_cmd = "svn checkout #{@repo.url} #{@current_repo_dir} --non-interactive --quiet" +
+        svn_cmd = "svn checkout \"#{@repo.url}\" \"#{@current_repo_dir}\" --non-interactive --quiet" +
         (!@repo.tag.nil? && !@repo.tag.empty? ? " --revision #{@repo.tag}" : '') +
         (@repo.first_credential ? " --username #{@repo.first_credential}" : '') +
         (@repo.second_credential ? " --password #{@repo.second_credential}" : '') +

data/right_scraper.gemspec

@@ -23,7 +23,7 @@ require 'rubygems'
 
 spec = Gem::Specification.new do |spec|
   spec.name      = 'right_scraper'
-  spec.version   = '1.0.1'
+  spec.version   = '1.0.2'
   spec.authors   = ['Raphael Simon']
   spec.email     = 'raphael@rightscale.com'
   spec.homepage  = 'https://github.com/rightscale/right_scraper'

data/spec/download_scraper_spec.rb

@@ -21,6 +21,9 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
+# Not supported on Windows
+unless RUBY_PLATFORM=~/mswin/
+
 require File.join(File.dirname(__FILE__), 'spec_helper')
 require 'scraper_base'
 require 'repository'
@@ -41,7 +44,7 @@ describe RightScale::DownloadScraper do
     create_file_layout(@download_repo_path, @repo_content)
     @download_file = File.expand_path(File.join(File.dirname(__FILE__), '__download_file.tar'))
     Dir.chdir(@download_repo_path) do
-      res, status = exec("tar cf #{@download_file} *")
+      res, status = exec("tar cf \"#{@download_file}\" *")
       raise "Failed to create tarball: #{res}" unless status.success?
     end
   end
@@ -65,7 +68,7 @@ describe RightScale::DownloadScraper do
       @scraper = RightScale::DownloadScraper.new(@repo_path)
       @repo = RightScale::Repository.from_hash(:display_name => 'test repo',
                                                :repo_type    => :download,
-                                               :url          => "file://#{@download_file}")
+                                               :url          => "file:///#{@download_file}")
     end
 
     after(:all) do
@@ -75,7 +78,7 @@ describe RightScale::DownloadScraper do
     it 'should scrape' do
       messages = []
       @scraper.scrape(@repo) { |m, progress| messages << m if progress }
-      puts "\n **ERRORS: #{@scraper.error_message}\n" unless @scraper.succeeded?
+      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
       @scraper.succeeded?.should be_true
       messages.size.should == 1
       File.directory?(@scraper.current_repo_dir.should be_true)
@@ -85,3 +88,5 @@ describe RightScale::DownloadScraper do
   end
 
 end
+
+end # unless RUBY_PLATFORM=~/mswin/

data/spec/git_scraper_spec.rb

@@ -41,15 +41,15 @@ describe RightScale::GitScraper do
     FileUtils.mkdir_p(@origin_path)
     Dir.chdir(@origin_path) do
       res, status = exec("git init --bare")
-      raise "Failed to initialize remote git repository: #{res}" unless status.success?
+      raise "Failed to initialize bare git repository: #{res}" unless status.success?
     end
     FileUtils.rm_rf(@repo_path)
-    res, status = exec("git clone --quiet #{@origin_path} #{@repo_path}")
+    res, status = exec("git clone --quiet \"#{@origin_path}\" \"#{@repo_path}\"")
     raise "Failed to initialize git repository: #{res}" unless status.success?
     create_file_layout(@repo_path, @repo_content)
     Dir.chdir(@repo_path) do
       res, status = exec("git add .")
-      res, status = exec("git commit --quiet -am 'Initial Commit'") if status.success?
+      res, status = exec("git commit --quiet -m \"Initial Commit\"") if status.success?
       res, status = exec("git push origin master") if status.success?
       raise "Failed to setup git repository: #{res}" unless status.success?
     end
@@ -83,7 +83,7 @@ describe RightScale::GitScraper do
     it 'should scrape' do
       messages = []
       @scraper.scrape(@repo) { |m, progress| messages << m if progress }
-      puts "\n **ERRORS: #{@scraper.error_message}\n" unless @scraper.succeeded?
+      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
       @scraper.succeeded?.should be_true
       messages.size.should == 1
       File.directory?(@scraper.current_repo_dir.should be_true)
@@ -92,11 +92,11 @@ describe RightScale::GitScraper do
     
     it 'should scrape incrementally' do
       @scraper.scrape(@repo)
-      puts "\n **ERRORS: #{@scraper.error_message}\n" unless @scraper.succeeded?
+      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
       @scraper.incremental_update?.should be_true
       messages = []
       @scraper.scrape(@repo) { |m, progress| messages << m if progress }
-      puts "\n **ERRORS: #{@scraper.error_message}\n" unless @scraper.succeeded?
+      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
       @scraper.succeeded?.should be_true
       messages.size.should == 1
       File.directory?(@scraper.current_repo_dir.should be_true)

data/spec/spec.opts

@@ -0,0 +1,2 @@
+--colour
+--format=nested

data/spec/svn_scraper_spec.rb

@@ -38,15 +38,15 @@ describe RightScale::SvnScraper do
     @repo_path = File.join(File.dirname(__FILE__), '__repo')
     @repo_content = [ 'file1', { 'folder1' => [ 'file2', 'file3' ] }, { 'folder2' => [ { 'folder3' => [ 'file4' ] } ] } ]
     FileUtils.rm_rf(@svn_repo_path)
-    res, status = exec("svnadmin create #{@svn_repo_path}")
+    res, status = exec("svnadmin create \"#{@svn_repo_path}\"")
     raise "Failed to initialize SVN repository: #{res}" unless status.success?
     FileUtils.rm_rf(@repo_path)
-    res, status = exec("svn checkout file://#{@svn_repo_path} #{@repo_path}")
+    res, status = exec("svn checkout \"file:///#{@svn_repo_path}\" \"#{@repo_path}\"")
     raise "Failed to checkout repository: #{res}" unless status.success?
     create_file_layout(@repo_path, @repo_content)
     Dir.chdir(@repo_path) do
       res, status = exec("svn add *")
-      res, status = exec("svn commit --quiet -m 'Initial Commit'") if status.success?
+      res, status = exec("svn commit --quiet -m \"Initial Commit\"") if status.success?
       raise "Failed to setup repository: #{res}" unless status.success?
     end
   end
@@ -79,7 +79,7 @@ describe RightScale::SvnScraper do
     it 'should scrape' do
       messages = []
       @scraper.scrape(@repo) { |m, progress| messages << m if progress }
-      puts "\n **ERRORS: #{@scraper.error_message}\n" unless @scraper.succeeded?
+      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
       @scraper.succeeded?.should be_true
       messages.size.should == 1
       File.directory?(@scraper.current_repo_dir.should be_true)
@@ -87,12 +87,13 @@ describe RightScale::SvnScraper do
     end
     
     it 'should scrape incrementally' do
+	  pending "File URLs comparison on Windows is tricky" if RUBY_PLATFORM=~/mswin/
       @scraper.scrape(@repo)
-      puts "\n **ERRORS: #{@scraper.error_message}\n" unless @scraper.succeeded?
+      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
       @scraper.incremental_update?.should be_true
       messages = []
       @scraper.scrape(@repo) { |m, progress| messages << m if progress }
-      puts "\n **ERRORS: #{@scraper.error_message}\n" unless @scraper.succeeded?
+      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
       @scraper.succeeded?.should be_true
       messages.size.should == 1
       File.directory?(@scraper.current_repo_dir.should be_true)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: right_scraper
 version: !ruby/object:Gem::Version 
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors: 
 - Raphael Simon
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-26 00:00:00 -08:00
+date: 2010-02-02 00:00:00 -08:00
 default_executable: 
 dependencies: []
 
@@ -39,6 +39,7 @@ files:
 - spec/repository_spec.rb
 - spec/scraper_base_spec.rb
 - spec/scraper_spec.rb
+- spec/spec.opts
 - spec/spec_helper.rb
 - spec/svn_scraper_spec.rb
 has_rdoc: true