right_scraper 1.0.0 → 1.0.1

data/lib/right_scraper/scraper.rb

@@ -34,7 +34,7 @@ module RightScale
 
     # (String) Path to directory where remote repository was downloaded
     # Note: This will be a subfolder of the scrape directory (directory given to initializer)
-    attr_reader :repo_dir
+    attr_reader :last_repo_dir
     
     # Initialize scrape destination directory
     #
@@ -72,9 +72,20 @@ module RightScale
       raise "Invalid repository type" unless SCRAPERS.include?(repo.repo_type)
       @scraper = @scrapers[repo.repo_type] ||= SCRAPERS[repo.repo_type].new(@scrape_dir)
       @scraper.scrape(repo, &callback)
-      @repo_dir = @scraper.repo_dir
+      @last_repo_dir = @scraper.current_repo_dir
       @scraper.succeeded?
     end
+    
+    # Retrieve directory path where repo was or would be downloaded
+    #
+    # === Parameters
+    # repo(Hash|RightScale::Repository):: Remote repository corresponding to local directory
+    #
+    # === Return 
+    # repo_dir(String):: Path to local directory that corresponds to given repository
+    def repo_dir(repo)
+      repo_dir = RightScale::ScraperBase.repo_dir(scrape_dir, repo)
+    end
 
     # Error messages in case of failure
     #

data/lib/right_scraper/scraper_base.rb

@@ -39,7 +39,7 @@ module RightScale
     attr_reader :errors
 
     # (String) Path to local directory where repository was downloaded
-    attr_reader :repo_dir
+    attr_reader :current_repo_dir
 
     # Set path to directory containing all scraped repos
     #
@@ -54,17 +54,31 @@ module RightScale
     # after this method initializes all the scraper attributes properly.
     # See RightScale::Scraper#scrape
     def scrape(repo, &callback)
-      @repo            = repo
-      @callback        = callback
-      @scrape_dir_name = Digest::MD5.hexdigest(repo.to_s)
-      @scrape_dir_path = File.join(root_dir, @scrape_dir_name)
-      @repo_dir        = "#{@scrape_dir_path}/repo"
-      @incremental     = incremental_update?
-      @errors          = []
-      FileUtils.rm_rf(@repo_dir) unless @incremental
+      @repo             = repo
+      @callback         = callback
+      @current_repo_dir = ScraperBase.repo_dir(root_dir, repo)
+      @scrape_dir_path  = File.expand_path(File.join(@current_repo_dir, '..'))
+      @incremental      = incremental_update?
+      @errors           = []
+      FileUtils.rm_rf(@current_repo_dir) unless @incremental
       scrape_imp
       true
     end
+    
+    # Path to directory where given repo should be or was downloaded
+    #
+    # === Parameters
+    # root_dir(String):: Path to directory containing all scraped repositories
+    # repo(Hash|RightScale::Repository):: Remote repository corresponding to local directory
+    #
+    # === Return 
+    # repo_dir(String):: Path to local directory that corresponds to given repository
+    def self.repo_dir(root_dir, repo)
+      repo = Repository.from_hash(repo) if repo.is_a?(Hash)
+      dir_name  = Digest::MD5.hexdigest(repo.to_s)
+      dir_path  = File.join(root_dir, dir_name)
+      repo_dir = "#{dir_path}/repo"
+    end
 
     # Was last call to scrapesuccessful?
     # Call errors to get error messages if false

data/lib/right_scraper/scrapers/download_scraper.rb

@@ -34,8 +34,8 @@ module RightScale
       @callback.call(msg, is_step=true) if @callback
       filename = @repo.url.split('/').last
       user_opt = @repo.first_credential && @repo.second_credential ? "--user #{@repo.first_credential}:#{@repo.second_credential}" : ''
-      cmd = "curl --fail --silent --show-error --insecure --location #{user_opt} --output '#{@repo_dir}/#{filename}' '#{@repo.url}' 2>&1"
-      FileUtils.mkdir_p(@repo_dir)
+      cmd = "curl --fail --silent --show-error --insecure --location #{user_opt} --output '#{@current_repo_dir}/#{filename}' '#{@repo.url}' 2>&1"
+      FileUtils.mkdir_p(@current_repo_dir)
       res = `#{cmd}`
       @errors << res if $? != 0
       if succeeded?
@@ -44,7 +44,7 @@ module RightScale
           when 'tgz', 'gzip' then 'z'
           else ''
         end
-        Dir.chdir(@repo_dir) do
+        Dir.chdir(@current_repo_dir) do
           cmd = "tar x#{unzip_opt}f #{filename} 2>&1"
           res = `#{cmd}`
           @errors << res if $? != 0

data/lib/right_scraper/scrapers/git_scraper.rb

@@ -32,8 +32,8 @@ module RightScale
     #        incremental updates
     # false:: Otherwise
     def incremental_update?
-      return false unless File.directory?(@repo_dir)
-      Dir.chdir(@repo_dir) do
+      return false unless File.directory?(@current_repo_dir)
+      Dir.chdir(@current_repo_dir) do
         remote_url = `git config --get remote.origin.url`.chomp
         $?.success? && remote_url == @repo.url
       end
@@ -53,7 +53,7 @@ module RightScale
       is_branch = nil
 
       if @incremental
-        Dir.chdir(@repo_dir) do
+        Dir.chdir(@current_repo_dir) do
           is_tag, is_branch, res = git_tag_kind(ssh_cmd)
           if !is_tag && !is_branch
             @callback.call("Nothing to update: repo tag refers to neither a branch nor a tag", is_step=false)
@@ -66,17 +66,17 @@ module RightScale
             res += `#{ssh_cmd} git pull --quiet --depth 1 origin #{tag} 2>&1`
             if $? != 0
               @callback.call("Failed to pull repo: #{res}, falling back to cloning", is_step=false) if @callback
-              FileUtils.rm_rf(@repo_dir)
+              FileUtils.rm_rf(@current_repo_dir)
               @incremental = false
             end
           end
         end
       end
       if !@incremental
-        res += `#{ssh_cmd} git clone --quiet --depth 1 #{@repo.url} #{@repo_dir} 2>&1`
+        res += `#{ssh_cmd} git clone --quiet --depth 1 #{@repo.url} #{@current_repo_dir} 2>&1`
         @errors << res if $? != 0
         if !@repo.tag.nil? && !@repo.tag.empty? && @repo.tag != 'master' && succeeded?
-          Dir.chdir(@repo_dir) do
+          Dir.chdir(@current_repo_dir) do
             if is_tag.nil?
               is_tag, is_branch, out = git_tag_kind(ssh_cmd)
               res += out

data/lib/right_scraper/scrapers/svn_scraper.rb

@@ -32,8 +32,8 @@ module RightScale
     #        incremental updates
     # false:: Otherwise
     def incremental_update?
-      return false unless File.directory?(@repo_dir)
-      Dir.chdir(@repo_dir) do
+      return false unless File.directory?(@current_repo_dir)
+      Dir.chdir(@current_repo_dir) do
         info = `svn info`
         $?.success? && info =~ (/^URL: (.*)$/) && $1 == @repo.url
       end
@@ -52,17 +52,17 @@ module RightScale
         (@repo.first_credential ? " --username #{@repo.first_credential}" : '') +
         (@repo.second_credential ? " --password #{@repo.second_credential}" : '') +
         ' 2>&1'
-        Dir.chdir(@repo_dir) do
+        Dir.chdir(@current_repo_dir) do
           res = `#{svn_cmd}`
           if $? != 0
             @callback.call("Failed to update repo: #{res}, falling back to checkout", is_step=false) if @callback
-            FileUtils.rm_rf(@repo_dir)
+            FileUtils.rm_rf(@current_repo_dir)
             @incremental = false
           end
         end
       end
       if !@incremental
-        svn_cmd = "svn checkout #{@repo.url} #{@repo_dir} --non-interactive --quiet" +
+        svn_cmd = "svn checkout #{@repo.url} #{@current_repo_dir} --non-interactive --quiet" +
         (!@repo.tag.nil? && !@repo.tag.empty? ? " --revision #{@repo.tag}" : '') +
         (@repo.first_credential ? " --username #{@repo.first_credential}" : '') +
         (@repo.second_credential ? " --password #{@repo.second_credential}" : '') +

data/right_scraper.gemspec

@@ -23,7 +23,7 @@ require 'rubygems'
 
 spec = Gem::Specification.new do |spec|
   spec.name      = 'right_scraper'
-  spec.version   = '1.0.0'
+  spec.version   = '1.0.1'
   spec.authors   = ['Raphael Simon']
   spec.email     = 'raphael@rightscale.com'
   spec.homepage  = 'https://github.com/rightscale/right_scraper'

data/spec/download_scraper_spec.rb

@@ -78,8 +78,8 @@ describe RightScale::DownloadScraper do
       puts "\n **ERRORS: #{@scraper.error_message}\n" unless @scraper.succeeded?
       @scraper.succeeded?.should be_true
       messages.size.should == 1
-      File.directory?(@scraper.repo_dir.should be_true)
-      extract_file_layout(@scraper.repo_dir).should == @repo_content
+      File.directory?(@scraper.current_repo_dir.should be_true)
+      extract_file_layout(@scraper.current_repo_dir).should == @repo_content
     end
 
   end

data/spec/git_scraper_spec.rb

@@ -86,8 +86,8 @@ describe RightScale::GitScraper do
       puts "\n **ERRORS: #{@scraper.error_message}\n" unless @scraper.succeeded?
       @scraper.succeeded?.should be_true
       messages.size.should == 1
-      File.directory?(@scraper.repo_dir.should be_true)
-      extract_file_layout(@scraper.repo_dir, [ '.git', '.ssh' ]).should == @repo_content
+      File.directory?(@scraper.current_repo_dir.should be_true)
+      extract_file_layout(@scraper.current_repo_dir, [ '.git', '.ssh' ]).should == @repo_content
     end
     
     it 'should scrape incrementally' do
@@ -99,8 +99,8 @@ describe RightScale::GitScraper do
       puts "\n **ERRORS: #{@scraper.error_message}\n" unless @scraper.succeeded?
       @scraper.succeeded?.should be_true
       messages.size.should == 1
-      File.directory?(@scraper.repo_dir.should be_true)
-      extract_file_layout(@scraper.repo_dir, [ '.git', '.ssh' ]).should == @repo_content
+      File.directory?(@scraper.current_repo_dir.should be_true)
+      extract_file_layout(@scraper.current_repo_dir, [ '.git', '.ssh' ]).should == @repo_content
     end
 
   end

data/spec/scraper_base_spec.rb

@@ -38,4 +38,9 @@ describe RightScale::ScraperBase do
     @base.send(:incremental_update?).should be_false
   end
     
-end
+  it 'should allow retrieving the download directory path' do
+    repo_dir = RightScale::ScraperBase.repo_dir('root_dir', { :repo_type => :git, :url => 'git://github.com/rightscale/right_scraper.git' })
+    repo_dir.should =~ /^root_dir\//
+  end
+  
+end 

data/spec/scraper_spec.rb

@@ -45,22 +45,22 @@ describe RightScale::Scraper do
     repo.repo_type = :mock
     @mock_scraper.should_receive(:scrape).with(repo, Proc).and_return(true)
     @mock_scraper.should_receive(:succeeded?).and_return(true)
-    @mock_scraper.should_receive(:repo_dir).and_return('42')
+    @mock_scraper.should_receive(:current_repo_dir).and_return('42')
     @scraper.scrape(repo) { }.should be_true
-    @scraper.repo_dir.should == '42'
+    @scraper.last_repo_dir.should == '42'
   end
   
   it 'should scrape from a hash' do
     @mock_scraper.should_receive(:scrape).with(RightScale::Repository, Proc).and_return(true)
     @mock_scraper.should_receive(:succeeded?).and_return(true)
-    @mock_scraper.should_receive(:repo_dir).and_return('42')
+    @mock_scraper.should_receive(:current_repo_dir).and_return('42')
     @scraper.scrape({:repo_type => :mock}) { }.should be_true
   end
   
   it 'should report failures' do
     @mock_scraper.should_receive(:scrape).with(RightScale::Repository, Proc).and_return(true)
     @mock_scraper.should_receive(:succeeded?).and_return(false)
-    @mock_scraper.should_receive(:repo_dir).and_return('42')
+    @mock_scraper.should_receive(:current_repo_dir).and_return('42')
     @scraper.scrape({:repo_type => :mock}) { }.should be_false
   end
     

data/spec/svn_scraper_spec.rb

@@ -82,8 +82,8 @@ describe RightScale::SvnScraper do
       puts "\n **ERRORS: #{@scraper.error_message}\n" unless @scraper.succeeded?
       @scraper.succeeded?.should be_true
       messages.size.should == 1
-      File.directory?(@scraper.repo_dir.should be_true)
-      extract_file_layout(@scraper.repo_dir, [ '.svn' ]).should == @repo_content
+      File.directory?(@scraper.current_repo_dir.should be_true)
+      extract_file_layout(@scraper.current_repo_dir, [ '.svn' ]).should == @repo_content
     end
     
     it 'should scrape incrementally' do
@@ -95,8 +95,8 @@ describe RightScale::SvnScraper do
       puts "\n **ERRORS: #{@scraper.error_message}\n" unless @scraper.succeeded?
       @scraper.succeeded?.should be_true
       messages.size.should == 1
-      File.directory?(@scraper.repo_dir.should be_true)
-      extract_file_layout(@scraper.repo_dir, [ '.svn' ]).should == @repo_content
+      File.directory?(@scraper.current_repo_dir.should be_true)
+      extract_file_layout(@scraper.current_repo_dir, [ '.svn' ]).should == @repo_content
     end
 
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: right_scraper
 version: !ruby/object:Gem::Version 
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors: 
 - Raphael Simon
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-25 00:00:00 -08:00
+date: 2010-01-26 00:00:00 -08:00
 default_executable: 
 dependencies: []