right_scraper 1.0.11 → 1.0.12

data/lib/right_scraper/repository.rb

@@ -39,6 +39,10 @@ module RightScale
     # Not used for 'download' repositories
     attr_accessor :tag
     
+    # (Array) List of directories containing cookbooks in repository
+    # Root directory is used if this is nil or empty
+    attr_accessor :cookbooks_path
+    
     # (String) Optional, SVN username or git private SSH key content
     attr_accessor :first_credential
     

data/lib/right_scraper/scrapers/svn_scraper.rb

@@ -33,10 +33,24 @@ module RightScale
     # false:: Otherwise
     def incremental_update?
       return false unless File.directory?(@current_repo_dir)
-      Dir.chdir(@current_repo_dir) do
-        info = `svn info`
-        $?.success? && info =~ (/^URL: (.*)$/) && $1 == @repo.url
+      inc = false
+      cookbooks_path = repo.cookbooks_path || []
+      cookbooks_path = [ cookbooks_path ] unless cookbooks_path.is_a?(Array)
+      if cookbooks_path.empty?
+        Dir.chdir(@current_repo_dir) do
+          info = `svn info`
+          inc = $?.success? && info =~ (/^URL: (.*)$/) && $1 == @repo.url
+        end
+      else
+        cookbooks_path.each do |path|
+          Dir.chdir(File.join(@current_repo_dir, path)) do
+            info = `svn info`
+            inc = $?.success? && info =~ (/^URL: (.*)$/) && $1 == File.join(@repo.url, path)
+            break unless inc
+          end
+        end
       end
+      inc
     end
 
     # Scrape SVN repository, see RightScale::Scraper#scrape
@@ -47,27 +61,57 @@ module RightScale
       msg = @incremental ? "Updating " : "Checking out "
       msg += "SVN repository '#{@repo.display_name}'"
       @callback.call(msg, is_step=true) if @callback
+      cookbooks_path = repo.cookbooks_path || []
+      cookbooks_path = [ cookbooks_path ] unless cookbooks_path.is_a?(Array)
       if @incremental
         svn_cmd = "svn update --no-auth-cache --non-interactive --quiet" +
         (@repo.first_credential ? " --username #{@repo.first_credential}" : '') +
         (@repo.second_credential ? " --password #{@repo.second_credential}" : '') +
         ' 2>&1'
-        Dir.chdir(@current_repo_dir) do
-          res = @watcher.launch_and_watch(svn_cmd, @current_repo_dir)
-          handle_watcher_result(res, 'SVN update', update=true)
+        if cookbooks_path.empty?
+          Dir.chdir(@current_repo_dir) do
+            res = @watcher.launch_and_watch(svn_cmd, @current_repo_dir)
+            handle_watcher_result(res, 'SVN update', update=true)
+          end
+        else
+          cookbooks_path.each do |path|
+            break unless succeeded?
+            full_path = File.join(@current_repo_dir, path)
+            Dir.chdir(full_path) do
+              res = @watcher.launch_and_watch(svn_cmd, @current_repo_dir)
+              handle_watcher_result(res, 'SVN update', update=true)
+            end
+          end
         end
       end
       if !@incremental && succeeded?
-        svn_cmd = "svn checkout \"#{@repo.url}\" \"#{@current_repo_dir}\" --no-auth-cache --non-interactive --quiet" +
-        (!@repo.tag.nil? && !@repo.tag.empty? ? " --revision #{@repo.tag}" : '') +
-        (@repo.first_credential ? " --username #{@repo.first_credential}" : '') +
-        (@repo.second_credential ? " --password #{@repo.second_credential}" : '') +
-        ' 2>&1'
-        res = @watcher.launch_and_watch(svn_cmd, @current_repo_dir)
-        handle_watcher_result(res, 'SVN checkout', update=false)
+        if cookbooks_path.empty?
+          res = @watcher.launch_and_watch(svn_checkout_cmd, @current_repo_dir)
+          handle_watcher_result(res, 'SVN checkout', update=false)
+        else
+          cookbooks_path.each do |path|
+            break unless succeeded?
+            res = @watcher.launch_and_watch(svn_checkout_cmd(path), @current_repo_dir)
+            handle_watcher_result(res, 'SVN checkout', update=false)
+          end
+        end
       end
       true
     end
 
+    # SVN checkout command using current repo definition and given path into it
+    #
+    # === Parameters
+    # path(String):: Relative path inside repo that should be checked out
+    #
+    # === Return
+    # svn_cmd(String):: Corresponding SVN command line
+    def svn_checkout_cmd(path='')
+      svn_cmd = "svn checkout \"#{File.join(@repo.url, path)}\" \"#{File.join(@current_repo_dir, path)}\" --no-auth-cache --non-interactive --quiet" +
+      (!@repo.tag.nil? && !@repo.tag.empty? ? " --revision #{@repo.tag}" : '') +
+      (@repo.first_credential ? " --username #{@repo.first_credential}" : '') +
+      (@repo.second_credential ? " --password #{@repo.second_credential}" : '') +
+      ' 2>&1'
+    end
   end
 end

data/lib/right_scraper/watcher.rb

@@ -82,7 +82,7 @@ module RightScale
 
       # Run external process and monitor it in a new thread, platform specific
       pid = monitor.spawn(cmd) do |data|
-        output << data[:output] if data.include?(:output)
+        output << data[:output] if data[:output]
         exit_code = data[:exit_code] if data.include?(:exit_code)
       end
 

data/right_scraper.gemspec

@@ -23,7 +23,7 @@ require 'rubygems'
 
 spec = Gem::Specification.new do |spec|
   spec.name      = 'right_scraper'
-  spec.version   = '1.0.11'
+  spec.version   = '1.0.12'
   spec.authors   = ['Raphael Simon']
   spec.email     = 'raphael@rightscale.com'
   spec.homepage  = 'https://github.com/rightscale/right_scraper'

data/spec/download_scraper_spec.rb

@@ -70,6 +70,7 @@ describe RightScale::DownloadScraper do
       @repo = RightScale::Repository.from_hash(:display_name => 'test repo',
                                                :repo_type    => :download,
                                                :url          => "file:///#{@download_file}")
+      FileUtils.rm_rf(RightScale::ScraperBase.repo_dir(@repo_path, @repo))
     end
 
     after(:all) do
@@ -82,7 +83,7 @@ describe RightScale::DownloadScraper do
       puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
       @scraper.succeeded?.should be_true
       messages.size.should == 1
-      File.directory?(@scraper.current_repo_dir.should be_true)
+      File.directory?(@scraper.current_repo_dir).should be_true
       extract_file_layout(@scraper.current_repo_dir).should == @repo_content
     end
 

data/spec/git_scraper_spec.rb

@@ -75,6 +75,7 @@ describe RightScale::GitScraper do
       @repo = RightScale::Repository.from_hash(:display_name => 'test repo',
                                                :repo_type    => :git,
                                                :url          => @origin_path)
+      FileUtils.rm_rf(RightScale::ScraperBase.repo_dir(@repo_path, @repo))
     end
     
     after(:all) do
@@ -87,7 +88,7 @@ describe RightScale::GitScraper do
       puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
       @scraper.succeeded?.should be_true
       messages.size.should == 1
-      File.directory?(@scraper.current_repo_dir.should be_true)
+      File.directory?(@scraper.current_repo_dir).should be_true
       extract_file_layout(@scraper.current_repo_dir, [ '.git', '.ssh' ]).should == @repo_content
     end
     
@@ -100,7 +101,7 @@ describe RightScale::GitScraper do
       puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
       @scraper.succeeded?.should be_true
       messages.size.should == 1
-      File.directory?(@scraper.current_repo_dir.should be_true)
+      File.directory?(@scraper.current_repo_dir).should be_true
       extract_file_layout(@scraper.current_repo_dir, [ '.git', '.ssh' ]).should == @repo_content
     end
 

data/spec/svn_scraper_spec.rb

@@ -37,7 +37,9 @@ describe RightScale::SvnScraper do
   def setup_svn_repo
     @svn_repo_path = File.expand_path(File.join(File.dirname(__FILE__), '__svn_repo'))
     @repo_path = File.join(File.dirname(__FILE__), '__repo')
-    @repo_content = [ 'file1', { 'folder1' => [ 'file2', 'file3' ] }, { 'folder2' => [ { 'folder3' => [ 'file4' ] } ] } ]
+    @repo_content = [ 'file1', { 'folder1' => [ 'file2', 'file3' ] },
+                               { 'folder2' => [ { 'folder3' => [ 'file4' ] },
+                                                { 'folder5' => [ 'file6' ] } ] } ]
     FileUtils.rm_rf(@svn_repo_path)
     res, status = exec("svnadmin create \"#{@svn_repo_path}\"")
     raise "Failed to initialize SVN repository: #{res}" unless status.success?
@@ -73,6 +75,7 @@ describe RightScale::SvnScraper do
       @repo = RightScale::Repository.from_hash(:display_name => 'test repo',
                                                :repo_type    => :svn,
                                                :url          => "#{file_prefix}#{@svn_repo_path}")
+      FileUtils.rm_rf(RightScale::ScraperBase.repo_dir(@repo_path, @repo))
     end
     
     after(:all) do
@@ -85,7 +88,7 @@ describe RightScale::SvnScraper do
       puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
       @scraper.succeeded?.should be_true
       messages.size.should == 1
-      File.directory?(@scraper.current_repo_dir.should be_true)
+      File.directory?(@scraper.current_repo_dir).should be_true
       extract_file_layout(@scraper.current_repo_dir, [ '.svn' ]).should == @repo_content
     end
     
@@ -99,10 +102,36 @@ describe RightScale::SvnScraper do
       puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
       @scraper.succeeded?.should be_true
       messages.size.should == 1
-      File.directory?(@scraper.current_repo_dir.should be_true)
+      File.directory?(@scraper.current_repo_dir).should be_true
       extract_file_layout(@scraper.current_repo_dir, [ '.svn' ]).should == @repo_content
     end
 
+    it 'should only scrape cookbooks directories' do
+      messages = []
+      @repo.cookbooks_path = [ 'folder1', File.join('folder2', 'folder3') ]
+      @scraper.scrape(@repo) { |m, progress| messages << m if progress }
+      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
+      @scraper.succeeded?.should be_true
+      messages.size.should == 1
+      File.directory?(@scraper.current_repo_dir).should be_true
+      extract_file_layout(@scraper.current_repo_dir, [ '.svn' ]).should == [ { 'folder1' => [ 'file2', 'file3' ] }, { 'folder2' => ['folder3' => [ 'file4' ] ] } ]
+    end
+
+    it 'should only scrape cookbooks directories incrementally' do
+      pending "File URLs comparison on Windows is tricky" if RUBY_PLATFORM=~/mswin/
+      @repo.cookbooks_path = [ 'folder1', File.join('folder2', 'folder3') ]
+      @scraper.scrape(@repo)
+      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
+      @scraper.incremental_update?.should be_true
+      messages = []
+      @scraper.scrape(@repo) { |m, progress| messages << m if progress }
+      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
+      @scraper.succeeded?.should be_true
+      messages.size.should == 1
+      File.directory?(@scraper.current_repo_dir).should be_true
+      extract_file_layout(@scraper.current_repo_dir, [ '.svn' ]).should == [ { 'folder1' => [ 'file2', 'file3' ] }, { 'folder2' => ['folder3' => [ 'file4' ] ] } ]
+    end
+
   end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: right_scraper
 version: !ruby/object:Gem::Version 
-  version: 1.0.11
+  version: 1.0.12
 platform: ruby
 authors: 
 - Raphael Simon
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-02-28 00:00:00 -08:00
+date: 2010-03-26 00:00:00 -07:00
 default_executable: 
 dependencies: []