right_scraper 1.0.20 → 1.0.21

data/lib/right_scraper/scrapers/git_scraper.rb

@@ -54,8 +54,7 @@ module RightScale
       if @incremental
         checkout = false
         Dir.chdir(@current_repo_dir) do
-          git_fetch(:depth => 1, :remote_tag => @repo.tag)
-          if succeeded? && @incremental && has_tag
+          if has_tag
             analysis = analyze_repo_tag
             if succeeded?
               is_tag = analysis[:tag]
@@ -69,23 +68,27 @@ module RightScale
                 if current_sha == @repo.tag
                   @callback.call("Nothing to update: already using #{@repo.tag}", is_step=false) if @callback
                   return true
-                else 
+                else
                   # Probably a SHA, retrieve all commits
                   git_fetch(:depth => 2**31 - 1)
                   checkout = true
                 end
               end
+              if succeeded?
+                if checkout
+                  git_checkout(@repo.tag)
+                else
+                  git_checkout(@repo.tag) if is_branch && !on_branch
+                  git_fetch(:depth => 1, :merge => true, :remote_tag => @repo.tag)
+                end
+              end
             end
-          end
-          if succeeded?
-            if checkout || is_branch && !on_branch
-              git_checkout(@repo.tag)
-            else # Pull latest commits on same branch
-              git_fetch(:depth => 1, :merge => true, :remote_tag => @repo.tag)
-            end
+          else
+            git_fetch(:depth => 1, :merge => true)
           end
         end
       end
+
       if !@incremental && succeeded?
         git_cmd = "#{@ssh_cmd} git clone --quiet --depth 1 \"#{@repo.url}\" \"#{@current_repo_dir}\" 2>&1"
         res = @watcher.launch_and_watch(git_cmd, @current_repo_dir)

data/lib/right_scraper/scrapers/svn_scraper.rb

@@ -65,6 +65,7 @@ module RightScale
       cookbooks_path = [ cookbooks_path ] unless cookbooks_path.is_a?(Array)
       if @incremental
         svn_cmd = "svn update --no-auth-cache --non-interactive --quiet" +
+        (!@repo.tag.nil? && !@repo.tag.empty? ? " --revision #{@repo.tag}" : '') +
         (@repo.first_credential ? " --username #{@repo.first_credential}" : '') +
         (@repo.second_credential ? " --password #{@repo.second_credential}" : '') +
         ' 2>&1'

data/right_scraper.gemspec

@@ -23,7 +23,7 @@ require 'rubygems'
 
 spec = Gem::Specification.new do |spec|
   spec.name      = 'right_scraper'
-  spec.version   = '1.0.20'
+  spec.version   = '1.0.21'
   spec.authors   = ['Raphael Simon']
   spec.email     = 'raphael@rightscale.com'
   spec.homepage  = 'https://github.com/rightscale/right_scraper'

data/spec/{download_scraper_spec.rb → download/download_scraper_spec.rb}

@@ -24,7 +24,7 @@
 # Not supported on Windows
 unless RUBY_PLATFORM=~/mswin/
 
-require File.join(File.dirname(__FILE__), 'spec_helper')
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec_helper'))
 require 'scraper_base'
 require 'repository'
 require 'watcher'

data/spec/git/git_scraper_spec.rb

@@ -0,0 +1,165 @@
+#--
+# Copyright: Copyright (c) 2010 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to 
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require File.expand_path(File.join(File.dirname(__FILE__), 'git_scraper_spec_helper'))
+require 'scraper_base'
+require 'repository'
+require 'watcher'
+require File.join('scrapers', 'git_scraper')
+require 'set'
+
+describe RightScale::GitScraper do
+
+  context 'given a git repository' do
+
+    before(:each) do
+      @helper = RightScale::GitScraperSpecHelper.new
+      @helper.setup_test_repo
+      @scrape_dir = File.expand_path(File.join(File.dirname(__FILE__), '__scrape'))
+      @scraper = RightScale::GitScraper.new(@scrape_dir, max_bytes=1024**2, max_seconds=20)
+      @repo = RightScale::Repository.from_hash(:display_name => 'test repo',
+                                               :repo_type    => :git,
+                                               :url          => @helper.repo_path)
+      @current_repo_dir = RightScale::ScraperBase.repo_dir(@scrape_dir, @repo)
+      FileUtils.rm_rf(@current_repo_dir)
+    end
+    
+    after(:each) do
+      @helper.delete_test_repo
+      FileUtils.rm_rf(@scrape_dir)
+    end
+
+    it 'should scrape the master branch' do
+      messages = []
+      @scraper.instance_variable_set(:@current_repo_dir, @current_repo_dir)
+      @scraper.incremental_update?.should be_false
+      @scraper.scrape(@repo) { |m, progress| messages << m if progress }
+      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
+      @scraper.succeeded?.should be_true
+      messages.size.should == 1
+      File.directory?(@scraper.current_repo_dir).should be_true
+      Set.new(@helper.extract_file_layout(@scraper.current_repo_dir, [ '.git', '.ssh' ])).should ==
+        Set.new(@helper.repo_content)
+    end
+    
+    it 'should scrape the master branch incrementally' do
+      @scraper.scrape(@repo)
+      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
+      @scraper.incremental_update?.should be_true
+      @helper.create_file_layout(@helper.repo_path, @helper.additional_content)
+      @helper.commit_content
+      messages = []
+      @scraper.scrape(@repo) { |m, progress| messages << m if progress }
+      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
+      @scraper.succeeded?.should be_true
+      messages.size.should == 1
+      @scraper.instance_variable_get(:@incremental).should == true
+      File.directory?(@scraper.current_repo_dir).should be_true
+      Set.new(@helper.extract_file_layout(@scraper.current_repo_dir, [ '.git', '.ssh' ])).should ==
+        Set.new(@helper.repo_content + @helper.additional_content)
+    end
+
+    context 'and a branch' do
+
+      before(:each) do
+        @helper.setup_branch('test_branch', @helper.branch_content)
+        @branch_repo = RightScale::Repository.from_hash(:display_name => 'test repo',
+                                                        :repo_type    => :git,
+                                                        :url          => @helper.repo_path,
+                                                        :tag          => 'test_branch')
+      end
+
+      it 'should scrape a branch' do
+        messages = []
+        @scraper.scrape(@branch_repo) { |m, progress| messages << m if progress }
+        puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
+        @scraper.succeeded?.should be_true
+        messages.size.should == 1
+        File.directory?(@scraper.current_repo_dir).should be_true
+        Set.new(@helper.extract_file_layout(@scraper.current_repo_dir, [ '.git', '.ssh' ])).should ==
+          Set.new((@helper.repo_content + @helper.branch_content))
+      end
+
+      it 'should scrape a branch incrementally' do
+        @scraper.scrape(@branch_repo)
+        puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
+        @scraper.incremental_update?.should be_true
+        @helper.create_file_layout(@helper.repo_path, @helper.additional_content)
+        @helper.commit_content
+        messages = []
+        @scraper.scrape(@branch_repo) { |m, progress| messages << m if progress }
+        puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
+        @scraper.succeeded?.should be_true
+        messages.size.should == 1
+        @scraper.instance_variable_get(:@incremental).should == true
+        File.directory?(@scraper.current_repo_dir).should be_true
+        Set.new(@helper.extract_file_layout(@scraper.current_repo_dir, [ '.git', '.ssh' ])).should ==
+          Set.new((@helper.repo_content + @helper.branch_content + @helper.additional_content))
+      end
+
+    end
+
+    context 'and a sha ref' do
+
+      before(:each) do
+        @helper.create_file_layout(@helper.repo_path, @helper.branch_content)
+        @helper.commit_content
+        @sha_repo = RightScale::Repository.from_hash(:display_name => 'test repo',
+                                                     :repo_type    => :git,
+                                                     :url          => @helper.repo_path,
+                                                     :tag          => @helper.commit_id(1))
+      end
+
+      it 'should scrape a sha' do
+        messages = []
+        @scraper.scrape(@sha_repo) { |m, progress| messages << m if progress }
+        puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
+        @scraper.succeeded?.should be_true
+        messages.size.should == 1
+        File.directory?(@scraper.current_repo_dir).should be_true
+        Set.new(@helper.extract_file_layout(@scraper.current_repo_dir, [ '.git', '.ssh' ])).should ==
+          Set.new(@helper.repo_content)
+      end
+
+      it 'should scrape a sha incrementally' do
+        @scraper.scrape(@sha_repo)
+        puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
+        @scraper.incremental_update?.should be_true
+        @helper.create_file_layout(@helper.repo_path, @helper.additional_content)
+        @helper.commit_content
+        messages = []
+        @scraper.scrape(@sha_repo) { |m, progress| messages << m if progress }
+        puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
+        @scraper.succeeded?.should be_true
+        messages.size.should == 1
+        @scraper.instance_variable_get(:@incremental).should == true
+        File.directory?(@scraper.current_repo_dir).should be_true
+        Set.new(@helper.extract_file_layout(@scraper.current_repo_dir, [ '.git', '.ssh' ])).should ==
+          Set.new(@helper.repo_content)
+      end
+
+    end
+
+  end
+
+end

data/spec/git/git_scraper_spec_helper.rb

@@ -0,0 +1,72 @@
+#--
+# Copyright: Copyright (c) 2010 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'scraper_spec_helper_base'))
+
+module RightScale
+
+  # Git implementation of scraper spec helper
+  # See parent class for methods headers comments
+  class GitScraperSpecHelper < ScraperSpecHelperBase
+
+    def setup_test_repo
+      FileUtils.rm_rf(repo_path)
+      FileUtils.mkdir_p(repo_path)
+      Dir.chdir(repo_path) do
+        res, status = exec('git init')
+        raise "Failed to initialize bare git repository: #{res}" unless status.success?
+      end
+      create_file_layout(repo_path, repo_content)
+      commit_content(repo_path)
+    end
+
+    def commit_content(commit_message='commit')
+      Dir.chdir(repo_path) do
+        res, status = exec('git add .')
+        res, status = exec("git commit --quiet -m \"#{commit_message}\"") if status.success?
+        raise "Failed to commit changes from #{repo_path}: #{res}" unless status.success?
+      end
+    end
+
+    def setup_branch(branch, new_content=nil)
+      Dir.chdir(repo_path) do
+        res, status = exec("git checkout -b #{branch}")
+        raise "Failed to setup branch #{branch}: #{res}" unless status.success?
+      end
+      unless new_content.nil?
+        create_file_layout(repo_path, new_content)
+        commit_content("Branch #{branch}")
+      end
+    end
+
+    def commit_id(index_from_last=0)
+      res = nil
+      Dir.chdir(repo_path) do
+        res, status = exec("git log --format=%H -#{index_from_last + 1}")
+        raise "Failed to retrieve commit sha #{index_from_last}: #{res}" unless status.success?
+      end
+      commit_id = res.split("\n").last
+    end
+
+  end
+end

data/spec/scraper_spec_helper_base.rb

@@ -0,0 +1,132 @@
+#--
+# Copyright: Copyright (c) 2010 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require File.expand_path(File.join(File.dirname(__FILE__), 'spec_helper'))
+
+module RightScale
+
+  # Base class for all scrapers spec helpers
+  # Define helper methods used to manage repositories using each
+  # source control software
+  class ScraperSpecHelperBase
+
+    include SpecHelpers
+
+    # Path to test repository
+    #
+    # === Return
+    # repo_path(String):: Path to test repository
+    def repo_path
+      repo_path = File.join(File.dirname(__FILE__), '__repo')
+    end
+
+    # Default test repo content
+    #
+    # === Return
+    # content(String):: Default test repo content
+    def repo_content
+      content = [ { 'folder1' => [ 'file2', 'file3' ] }, { 'folder2' => [ { 'folder3' => [ 'file4' ] } ] }, 'file1' ]
+    end
+
+    # Test branch content
+    #
+    # === Return
+    # content(String):: Branch content
+    def branch_content
+      content = [ { 'branch_folder' => [ 'bfile1', 'bfile2' ] }, 'bfile3' ]
+    end
+
+    # Additional content used to test incremental updates
+    #
+    # === Return
+    # content(String):: Additional content
+    def additional_content
+      content = [ { 'additional_folder' => [ 'afile1', 'afile2' ] }, 'afile3' ]
+    end
+
+    # Create test repository following given layout
+    # Delete any previously created repo
+    #
+    # === Return
+    # repo_path(String):: Path to created repositoy
+    #
+    # === Raise
+    # Exception:: If repository initialization fails
+    def setup_test_repo
+      raise 'Not supported'
+    end
+
+    # Delete test repository
+    #
+    # === Return
+    # true:: Always return true
+    def delete_test_repo
+      FileUtils.rm_rf(repo_path)
+    end
+
+    # Commit any non-commited changes of given directory
+    #
+    # === Parameters
+    # repo_path(String):: Path to directory where commit should be created
+    # commit_message(String):: Optional, commit message
+    #
+    # === Raise
+    # Exception:: If commit command fails
+    def commit_content(repo_path, commit_message='Initial commit')
+      raise 'Not implemented'
+    end
+
+    # Create a branch in given repository
+    # Optionally adds a new commit with given file layout
+    # Switch to branch so that next call to 'create_file_layout' will act
+    # on given branch
+    #
+    # === Parameters
+    # branch(String):: Name of branch that should be created
+    # new_content(Hash):: Layout of files to be added to branch as a single commit
+    #
+    # === Return
+    # true:: Always return true
+    #
+    # === Raise
+    # Exception:: If branch command fails
+    def setup_branch(branch, new_content=nil)
+      raise 'Not implemented'
+    end
+
+    # Commit id for commit in test repo
+    # i.e. git sha or svn rev
+    #
+    # === Parameters
+    # index_from_last(Integer):: Commit whose id should be returned:
+    #                              - 0 means last commit
+    #                              - 1 means 1 before last
+    #                              - etc.
+    #
+    # === Return
+    # commit_id(String):: Corresponding commit id
+    def commit_id(index_from_last=0)
+      raise 'Not implemented'
+    end
+  end
+end

data/spec/{svn_scraper_spec.rb → svn/svn_scraper_spec.rb}

@@ -21,66 +21,32 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
-require File.join(File.dirname(__FILE__), 'spec_helper')
+require File.expand_path(File.join(File.dirname(__FILE__), 'svn_scraper_spec_helper'))
 require 'scraper_base'
 require 'repository'
 require 'watcher'
 require File.join('scrapers', 'svn_scraper')
+require 'set'
 
 describe RightScale::SvnScraper do
-  
-  include RightScale::SpecHelpers
-  
-  # Create svn repository following given layout
-  # Update @repo_path with path to repository
-  # Delete any previously created repo
-  def setup_svn_repo
-    @svn_repo_path = File.expand_path(File.join(File.dirname(__FILE__), '__svn_repo'))
-    @repo_path = File.join(File.dirname(__FILE__), '__repo')
-    @repo_content = [ { 'folder1' => [ 'file2', 'file3' ] },
-                      { 'folder2' => [ { 'folder3' => [ 'file4' ] },
-                                       { 'folder5' => [ 'file6' ] } ] },
-                      'file1' ]
-    FileUtils.rm_rf(@svn_repo_path)
-    res, status = exec("svnadmin create \"#{@svn_repo_path}\"")
-    raise "Failed to initialize SVN repository: #{res}" unless status.success?
-    FileUtils.rm_rf(@repo_path)
-    res, status = exec("svn checkout \"file:///#{@svn_repo_path}\" \"#{@repo_path}\"")
-    raise "Failed to checkout repository: #{res}" unless status.success?
-    create_file_layout(@repo_path, @repo_content)
-    Dir.chdir(@repo_path) do
-      res, status = exec("svn add *")
-      res, status = exec("svn commit --quiet -m \"Initial Commit\"") if status.success?
-      raise "Failed to setup repository: #{res}" unless status.success?
-    end
-  end
-
-  # Cleanup after ourselves
-  def delete_svn_repo
-    FileUtils.rm_rf(@svn_repo_path) if @svn_repo_path
-    @svn_repo_path = nil
-    FileUtils.rm_rf(@repo_path) if @repo_path
-    @repo_path = nil
-  end
 
   context 'given a SVN repository' do
 
-    before(:all) do
-      setup_svn_repo
-    end
-
     before(:each) do
-	  file_prefix = 'file://'
-	  file_prefix += '/' if RUBY_PLATFORM =~ /mswin/
-      @scraper = RightScale::SvnScraper.new(@repo_path, max_bytes=1024**2, max_seconds=20)      
+      @helper = RightScale::SvnScraperSpecHelper.new
+      @helper.setup_test_repo
+      @scrape_dir = File.expand_path(File.join(File.dirname(__FILE__), '__scrape'))
+      @scraper = RightScale::SvnScraper.new(@scrape_dir, max_bytes=1024**2, max_seconds=20)
       @repo = RightScale::Repository.from_hash(:display_name => 'test repo',
                                                :repo_type    => :svn,
-                                               :url          => "#{file_prefix}#{@svn_repo_path}")
-      FileUtils.rm_rf(RightScale::ScraperBase.repo_dir(@repo_path, @repo))
+                                               :url          => @helper.repo_url)
+      FileUtils.rm_rf(RightScale::ScraperBase.repo_dir(@helper.repo_path, @repo))
     end
     
-    after(:all) do
-      delete_svn_repo
+    after(:each) do
+      @helper.delete_test_repo
+      FileUtils.rm_rf(@helper.svn_repo_path)
+      FileUtils.rm_rf(@scrape_dir)
     end
 
     it 'should scrape' do
@@ -90,7 +56,8 @@ describe RightScale::SvnScraper do
       @scraper.succeeded?.should be_true
       messages.size.should == 1
       File.directory?(@scraper.current_repo_dir).should be_true
-      extract_file_layout(@scraper.current_repo_dir, [ '.svn' ]).should == @repo_content
+      Set.new(@helper.extract_file_layout(@scraper.current_repo_dir, [ '.svn' ])).should ==
+        Set.new(@helper.repo_content)
     end
     
     it 'should scrape incrementally' do
@@ -98,13 +65,16 @@ describe RightScale::SvnScraper do
       @scraper.scrape(@repo)
       puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
       @scraper.incremental_update?.should be_true
+      @helper.create_file_layout(@helper.repo_path, @helper.additional_content)
+      @helper.commit_content
       messages = []
       @scraper.scrape(@repo) { |m, progress| messages << m if progress }
       puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
       @scraper.succeeded?.should be_true
       messages.size.should == 1
       File.directory?(@scraper.current_repo_dir).should be_true
-      extract_file_layout(@scraper.current_repo_dir, [ '.svn' ]).should == @repo_content
+      Set.new(@helper.extract_file_layout(@scraper.current_repo_dir, [ '.svn' ])).should ==
+        Set.new(@helper.repo_content + @helper.additional_content)
     end
 
     it 'should only scrape cookbooks directories' do
@@ -115,7 +85,7 @@ describe RightScale::SvnScraper do
       @scraper.succeeded?.should be_true
       messages.size.should == 1
       File.directory?(@scraper.current_repo_dir).should be_true
-      extract_file_layout(@scraper.current_repo_dir, [ '.svn' ]).should == [ { 'folder1' => [ 'file2', 'file3' ] }, { 'folder2' => ['folder3' => [ 'file4' ] ] } ]
+      @helper.extract_file_layout(@scraper.current_repo_dir, [ '.svn' ]).should == [ { 'folder1' => [ 'file2', 'file3' ] }, { 'folder2' => ['folder3' => [ 'file4' ] ] } ]
     end
 
     it 'should only scrape cookbooks directories incrementally' do
@@ -130,9 +100,49 @@ describe RightScale::SvnScraper do
       @scraper.succeeded?.should be_true
       messages.size.should == 1
       File.directory?(@scraper.current_repo_dir).should be_true
-      extract_file_layout(@scraper.current_repo_dir, [ '.svn' ]).should == [ { 'folder1' => [ 'file2', 'file3' ] }, { 'folder2' => ['folder3' => [ 'file4' ] ] } ]
+      @helper.extract_file_layout(@scraper.current_repo_dir, [ '.svn' ]).should == [ { 'folder1' => [ 'file2', 'file3' ] }, { 'folder2' => ['folder3' => [ 'file4' ] ] } ]
     end
 
+    context 'and a revision' do
+
+      before(:each) do
+        @helper.create_file_layout(@helper.repo_path, @helper.branch_content)
+        @helper.commit_content
+        @rev_repo = RightScale::Repository.from_hash(:display_name => 'test repo',
+                                                     :repo_type    => :svn,
+                                                     :url          => @helper.repo_url,
+                                                     :tag          => @helper.commit_id(1))
+      end
+
+      it 'should scrape a revision' do
+        messages = []
+        @scraper.scrape(@rev_repo) { |m, progress| messages << m if progress }
+        puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
+        @scraper.succeeded?.should be_true
+        messages.size.should == 1
+        File.directory?(@scraper.current_repo_dir).should be_true
+        Set.new(@helper.extract_file_layout(@scraper.current_repo_dir, [ '.svn' ])).should ==
+          Set.new(@helper.repo_content)
+      end
+
+      it 'should scrape a revision incrementally' do
+        @scraper.scrape(@rev_repo)
+        puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
+        @scraper.incremental_update?.should be_true
+        @helper.create_file_layout(@helper.repo_path, @helper.additional_content)
+        @helper.commit_content
+        messages = []
+        @scraper.scrape(@rev_repo) { |m, progress| messages << m if progress }
+        puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
+        @scraper.succeeded?.should be_true
+        messages.size.should == 1
+        @scraper.instance_variable_get(:@incremental).should == true
+        File.directory?(@scraper.current_repo_dir).should be_true
+        Set.new(@helper.extract_file_layout(@scraper.current_repo_dir, [ '.svn' ])).should ==
+          Set.new(@helper.repo_content)
+      end
+
+    end
   end
 
 end

data/spec/svn/svn_scraper_spec_helper.rb

@@ -0,0 +1,72 @@
+#--
+# Copyright: Copyright (c) 2010 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'scraper_spec_helper_base'))
+
+module RightScale
+
+  # SVN implementation of scraper spec helper
+  # See parent class for methods headers comments
+  class SvnScraperSpecHelper < ScraperSpecHelperBase
+
+    def svn_repo_path
+      svn_repo_path = File.expand_path(File.join(File.dirname(__FILE__), '__svn_repo'))
+    end
+
+    def repo_url
+      file_prefix = 'file://'
+      file_prefix += '/' if RUBY_PLATFORM =~ /mswin/
+      url = "#{file_prefix}#{svn_repo_path}"
+    end
+
+    def setup_test_repo
+      FileUtils.rm_rf(repo_path)
+      FileUtils.mkdir_p(repo_path)
+      FileUtils.rm_rf(svn_repo_path)
+      res, status = exec("svnadmin create \"#{svn_repo_path}\"")
+      raise "Failed to initialize SVN repository: #{res}" unless status.success?
+      res, status = exec("svn checkout \"#{repo_url}\" \"#{repo_path}\"")
+      raise "Failed to checkout repository: #{res}" unless status.success?
+      create_file_layout(repo_path, repo_content)
+      commit_content
+    end
+
+    def commit_content(commit_message='commit message')
+      Dir.chdir(repo_path) do
+        res, status = exec('svn add *')
+        res, status = exec("svn commit --quiet -m \"#{commit_message}\"") if status.success?
+        raise "Failed to commit changes from #{branch}: #{res}" unless status.success?
+      end
+    end
+
+    def commit_id(index_from_last=0)
+      res = nil
+      Dir.chdir(repo_path) do
+        res, status = exec("svn log --quiet #{repo_url}")
+        raise "Failed to retrieve commit revision #{index_from_last}: #{res}" unless status.success?
+      end
+      commit_id = res.split("\n")[ 1 + index_from_last * 2].split(' ').first
+    end
+
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: right_scraper
 version: !ruby/object:Gem::Version 
-  version: 1.0.20
+  version: 1.0.21
 platform: ruby
 authors: 
 - Raphael Simon
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-06-22 00:00:00 -07:00
+date: 2010-07-06 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
@@ -36,15 +36,18 @@ files:
 - lib/right_scraper/watcher.rb
 - lib/right_scraper/win32/process_monitor.rb
 - right_scraper.gemspec
-- spec/download_scraper_spec.rb
-- spec/git_scraper_spec.rb
+- spec/download/download_scraper_spec.rb
+- spec/git/git_scraper_spec.rb
+- spec/git/git_scraper_spec_helper.rb
 - spec/rcov.opts
 - spec/repository_spec.rb
 - spec/scraper_base_spec.rb
 - spec/scraper_spec.rb
+- spec/scraper_spec_helper_base.rb
 - spec/spec.opts
 - spec/spec_helper.rb
-- spec/svn_scraper_spec.rb
+- spec/svn/svn_scraper_spec.rb
+- spec/svn/svn_scraper_spec_helper.rb
 - spec/watcher_spec.rb
 has_rdoc: true
 homepage: https://github.com/rightscale/right_scraper

data/spec/git_scraper_spec.rb

@@ -1,110 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to 
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require File.join(File.dirname(__FILE__), 'spec_helper')
-require 'scraper_base'
-require 'repository'
-require 'watcher'
-require File.join('scrapers', 'git_scraper')
-
-describe RightScale::GitScraper do
-  
-  include RightScale::SpecHelpers
-  
-  # Create git repository following given layout
-  # Update @repo_path with path to repository
-  # Delete any previously created repo
-  def setup_git_repo
-    @origin_path = File.expand_path(File.join(File.dirname(__FILE__), '__origin'))
-    @repo_path = File.join(File.dirname(__FILE__), '__repo')
-    @repo_content = [ { 'folder1' => [ 'file2', 'file3' ] }, { 'folder2' => [ { 'folder3' => [ 'file4' ] } ] }, 'file1' ]
-    FileUtils.rm_rf(@origin_path)
-    FileUtils.mkdir_p(@origin_path)
-    Dir.chdir(@origin_path) do
-      res, status = exec("git init --bare")
-      raise "Failed to initialize bare git repository: #{res}" unless status.success?
-    end
-    FileUtils.rm_rf(@repo_path)
-    res, status = exec("git clone --quiet \"#{@origin_path}\" \"#{@repo_path}\"")
-    raise "Failed to initialize git repository: #{res}" unless status.success?
-    create_file_layout(@repo_path, @repo_content)
-    Dir.chdir(@repo_path) do
-      res, status = exec("git add .")
-      res, status = exec("git commit --quiet -m \"Initial Commit\"") if status.success?
-      res, status = exec("git push origin master") if status.success?
-      raise "Failed to setup git repository: #{res}" unless status.success?
-    end
-  end
-
-  # Cleanup after ourselves
-  def delete_git_repo
-    FileUtils.rm_rf(@origin_path) if @origin_path
-    @origin_path = nil
-    FileUtils.rm_rf(@repo_path) if @repo_path
-    @repo_path = nil
-  end
-
-  context 'given a git repository' do
-
-    before(:all) do
-      setup_git_repo
-    end
-
-    before(:each) do
-      @scraper = RightScale::GitScraper.new(@repo_path, max_bytes=1024**2, max_seconds=20)      
-      @repo = RightScale::Repository.from_hash(:display_name => 'test repo',
-                                               :repo_type    => :git,
-                                               :url          => @origin_path)
-      FileUtils.rm_rf(RightScale::ScraperBase.repo_dir(@repo_path, @repo))
-    end
-    
-    after(:all) do
-      delete_git_repo
-    end
-
-    it 'should scrape' do
-      messages = []
-      @scraper.scrape(@repo) { |m, progress| messages << m if progress }
-      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-      @scraper.succeeded?.should be_true
-      messages.size.should == 1
-      File.directory?(@scraper.current_repo_dir).should be_true
-      extract_file_layout(@scraper.current_repo_dir, [ '.git', '.ssh' ]).should == @repo_content
-    end
-    
-    it 'should scrape incrementally' do
-      @scraper.scrape(@repo)
-      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-      @scraper.incremental_update?.should be_true
-      messages = []
-      @scraper.scrape(@repo) { |m, progress| messages << m if progress }
-      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-      @scraper.succeeded?.should be_true
-      messages.size.should == 1
-      File.directory?(@scraper.current_repo_dir).should be_true
-      extract_file_layout(@scraper.current_repo_dir, [ '.git', '.ssh' ]).should == @repo_content
-    end
-
-  end
-
-end