right_scraper 1.0.26 → 3.0.0

data/lib/right_scraper/scrapers/svn_scraper.rb

@@ -1,119 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to 
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-module RightScale
-
-  class SvnScraper < ScraperBase
-
-    # Check whether it is possible to perform an incremental update of the repo
-    #
-    # === Return
-    # true:: Scrape directory contains files belonging to the scraped repo and protocol supports
-    #        incremental updates
-    # false:: Otherwise
-    def incremental_update?
-      return false unless File.directory?(@current_repo_dir)
-      inc = false
-      cookbooks_path = repo.cookbooks_path || []
-      cookbooks_path = [ cookbooks_path ] unless cookbooks_path.is_a?(Array)
-      if cookbooks_path.empty?
-        Dir.chdir(@current_repo_dir) do
-          info = run('svn', 'info')
-          inc = $?.success? && info =~ (/^URL: (.*)$/) && $1 == @repo.url
-        end
-      else
-        cookbooks_path.each do |path|
-          Dir.chdir(File.join(@current_repo_dir, path)) do
-            info = run('svn', 'info')
-            inc = $?.success? && info =~ (/^URL: (.*)$/) && $1 == File.join(@repo.url, path)
-            break unless inc
-          end
-        end
-      end
-      inc
-    end
-
-    # Scrape SVN repository, see RightScale::Scraper#scrape
-    #
-    # === Return
-    # true:: Always return true
-    def scrape_imp
-      msg = @incremental ? "Updating " : "Checking out "
-      msg += "SVN repository '#{@repo.display_name}'"
-      @callback.call(msg, is_step=true) if @callback
-      cookbooks_path = repo.cookbooks_path || []
-      cookbooks_path = [ cookbooks_path ] unless cookbooks_path.is_a?(Array)
-      if @incremental
-        args = ['update', '--no-auth-cache', '--non-interactive', '--quiet']        
-        args += ['--revision', @repo.tag] if (!@repo.tag.nil? && !@repo.tag.empty?)
-        args += ['--username', @repo.first_credential] if @repo.first_credential
-        args += ['--password', @repo.second_credential] if @repo.second_credential
-        
-        if cookbooks_path.empty?
-          Dir.chdir(@current_repo_dir) do
-            res = @watcher.launch_and_watch('svn', args, @current_repo_dir)
-            handle_watcher_result(res, 'SVN update')
-          end
-        else
-          cookbooks_path.each do |path|
-            break unless succeeded?
-            full_path = File.join(@current_repo_dir, path)
-            Dir.chdir(full_path) do
-              res = @watcher.launch_and_watch('svn', args, @current_repo_dir)
-              handle_watcher_result(res, 'SVN update')
-            end
-          end
-        end
-      end
-      if !@incremental && succeeded?
-        if cookbooks_path.empty?
-          res = @watcher.launch_and_watch('svn', svn_checkout_args, @current_repo_dir)
-          handle_watcher_result(res, 'SVN checkout')
-        else
-          cookbooks_path.each do |path|
-            break unless succeeded?
-            res = @watcher.launch_and_watch('svn', svn_checkout_args(path), @current_repo_dir)
-            handle_watcher_result(res, 'SVN checkout')
-          end
-        end
-      end
-      true
-    end
-
-    # SVN checkout command using current repo definition and given path into it
-    #
-    # === Parameters
-    # path(String):: Relative path inside repo that should be checked out
-    #
-    # === Return
-    # svn_cmd(String):: Corresponding SVN command line
-    def svn_checkout_args(path='')
-      args = ['checkout', File.join(@repo.url, path), File.join(@current_repo_dir, path),
-              '--no-auth-cache', '--non-interactive', '--quiet']
-      args += ['--revision', @repo.tag] if !@repo.tag.nil? && !@repo.tag.empty?
-      args += ['--username', @repo.first_credential] if @repo.first_credential
-      args += ['--password', @repo.second_credential] if @repo.second_credential
-      return args
-    end
-  end
-end

data/lib/right_scraper/watcher.rb

@@ -1,158 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require 'find'
-if RUBY_PLATFORM =~ /mswin/
-  require File.join(File.dirname(__FILE__), 'win32', 'process_monitor')
-else
-  require File.join(File.dirname(__FILE__), 'linux', 'process_monitor')
-end
-
-module RightScale
-
-  # Encapsulate information returned by watcher
-  class WatchStatus
-
-    # Potential outcome of watcher
-    VALID_STATUSES = [ :success, :timeout, :size_exceeded ]
-
-    attr_reader :status    # One of VALID_STATUSES
-    attr_reader :exit_code # Watched process exit code or -1 if process was killed
-    attr_reader :output    # Watched process combined output
-
-    # Initialize attibutes
-    def initialize(status, exit_code, output)
-      @status    = status
-      @exit_code = exit_code
-      @output    = output
-    end
-
-  end
-
-  class Watcher
-
-    attr_reader :max_bytes   # Maximum size in bytes of watched directory before process is killed
-    attr_reader :max_seconds # Maximum number of elapased seconds before external process is killed
-
-    # Initialize attributes
-    #
-    # max_bytes(Integer):: Maximum size in bytes of watched directory before process is killed
-    # max_seconds(Integer):: Maximum number of elapased seconds before external process is killed
-    def initialize(max_bytes, max_seconds)
-      @max_bytes   = max_bytes
-      @max_seconds = max_seconds
-    end
-
-    # Launch given command as external process and watch given directory
-    # so it doesn't exceed given size. Also watch time elapsed and kill
-    # external process if either the size of the watched directory exceed
-    # @max_bytes or the time elapsed exceeds @max_seconds.
-    # Note: This method is not thread-safe, instantiate one watcher per thread
-    #
-    # === Parameters
-    # cmd(String):: Command line to be launched
-    # dest_dir(String):: Watched directory
-    #
-    # === Return
-    # res(RightScale::WatchStatus):: Outcome of watch, see RightScale::WatchStatus
-    def launch_and_watch(cmd, args, dest_dir)
-      exit_code = nil
-      output    = ''
-      monitor   = ProcessMonitor.new
-
-      # Run external process and monitor it in a new thread, platform specific
-      pid = monitor.spawn(cmd, *args) do |data|
-        output << data[:output] if data[:output]
-        exit_code = data[:exit_code] if data.include?(:exit_code)
-      end
-
-      # Loop until process is done or times out or takes too much space
-      timed_out = repeat(1, @max_seconds) do
-        if @max_bytes < 0
-          exit_code
-        else
-          size = 0
-          Find.find(dest_dir) { |f| size += File.stat(f).size rescue 0 if File.file?(f) } if File.directory?(dest_dir)
-          size > @max_bytes || exit_code
-        end
-      end
-
-      # Cleanup and report status
-      # Note: We need to store the exit status before we kill the underlying process so that
-      # if it finished in the mean time we still report -1 as exit code
-      if exit_code
-        exit_status = exit_code
-        outcome = :success
-      else
-        exit_status = -1
-        outcome = (timed_out ? :timeout : :size_exceeded)
-        Process.kill('INT', pid)
-      end
-
-      # Cleanup any open handle etc., platform specific
-      monitor.cleanup
-
-      res = WatchStatus.new(outcome, exit_status, output)
-    end
-
-    protected
-
-    # Run given block in thread and time execution
-    #
-    # === Block
-    # Block whose execution is timed
-    #
-    # === Return
-    # elapsed(Integer):: Number of seconds elapsed while running given block
-    def timed
-      start_at = Time.now
-      yield
-      elapsed = Time.now - start_at
-    end
-
-    # Repeat given block at regular intervals
-    #
-    # === Parameters
-    # seconds(Integer):: Number of seconds between executions
-    # timeout(Integer):: Timeout after which execution stops and method returns
-    #
-    # === Block
-    # Given block gets executed every period seconds until timeout is reached
-    # *or* block returns true
-    #
-    # === Return
-    # res(TrueClass|FalseClass):: true if timeout is reached, false otherwise.
-    def repeat(period, timeout)
-      end_at = Time.now + timeout
-      while res = (timeout < 0 || Time.now < end_at)
-        exit = false
-        elapsed = timed { exit = yield }
-        break if exit
-        sleep(period - elapsed) if elapsed < period
-      end
-      !res
-    end
-
-  end
-
-end

data/lib/right_scraper/win32/process_monitor.rb

@@ -1,98 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require 'win32/process'
-
-module RightScale
-
-  # Windows specific watcher implementation
-  class ProcessMonitor
-
-    include ::Windows::Process
-    include ::Windows::Synchronize
-    include ::Windows::Handle
-
-    # Spawn given process and callback given block with output and exit code
-    #
-    # === Parameters
-    # cmd(String):: Name of the command to run
-    # arg1(String):: Optional, first command-line argumument
-    # arg2(String):: Optional, first command-line argumument
-    # ...
-    # argN(String):: Optional, Nth command-line argumument
-    #
-    # === Block
-    # Given block should take one argument which is a hash which may contain
-    # the keys :output and :exit_code. The value associated with :output is a chunk
-    # of output while the value associated with :exit_code is the process exit code
-    # This block won't be called anymore once the :exit_code key has associated value
-    #
-    # === Return
-    # pid(Integer):: Spawned process pid
-    def spawn(cmd, *args)
-      args = args.map { |a| a.to_s }
-      cmd = ([cmd] + args).join(' ')
-
-      # Run external process and monitor it in a new thread
-      @io = IO.popen(cmd)
-      @handle = OpenProcess(PROCESS_ALL_ACCESS, 0, @io.pid)
-      case @handle
-      when INVALID_HANDLE_VALUE
-        # Something bad happened
-        yield(:exit_code => 1)
-      when 0
-        # Process already finished
-        yield(:exit_code => 0)
-      else
-        # Start output read
-        @reader = Thread.new do
-          o = @io.read
-          until o == ''
-            yield(:output => o)
-            o = @io.read
-          end
-          status = WaitForSingleObject(@handle, INFINITE)
-          exit_code = [0].pack('L')
-          if GetExitCodeProcess(@handle, exit_code)
-            exit_code = exit_code.unpack('L').first
-          else
-            exit_code = 1
-          end
-          yield(:exit_code => exit_code)
-        end
-      end
-      @io.pid
-    end
-
-    # Cleanup underlying handle
-    #
-    # === Return
-    # true:: Always return true
-    def cleanup
-      @reader.join
-      CloseHandle(@handle) if @handle > 0
-      @io.close
-    end
-
-  end
-end

data/spec/download/download_scraper_spec.rb

@@ -1,94 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-# Not supported on Windows
-unless RUBY_PLATFORM=~/mswin/
-
-require File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec_helper'))
-require 'scraper_base'
-require 'repository'
-require 'watcher'
-require File.join('scrapers', 'download_scraper')
-
-describe RightScale::DownloadScraper do
-
-  include RightScale::SpecHelpers
-
-  # Create download repository following given layout
-  # Update @repo_path with path to repository
-  # Delete any previously created repo
-  def setup_download_repo
-    @download_repo_path = File.expand_path(File.join(File.dirname(__FILE__), '__download_repo'))
-    @repo_path = File.join(File.dirname(__FILE__), '__repo')
-    @repo_content = [ { 'folder1' => [ 'file2', 'file3' ] }, { 'folder2' => [ { 'folder3' => [ 'file4' ] } ] }, 'file1' ]
-    FileUtils.rm_rf(@download_repo_path)
-    create_file_layout(@download_repo_path, @repo_content)
-    @download_file = File.expand_path(File.join(File.dirname(__FILE__), '__download_file.tar'))
-    Dir.chdir(@download_repo_path) do
-      res, status = exec("tar cf \"#{@download_file}\" *")
-      raise "Failed to create tarball: #{res}" unless status.success?
-    end
-  end
-
-  # Cleanup after ourselves
-  def delete_download_repo
-    FileUtils.rm_rf(@download_repo_path) if @download_repo_path
-    @download_repo_path = nil
-    FileUtils.rm_rf(@repo_path) if @repo_path
-    @repo_path = nil
-    File.delete(@download_file) if File.exist?(@download_file)
-  end
-
-  context 'given a download repository' do
-
-    before(:all) do
-      setup_download_repo
-    end
-
-    before(:each) do
-      @scraper = RightScale::DownloadScraper.new(@repo_path, max_bytes=1024**2, max_seconds=20)
-      @repo = RightScale::Repository.from_hash(:display_name => 'test repo',
-                                               :repo_type    => :download,
-                                               :url          => "file:///#{@download_file}")
-      FileUtils.rm_rf(RightScale::ScraperBase.repo_dir(@repo_path, @repo))
-    end
-
-    after(:all) do
-      delete_download_repo
-    end
-
-    it 'should scrape' do
-      messages = []
-      @scraper.scrape(@repo) { |m, progress| messages << m if progress }
-      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-      @scraper.succeeded?.should be_true
-      messages.size.should == 1
-      File.directory?(@scraper.current_repo_dir).should be_true
-      extract_file_layout(@scraper.current_repo_dir).should == @repo_content
-    end
-
-  end
-
-end
-
-end # unless RUBY_PLATFORM=~/mswin/

data/spec/git/git_scraper_spec.rb

@@ -1,165 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to 
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require File.expand_path(File.join(File.dirname(__FILE__), 'git_scraper_spec_helper'))
-require 'scraper_base'
-require 'repository'
-require 'watcher'
-require File.join('scrapers', 'git_scraper')
-require 'set'
-
-describe RightScale::GitScraper do
-
-  context 'given a git repository' do
-
-    before(:each) do
-      @helper = RightScale::GitScraperSpecHelper.new
-      @helper.setup_test_repo
-      @scrape_dir = File.expand_path(File.join(File.dirname(__FILE__), '__scrape'))
-      @scraper = RightScale::GitScraper.new(@scrape_dir, max_bytes=1024**2, max_seconds=20)
-      @repo = RightScale::Repository.from_hash(:display_name => 'test repo',
-                                               :repo_type    => :git,
-                                               :url          => @helper.repo_path)
-      @current_repo_dir = RightScale::ScraperBase.repo_dir(@scrape_dir, @repo)
-      FileUtils.rm_rf(@current_repo_dir)
-    end
-    
-    after(:each) do
-      @helper.delete_test_repo
-      FileUtils.rm_rf(@scrape_dir)
-    end
-
-    it 'should scrape the master branch' do
-      messages = []
-      @scraper.instance_variable_set(:@current_repo_dir, @current_repo_dir)
-      @scraper.incremental_update?.should be_false
-      @scraper.scrape(@repo) { |m, progress| messages << m if progress }
-      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-      @scraper.succeeded?.should be_true
-      messages.size.should == 1
-      File.directory?(@scraper.current_repo_dir).should be_true
-      Set.new(@helper.extract_file_layout(@scraper.current_repo_dir, [ '.git', '.ssh' ])).should ==
-        Set.new(@helper.repo_content)
-    end
-    
-    it 'should scrape the master branch incrementally' do
-      @scraper.scrape(@repo)
-      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-      @scraper.incremental_update?.should be_true
-      @helper.create_file_layout(@helper.repo_path, @helper.additional_content)
-      @helper.commit_content
-      messages = []
-      @scraper.scrape(@repo) { |m, progress| messages << m if progress }
-      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-      @scraper.succeeded?.should be_true
-      messages.size.should == 1
-      @scraper.instance_variable_get(:@incremental).should == true
-      File.directory?(@scraper.current_repo_dir).should be_true
-      Set.new(@helper.extract_file_layout(@scraper.current_repo_dir, [ '.git', '.ssh' ])).should ==
-        Set.new(@helper.repo_content + @helper.additional_content)
-    end
-
-    context 'and a branch' do
-
-      before(:each) do
-        @helper.setup_branch('test_branch', @helper.branch_content)
-        @branch_repo = RightScale::Repository.from_hash(:display_name => 'test repo',
-                                                        :repo_type    => :git,
-                                                        :url          => @helper.repo_path,
-                                                        :tag          => 'test_branch')
-      end
-
-      it 'should scrape a branch' do
-        messages = []
-        @scraper.scrape(@branch_repo) { |m, progress| messages << m if progress }
-        puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-        @scraper.succeeded?.should be_true
-        messages.size.should == 1
-        File.directory?(@scraper.current_repo_dir).should be_true
-        Set.new(@helper.extract_file_layout(@scraper.current_repo_dir, [ '.git', '.ssh' ])).should ==
-          Set.new((@helper.repo_content + @helper.branch_content))
-      end
-
-      it 'should scrape a branch incrementally' do
-        @scraper.scrape(@branch_repo)
-        puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-        @scraper.incremental_update?.should be_true
-        @helper.create_file_layout(@helper.repo_path, @helper.additional_content)
-        @helper.commit_content
-        messages = []
-        @scraper.scrape(@branch_repo) { |m, progress| messages << m if progress }
-        puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-        @scraper.succeeded?.should be_true
-        messages.size.should == 1
-        @scraper.instance_variable_get(:@incremental).should == true
-        File.directory?(@scraper.current_repo_dir).should be_true
-        Set.new(@helper.extract_file_layout(@scraper.current_repo_dir, [ '.git', '.ssh' ])).should ==
-          Set.new((@helper.repo_content + @helper.branch_content + @helper.additional_content))
-      end
-
-    end
-
-    context 'and a sha ref' do
-
-      before(:each) do
-        @helper.create_file_layout(@helper.repo_path, @helper.branch_content)
-        @helper.commit_content
-        @sha_repo = RightScale::Repository.from_hash(:display_name => 'test repo',
-                                                     :repo_type    => :git,
-                                                     :url          => @helper.repo_path,
-                                                     :tag          => @helper.commit_id(1))
-      end
-
-      it 'should scrape a sha' do
-        messages = []
-        @scraper.scrape(@sha_repo) { |m, progress| messages << m if progress }
-        puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-        @scraper.succeeded?.should be_true
-        messages.size.should == 1
-        File.directory?(@scraper.current_repo_dir).should be_true
-        Set.new(@helper.extract_file_layout(@scraper.current_repo_dir, [ '.git', '.ssh' ])).should ==
-          Set.new(@helper.repo_content)
-      end
-
-      it 'should scrape a sha incrementally' do
-        @scraper.scrape(@sha_repo)
-        puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-        @scraper.incremental_update?.should be_true
-        @helper.create_file_layout(@helper.repo_path, @helper.additional_content)
-        @helper.commit_content
-        messages = []
-        @scraper.scrape(@sha_repo) { |m, progress| messages << m if progress }
-        puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-        @scraper.succeeded?.should be_true
-        messages.size.should == 1
-        @scraper.instance_variable_get(:@incremental).should == true
-        File.directory?(@scraper.current_repo_dir).should be_true
-        Set.new(@helper.extract_file_layout(@scraper.current_repo_dir, [ '.git', '.ssh' ])).should ==
-          Set.new(@helper.repo_content)
-      end
-
-    end
-
-  end
-
-end