right_scraper 1.0.26 → 3.0.0

data/spec/spec_helper.rb

@@ -1,18 +1,18 @@
 #--
-# Copyright: Copyright (c) 2010 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
 # 'Software'), to deal in the Software without restriction, including
 # without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to 
+# distribute, sublicense, and/or sell copies of the Software, and to
 # permit persons to whom the Software is furnished to do so, subject to
 # the following conditions:
 #
 # The above copyright notice and this permission notice shall be
 # included in all copies or substantial portions of the Software.
 #
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
 # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
@@ -22,18 +22,57 @@
 #++
 
 require 'rubygems'
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'right_scraper'))
+
 require 'flexmock'
-require 'spec'
+require 'rspec'
 require 'find'
+require 'json'
 
-$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'right_scraper')))
+RSpec.configure do |c|
+  c.mock_with(:flexmock)
+end
 
-Spec::Runner.configuration.mock_with :flexmock
+ENV["DEVELOPMENT"] ||= "yes"
 
 # Helper module
-module RightScale
-  
+module RightScraper
+
   module SpecHelpers
+    module DevelopmentModeEnvironment
+      def DevelopmentModeEnvironment.included(mod)
+        mod.module_eval do
+          before(:each) do
+            @oldtest = ENV['DEVELOPMENT']
+            ENV['DEVELOPMENT'] = "yes"
+          end
+          after(:each) do
+            if @oldtest.nil?
+              ENV.delete('DEVELOPMENT')
+            else
+              ENV['DEVELOPMENT'] = @oldtest
+            end
+          end
+        end
+      end
+    end
+    module ProductionModeEnvironment
+      def ProductionModeEnvironment.included(mod)
+        mod.module_eval do
+          before(:each) do
+            @oldtest = ENV['DEVELOPMENT']
+            ENV.delete('DEVELOPMENT')
+          end
+          after(:each) do
+            if @oldtest.nil?
+              ENV.delete('DEVELOPMENT')
+            else
+              ENV['DEVELOPMENT'] = @oldtest
+            end
+          end
+        end
+      end
+    end
 
     # Set the 'verbose' environment variable for debugging a failing spec
     VERBOSE='verbose'
@@ -54,6 +93,16 @@ module RightScale
       return res, $?
     end
 
+    def create_cookbook(path, contents)
+      create_file_layout(path, contents)
+      File.open(File.join(path, 'metadata.json'), 'w') { |f| f.puts contents.to_json }
+    end
+
+    def create_workflow(path, name, definition, metadata)
+      File.open(File.join(path, "#{name}#{RightScraper::Resources::Workflow::DEFINITION_EXT}"), 'w') { |f| f.puts definition }
+      File.open(File.join(path, "#{name}#{RightScraper::Resources::Workflow::METADATA_EXT}"), 'w') { |f| f.puts metadata.to_json }
+    end
+
     # Create file layout from given array
     # Strings in array correspond to files while Hashes correspond to folders
     # File content is equal to filename
@@ -75,11 +124,11 @@ module RightScale
           end
         else
           File.open(File.join(path, elem.to_s), 'w') { |f| f.puts elem.to_s }
-        end  
+        end
       end
       true
     end
-    
+
     # Extract array representing file layout for given directory
     #
     # === Parameters
@@ -105,8 +154,32 @@ module RightScale
       end
       dirs + files.sort
     end
-           
- 
+
+    RSpec::Matchers.define :begin_with do |path|
+      match do |directory|
+        directory[0...path.length] == path
+      end
+    end
+  end
+
+  module SharedExamples
+    shared_examples_for "a normal repository" do
+      it 'should scrape' do
+        @scraper.scrape(@repo)
+        @scraper.succeeded?.should be_true
+        @scraper.resources.should_not == []
+        @scraper.resources.size.should == 1
+        @scraper.resources[0].manifest.should == {
+                "folder1/file3"=>"1eb2267bae4e47cab81f8866bbc7e06764ea9be0",
+                "file1"=>"38be7d1b981f2fb6a4a0a052453f887373dc1fe8",
+                "folder2/folder3/file4"=>"a441d6d72884e442ef02692864eee99b4ad933f5",
+                "metadata.json"=>"c2901d21c81ba5a152a37a5cfae35a8e092f7b39",
+                "folder1/file2"=>"639daad06642a8eb86821ff7649e86f5f59c6139"}
+        @scraper.resources[0].metadata.should == [{"folder1"=>["file2", "file3"]},
+                                                  {"folder2"=>[{"folder3"=>["file4"]}]},
+                                                  "file1"]
+      end
+    end
   end
-  
-end
+
+end

data/spec/svn/cookbook_spec.rb

@@ -0,0 +1,97 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec_helper'))
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'cookbook_helper'))
+
+describe RightScraper::Resources::Cookbook do
+  include RightScraper::SpecHelpers::DevelopmentModeEnvironment
+
+  include RightScraper::CookbookHelper
+
+  shared_examples_for 'svn repositories' do
+    it_should_behave_like 'a generic repository'
+
+    it 'should have the appropriate credentials' do
+      @repository.username.should == 'username'
+      @repository.password.should == 'password'
+    end
+  end
+
+  context 'with a SVN repository' do
+    before(:each) do
+      @repository = RightScraper::Repositories::Svn.from_hash(:display_name => 'test repo',
+                                                     :repo_type => :svn,
+                                                     :url => "http://a.site/foo/bar/baz",
+                                                     :first_credential => "username",
+                                                     :second_credential => "password")
+    end
+
+    it_should_behave_like 'svn repositories'
+
+    it 'should have a tag' do
+      @repository.tag.should == "HEAD"
+    end
+
+    it 'should have a cookbook hash' do
+      repo_hash = Digest::SHA1.hexdigest("1\000svn\000http://a.site/foo/bar/baz\000HEAD")
+      example_cookbook(@repository).resource_hash.should ==
+        Digest::SHA1.hexdigest("1\000#{repo_hash}\000")
+    end
+
+    it 'should have a cookbook hash invariant under credential changes' do
+      old_hash = example_cookbook(@repository).resource_hash
+      @repository.first_credential = "b-key"
+      example_cookbook(@repository).resource_hash.should == old_hash
+    end
+
+    it 'should have a cookbook hash that varies when the tag changes' do
+      old_hash = example_cookbook(@repository).resource_hash
+      @repository.tag = "tag"
+      example_cookbook(@repository).resource_hash.should_not == old_hash
+    end
+
+    it 'should have a cookbook hash that varies when the position changes' do
+      example_cookbook(@repository, "foo").resource_hash.should_not ==
+        example_cookbook(@repository, "bar").resource_hash
+    end
+  end
+
+  context 'with a SVN repository with a tag' do
+    before(:each) do
+      @repository = RightScraper::Repositories::Svn.from_hash(:display_name => 'test repo',
+                                                     :repo_type => :svn,
+                                                     :url => "http://a.site/foo/bar/baz",
+                                                     :tag => "foo",
+                                                     :first_credential => "username",
+                                                     :second_credential => "password")
+    end
+
+    it_should_behave_like 'svn repositories'
+
+    it 'should have a tag' do
+      @repository.tag.should == "foo"
+    end
+  end
+
+end

data/spec/svn/multi_svn_spec.rb

@@ -0,0 +1,64 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require File.expand_path(File.join(File.dirname(__FILE__), 'multi_svn_spec_helper'))
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'scraper_helper'))
+
+describe RightScraper::Retrievers::Svn do
+  context 'in a multiple directory situation with a resources_path set' do
+    include RightScraper::SpecHelpers::DevelopmentModeEnvironment
+
+    include RightScraper::ScraperHelper
+
+    before(:all) do
+      @retriever_class = RightScraper::Retrievers::Svn
+      @ignore = ['.svn']
+    end
+
+    context 'given a SVN repository' do
+      before(:each) do
+        @helper = RightScraper::MultiSvnSpecHelper.new
+        @repo = @helper.repo
+      end
+
+      after(:each) do
+        @helper.close unless @helper.nil?
+        @helper = nil
+      end
+
+      include RightScraper::SpecHelpers::FromScratchScraping
+      include RightScraper::SpecHelpers::CookbookScraping
+
+      it 'should see two cookbooks' do
+        @scraper.next_resource.should_not == nil
+        @scraper.next_resource.should_not == nil
+        @scraper.next_resource.should == nil
+      end
+
+      it 'should set the position correctly' do
+        check_resource @scraper.next_resource, :position => "subdir1"
+        check_resource @scraper.next_resource, :position => "subdir2"
+      end
+    end
+  end
+end

data/spec/svn/multi_svn_spec_helper.rb

@@ -0,0 +1,40 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require File.expand_path(File.join(File.dirname(__FILE__), "svn_retriever_spec_helper"))
+
+module RightScraper
+  class MultiSvnSpecHelper < SvnRetrieverSpecHelper
+    def make_cookbooks
+      super
+      create_cookbook(File.join(repo_path, 'subdir1'), repo_content)
+      create_cookbook(File.join(repo_path, 'subdir2'), repo_content)
+    end
+
+    def repo
+      r = super
+      r.resources_path = ['subdir1', 'subdir2']
+      r
+    end
+  end
+end

data/spec/svn/repository_spec.rb

@@ -0,0 +1,72 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec_helper'))
+
+describe RightScraper::Repositories::Svn do
+  include RightScraper::SpecHelpers::DevelopmentModeEnvironment
+
+  before(:each) do
+    @repo = RightScraper::Repositories::Base.from_hash(:display_name => 'test repo',
+                                             :repo_type => :svn,
+                                             :url => "http://foo.bar.baz.quux/%20CBLAH",
+                                             :tag => 'foo',
+                                             :first_credential => "foo:b/ar",
+                                             :second_credential => "foo@bar")
+  end
+
+  it 'should have the same repository hash with or without credentials' do
+    initial_hash = @repo.repository_hash
+    @repo.first_credential = nil
+    @repo.second_credential = nil
+    @repo.repository_hash.should == initial_hash
+  end
+
+  it 'should have the same checkout hash with or without credentials' do
+    initial_hash = @repo.checkout_hash
+    @repo.first_credential = nil
+    @repo.second_credential = nil
+    @repo.checkout_hash.should == initial_hash
+  end
+
+  it 'should have a checkout hash' do
+    @repo.checkout_hash.should ==
+      Digest::SHA1.hexdigest("1\000svn\000http://foo.bar.baz.quux/%20CBLAH\000foo")
+  end
+
+  it 'should have a different checkout hash from repository hash' do
+    @repo.repository_hash.should_not == @repo.checkout_hash
+  end
+
+  it 'should have the same repository hash regardless of tag' do
+    initial_hash = @repo.repository_hash
+    @repo.tag = 'bar'
+    @repo.repository_hash.should == initial_hash
+  end
+
+  it 'should have different checkout hashes as tags change' do
+    initial_hash = @repo.checkout_hash
+    @repo.tag = 'bar'
+    @repo.checkout_hash.should_not == initial_hash
+  end
+end

data/spec/svn/retriever_spec.rb

@@ -0,0 +1,261 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec_helper'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'svn_retriever_spec_helper'))
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'scraper_helper'))
+require 'set'
+
+describe RightScraper::Retrievers::Svn do
+  include RightScraper::SpecHelpers::DevelopmentModeEnvironment
+
+  include RightScraper::ScraperHelper
+
+  before(:all) do
+    @retriever_class = RightScraper::Retrievers::Svn
+    @ignore = ['.svn']
+  end
+
+  context 'given a remote SVN repository' do
+    before(:each) do
+      pending "Not run unless REMOTE_USER and REMOTE_PASSWORD set" unless ENV['REMOTE_USER'] && ENV['REMOTE_PASSWORD']
+      url = 'https://wush.net/svn/rightscale/cookbooks_test/'
+      @repo = RightScraper::Repositories::Base.from_hash(:display_name => 'wush',
+                                               :repo_type    => :svn,
+                                               :url          => url,
+                                               :first_credential => ENV['REMOTE_USER'],
+                                               :second_credential => ENV['REMOTE_PASSWORD'])
+      @retriever = @retriever_class.new(@repo, :max_bytes => 1024**2,
+                                               :max_seconds => 20)
+    end
+
+    it 'should scrape' do
+      @retriever.retrieve
+      @scraper = RightScraper::Scrapers::Base.scraper(:kind            => :cookbook,
+                                                      :ignorable_paths => @retriever.ignorable_paths,
+                                                      :repo_dir        => @retriever.repo_dir,
+                                                      :repository      => @retriever.repository)
+      first = @scraper.next_resource
+      first.should_not == nil
+    end
+
+    # quick_start not actually being a cookbook
+    it 'should scrape 5 repositories' do
+      locations = Set.new
+      (1..5).each {|n|
+        cookbook = @scraper.next_resource
+        locations << cookbook.pos
+        cookbook.should_not == nil
+      }
+      @retriever.retrieve
+      @scraper = RightScraper::Scrapers::Base.scraper(:kind            => :cookbook,
+                                                      :ignorable_paths => @retriever.ignorable_paths,
+                                                      :repo_dir        => @retriever.repo_dir,
+                                                      :repository      => @retriever.repository)
+      @scraper.next_resource.should == nil
+      locations.should == Set.new(["cookbooks/app_rails",
+                                   "cookbooks/db_mysql",
+                                   "cookbooks/repo_git",
+                                   "cookbooks/rs_utils",
+                                   "cookbooks/web_apache"])
+    end
+  end
+
+  context 'given a SVN repository' do
+    before(:each) do
+      @helper = RightScraper::SvnRetrieverSpecHelper.new
+      @repo = @helper.repo
+    end
+
+    after(:each) do
+      @helper.close unless @helper.nil?
+      @helper = nil
+    end
+
+    context 'with one cookbook' do
+      include RightScraper::SpecHelpers::FromScratchScraping
+      include RightScraper::SpecHelpers::CookbookScraping
+
+      it 'should scrape the master branch' do
+        check_resource @scraper.next_resource
+      end
+
+      it 'should only see one cookbook' do
+        @scraper.next_resource.should_not == nil
+        @scraper.next_resource.should == nil
+      end
+
+      it 'should record the head SHA' do
+        tag = @scraper.next_resource.repository.tag
+        tag.should_not == "master"
+        tag.should =~ /^[0-9]+$/
+      end
+    end
+
+    context 'with multiple cookbooks' do
+      def secondary_cookbook(where)
+        FileUtils.mkdir_p(where)
+        @helper.create_cookbook(where, @helper.repo_content)
+      end
+
+      before(:each) do
+        @helper.delete(File.join(@helper.repo_path, "metadata.json"))
+        @cookbook_places = [File.join(@helper.repo_path, "cookbooks", "first"),
+                            File.join(@helper.repo_path, "cookbooks", "second"),
+                            File.join(@helper.repo_path, "other_random_place")]
+        @cookbook_places.each {|place| secondary_cookbook(place)}
+        @helper.commit_content("secondary cookbooks added")
+      end
+
+      include RightScraper::SpecHelpers::FromScratchScraping
+      include RightScraper::SpecHelpers::CookbookScraping
+
+      it 'should scrape' do
+        @cookbook_places.each do |place|
+          check_resource @scraper.next_resource, :position => place[@helper.repo_path.length+1..-1]
+        end
+        scraped.should have(@cookbook_places.size).repositories
+      end
+
+    end
+
+    context 'and a revision' do
+      before(:each) do
+        @oldmetadata = @helper.repo_content
+        @helper.create_file_layout(@helper.repo_path, @helper.branch_content)
+        @helper.commit_content('added branch content')
+        @repo.tag = @helper.commit_id(1).to_s
+      end
+
+      include RightScraper::SpecHelpers::FromScratchScraping
+      include RightScraper::SpecHelpers::CookbookScraping
+
+      it 'should scrape a revision' do
+        check_resource @scraper.next_resource, :metadata => @oldmetadata, :rootdir => @retriever.repo_dir
+      end
+    end
+
+    context 'and an incremental scraper' do
+      before(:each) do
+        @retriever = @retriever_class.new(@repo,
+                                          :max_bytes   => 1024**2,
+                                          :basedir     => @helper.scraper_path,
+                                          :max_seconds => 20)
+        @retriever.retrieve
+        @scraper = RightScraper::Scrapers::Base.scraper(:kind            => :cookbook,
+                                                        :ignorable_paths => @retriever.ignorable_paths,
+                                                        :repo_dir        => @retriever.repo_dir,
+                                                        :repository      => @retriever.repository)
+      end
+
+      after(:each) do
+        @scraper.close
+        @scraper = nil
+      end
+
+      it 'the scraper should store intermediate versions where we expect' do
+        @retriever.repo_dir.should begin_with @helper.scraper_path
+      end
+
+      it 'the scraper should scrape' do
+        check_resource @scraper.next_resource
+      end
+
+      it 'the scraper should only see one cookbook' do
+        @scraper.next_resource.should_not == nil
+        @scraper.next_resource.should == nil
+      end
+
+      context 'when a change is made to the master repo' do
+        before(:each) do
+          @helper.create_file_layout(@helper.repo_path, @helper.branch_content)
+          @helper.commit_content
+        end
+
+        context 'a new scraper' do
+          before(:each) do
+            @olddir = @retriever.repo_dir
+            @retriever = @retriever_class.new(@repo,
+                                             :basedir     => @helper.scraper_path,
+                                             :max_bytes   => 1024**2,
+                                             :max_seconds => 20)
+            @retriever.retrieve
+          end
+
+          it 'should use the same directory for files' do
+            @olddir.should == @retriever.repo_dir
+          end
+
+          it 'should see the new change' do
+            File.exists?(File.join(@olddir, 'branch_folder', 'bfile1')).should be_true
+          end
+        end
+      end
+
+      context 'when a textual change is made to the master repo' do
+        before(:each) do
+          File.open(File.join(@helper.repo_path, "file1"), 'w') do |f|
+            f.puts "bar"
+          end
+          @helper.commit_content("appended bar")
+          File.open(File.join(@helper.repo_path, "file1"), 'a+') do |f|
+            f.puts "bar"
+          end
+          @helper.commit_content("appended bar again")
+          File.open(File.join(@helper.repo_path, "file1"), 'a+') do |f|
+            f.puts "bar"
+          end
+          @helper.commit_content("appended bar again^2")
+          File.open(File.join(@helper.repo_path, "file1"), 'a+') do |f|
+            f.puts "bar"
+          end
+          @helper.commit_content("appended bar again^3")
+        end
+
+        context 'a new scraper' do
+          before(:each) do
+            @olddir = @retriever.repo_dir
+            @retriever = @retriever_class.new(@repo,
+                                              :basedir => @helper.scraper_path,
+                                              :max_bytes => 1024**2,
+                                              :max_seconds => 20)
+            @retriever.retrieve
+            @scraper = RightScraper::Scrapers::Base.scraper(:kind            => :cookbook,
+                                                            :ignorable_paths => @retriever.ignorable_paths,
+                                                            :repo_dir        => @retriever.repo_dir,
+                                                            :repository      => @retriever.repository)
+          end
+
+          it 'should notice the new revision' do
+            cookbook = @scraper.next_resource
+            cookbook.repository.tag.should == "5"
+          end
+
+          it 'should see the new change' do
+            File.open(File.join(@olddir, 'file1')).read.should == "bar\n" * 4
+          end
+        end
+      end
+    end
+  end
+end