right_scraper 3.2.6 → 5.0.1

data/spec/cookbook_s3_upload_spec.rb

@@ -1,159 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require File.expand_path(File.join(File.dirname(__FILE__), 'spec_helper'))
-require 'tmpdir'
-require 'digest/sha1'
-
-describe RightScraper::Scanners::CookbookS3Upload do
-  include RightScraper::SpecHelpers::DevelopmentModeEnvironment
-
-  include RightScraper::SpecHelpers
-
-  # Create download repository following given layout
-  # Update @repo_path with path to repository
-  # Delete any previously created repo
-  def setup_download_repo
-    @tmpdir = Dir.mktmpdir
-    @download_repo_path = File.join(@tmpdir, "download")
-    @repo_path = File.join(@tmpdir, "repo")
-    @repo_content = [ { 'folder1' => [ 'file2', 'file3' ] }, { 'folder2' => [ { 'folder3' => [ 'file4' ] } ] }, 'file1' ]
-    create_cookbook(@download_repo_path, @repo_content)
-    @download_file = File.join(@tmpdir, "file.tar")
-    Dir.chdir(@download_repo_path) do
-      res, status = exec("tar cf \"#{@download_file}\" *")
-      raise "Failed to create tarball: #{res}" unless status.success?
-    end
-  end
-
-  # Cleanup after ourselves
-  def delete_download_repo
-    FileUtils.remove_entry_secure @tmpdir unless @tmpdir.nil?
-  end
-
-  before(:all) do
-    @scraperclass = RightScraper::Scrapers::Cookbook
-  end
-
-  before(:each) do
-    pending "Not run unless AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY set" unless
-            ENV['AWS_ACCESS_KEY_ID'] && ENV['AWS_SECRET_ACCESS_KEY']
-  end
-
-  context "given a bucket that doesn't exist" do
-    before(:each) do
-      setup_download_repo
-    end
-
-    after(:each) do
-      delete_download_repo
-    end
-
-    before(:each) do
-      @repo = RightScraper::Repositories::Base.from_hash(:display_name => 'test repo',
-                                               :repo_type    => :download,
-                                               :url          => "file:///#{@download_file}")
-      @s3 = RightAws::S3.new(aws_access_key_id=ENV['AWS_ACCESS_KEY_ID'],
-                            aws_secret_access_key=ENV['AWS_SECRET_ACCESS_KEY'],
-                            :logger => RightScraper::Logger.new)
-      FileUtils.rm_rf(RightScraper::Retrievers::Base.repo_dir(@repo_path, @repo))
-    end
-
-    it 'should raise an exception immediately' do
-      bucket_name = 'this-bucket-does-not-exist'
-      @s3.bucket(bucket_name).should be_nil
-      lambda {
-      @scraper = @scraperclass.new(:repository => @repo,
-                                   :repo_dir => @download_repo_path,
-                                   :scanners => [RightScraper::Scanners::CookbookMetadata,
-                                                 RightScraper::Scanners::CookbookManifest,
-                                                 RightScraper::Scanners::CookbookS3Upload],
-                                   :s3_key => ENV['AWS_ACCESS_KEY_ID'],
-                                   :s3_secret => ENV['AWS_SECRET_ACCESS_KEY'],
-                                   :s3_bucket => bucket_name,
-                                   :max_bytes => 1024**2,
-                                   :max_seconds => 20)
-        }.should raise_exception(/Need an actual, existing S3 bucket!/)
-    end
-  end
-
-  context 'given a download repository with the S3UploadScanner' do
-    before(:each) do
-      setup_download_repo
-    end
-
-    before(:each) do
-      @repo = RightScraper::Repositories::Base.from_hash(:display_name => 'test repo',
-                                               :repo_type    => :download,
-                                               :url          => "file:///#{@download_file}")
-      bucket_name = 'com.rightscale.test.20100823'
-      @scraper = @scraperclass.new(:repository => @repo,
-                                   :repo_dir => @download_repo_path,
-                                   :scanners => [RightScraper::Scanners::CookbookMetadata,
-                                                 RightScraper::Scanners::CookbookManifest,
-                                                 RightScraper::Scanners::CookbookS3Upload],
-                                   :s3_key => ENV['AWS_ACCESS_KEY_ID'],
-                                   :s3_secret => ENV['AWS_SECRET_ACCESS_KEY'],
-                                   :s3_bucket => bucket_name,
-                                   :max_bytes => 1024**2,
-                                   :max_seconds => 20)
-      s3 = RightAws::S3.new(aws_access_key_id=ENV['AWS_ACCESS_KEY_ID'],
-                            aws_secret_access_key=ENV['AWS_SECRET_ACCESS_KEY'],
-                            :logger => RightScraper::Logger.new)
-
-      # create=true is prone to the too-many-buckets error even when the bucket
-      # already exists. since the bucket always exists for the test account
-      # there is no need to try creating it programmatically and fail specs.
-      @bucket = s3.bucket(bucket_name, create=false)
-      FileUtils.rm_rf(RightScraper::Retrievers::Base.repo_dir(@repo_path, @repo))
-    end
-
-    after(:each) do
-      delete_download_repo
-    end
-
-    context 'that has scraped' do
-      before(:each) do
-        @cookbook = @scraper.next_resource
-        @cookbook.should_not be_nil
-      end
-
-      it 'the cookbook should exist' do
-        s3cookbook = @bucket.get(File.join('Cooks', @cookbook.resource_hash))
-        s3cookbook.should_not be_nil
-        ::Digest::MD5.hexdigest(s3cookbook).should == @cookbook.resource_hash
-        s3cookbook.should_not == ::RightScraper::Resources::Cookbook::EMPTY_MANIFEST_JSON
-        hash = JSON.parse(s3cookbook)
-        hash["manifest"].should == @cookbook.manifest
-      end
-
-      it 'every file in the manifest should exist' do
-        @cookbook.manifest.each do |key, value|
-          file = @bucket.get(File.join('Files', value))
-          file.should_not be_nil
-          value.should == Digest::MD5.hexdigest(file)
-        end
-      end
-    end
-  end
-end

data/spec/download/download_retriever_spec.rb

@@ -1,118 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require File.expand_path(File.join(File.dirname(__FILE__), 'download_retriever_spec_helper'))
-require File.expand_path(File.join(File.dirname(__FILE__), '..', 'scraper_helper'))
-
-describe RightScraper::Retrievers::Download do
-  include RightScraper::SpecHelpers::DevelopmentModeEnvironment
-
-  include RightScraper::ScraperHelper
-  include RightScraper::SpecHelpers
-
-  before(:each) do
-    @helper = RightScraper::DownloadRetrieverSpecHelper.new
-    @repo = @helper.repo
-    @retriever = @repo.retriever(:max_bytes => 1024**2, :max_seconds => 20, :basedir => @helper.download_repo_path)
-  end
-
-  after(:each) do
-    @helper.close unless @helper.nil?
-    @helper = nil
-  end
-
-  it 'should not initially have a tag' do
-    @repo.tag.should be_nil
-  end
-
-  context 'given a password protected repository' do
-    before(:each) do
-      pending "Not run unless REMOTE_USER and REMOTE_PASSWORD set" unless ENV['REMOTE_USER'] && ENV['REMOTE_PASSWORD']
-      url = 'https://wush.net/svn/rightscale/cookbooks_test/cookbooks/app_rails.tar.gz'
-      @repo.url = url
-      @repo.display_name = 'wush'
-    end
-
-    it 'should scrape' do
-      @retriever.retrieve
-      scraper = RightScraper::Scrapers::Base.scraper(:kind            => :cookbook,
-                                                     :ignorable_paths => @retriever.ignorable_paths,
-                                                     :repo_dir        => @retriever.repo_dir,
-                                                     :repository      => @retriever.repository)
-      cookbook = scraper.next_resource
-      cookbook.should_not be_nil
-      cookbook.metadata.should_not be_nil
-      cookbook.metadata["name"].should == "app_rails"
-      cookbook.repository.tag.should_not be_nil
-    end
-  end
-
-  context 'given a download repository' do
-    before(:each) do
-      @download_file = @helper.download_file
-    end
-
-    def get_scraper
-      @retriever.retrieve
-      RightScraper::Scrapers::Base.scraper(:kind            => :cookbook,
-                                           :ignorable_paths => @retriever.ignorable_paths,
-                                           :repo_dir        => @retriever.repo_dir,
-                                           :repository      => @retriever.repository)
-    end
-
-    it 'should only return one cookbook' do
-      scraper = get_scraper
-      scraper.next_resource
-      scraper.next_resource.should == nil
-    end
-
-    it 'should scrape' do
-      scraper = get_scraper
-      @helper.check_resource(scraper.next_resource, @download_file, @repo)
-    end
-
-    it 'should scrape a gzipped tarball' do
-      res, status = exec("gzip -c #{@download_file} > #{@download_file}.gz")
-      raise "Failed to gzip tarball: #{res}" unless status.success?
-      @repo.url += ".gz"
-      scraper = get_scraper
-      begin
-        @helper.check_resource(scraper.next_resource, @download_file + ".gz", @repo)
-      ensure
-        File.unlink(@download_file + ".gz")
-      end
-    end
-
-    it 'should scrape a bzipped tarball' do
-      res, status = exec("bzip2 -c #{@download_file} > #{@download_file}.bz2")
-      raise "Failed to bzip tarball: #{res}" unless status.success?
-      @repo.url += ".bz2"
-      scraper = get_scraper
-      begin
-        @helper.check_resource(scraper.next_resource, @download_file + ".bz2", @repo)
-      ensure
-        File.unlink(@download_file + ".bz2")
-      end
-    end
-  end
-end

data/spec/download/download_retriever_spec_helper.rb

@@ -1,72 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require File.expand_path(File.join(File.dirname(__FILE__), '..', 'retriever_spec_helper'))
-require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'lib', 'right_scraper'))
-
-module RightScraper
-  class DownloadRetrieverSpecHelper < RetrieverSpecHelper
-    def download_repo_path
-      File.join(@tmpdir, "download")
-    end
-
-    attr_reader :download_file
-
-    def repo
-      RightScraper::Repositories::Base.from_hash(:display_name => 'test repo',
-                                       :repo_type    => :download,
-                                       :url          => "file://#{@download_file}",
-                                       :first_credential => ENV['REMOTE_USER'],
-                                       :second_credential => ENV['REMOTE_PASSWORD'])
-    end
-
-    # Create download repository following given layout
-    # Update @repo_path with path to repository
-    # Delete any previously created repo
-    def initialize
-      super
-      @download_repo_path = File.join(@tmpdir, "download")
-      make_cookbooks
-      @download_file = File.join(@tmpdir, "file.tar")
-      Dir.chdir(download_repo_path) do
-        res, status = exec("tar cf \"#{@download_file}\" *")
-        raise "Failed to create tarball: #{res}" unless status.success?
-      end
-    end
-
-    # Cookbook creation method.  Meant to be extended by subclasses.
-    def make_cookbooks
-      create_cookbook(download_repo_path, repo_content)
-    end
-
-    def check_resource(resource, tarball, repository, position=".")
-      resource.should_not == nil
-      resource.repository.tag.should_not == nil
-      resource.repository.repository_hash.should == repository.repository_hash
-      resource.repository.checkout_hash.should_not == repository.checkout_hash
-      resource.pos.should == position
-      resource.metadata.should == repo_content
-      resource.manifest.should == manifest
-    end
-  end
-end

data/spec/download/download_spec.rb

@@ -1,128 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-require File.expand_path(File.join(File.dirname(__FILE__), '..', 'cookbook_helper'))
-
-describe RightScraper::Resources::Cookbook do
-  include RightScraper::SpecHelpers::DevelopmentModeEnvironment
-
-  include RightScraper::CookbookHelper
-
-  shared_examples_for 'a download repository' do
-    it_should_behave_like 'a generic repository'
-
-    it 'should have no tag' do
-      parse_url(@repository)[:tag].should be_nil
-    end
-    it 'should have no position' do
-      parse_url(@repository)[:position].should be_nil
-    end
-    it 'should have no credentials' do
-      parse_url(@repository)[:username].should be_nil
-      parse_url(@repository)[:password].should be_nil
-    end
-  end
-
-  context 'with a download repository' do
-    before(:each) do
-      @repository = RightScraper::Repositories::Base.from_hash(:display_name => 'test repo',
-                                                     :repo_type => :download,
-                                                     :url => "http://a.site/foo/bar/baz",
-                                                     :first_credential => "user",
-                                                     :second_credential => "password")
-    end
-
-    it_should_behave_like 'a download repository'
-
-    it 'should have a cookbook hash' do
-      example_cookbook(@repository).resource_hash.should ==
-        Digest::MD5.hexdigest(RightScraper::Resources::Cookbook::EMPTY_MANIFEST_JSON)
-    end
-
-    it 'should have a cookbook hash invariant under credential changes' do
-      old_hash = example_cookbook(@repository).resource_hash
-      @repository.first_credential = "b-key"
-      example_cookbook(@repository).resource_hash.should == old_hash
-    end
-
-    it 'should have a cookbook hash invariant under position changes' do
-      example_cookbook(@repository, "foo").resource_hash.should ==
-        example_cookbook(@repository, "bar").resource_hash
-    end
-  end
-
-  context 'with a download repository with a port' do
-    before(:each) do
-      @repository = RightScraper::Repositories::Base.from_hash(:display_name => 'test repo',
-                                                     :repo_type => :download,
-                                                     :url => "http://a.site:23/foo/bar/baz",
-                                                     :first_credential => "user",
-                                                     :second_credential => "password")
-    end
-
-    it_should_behave_like 'a download repository'
-  end
-
-  context 'with a download repository with just a user' do
-    before(:each) do
-      @repository = RightScraper::Repositories::Base.from_hash(:display_name => 'test repo',
-                                                     :repo_type => :download,
-                                                     :url => "http://a.site/foo/bar/baz",
-                                                     :first_credential => "user")
-    end
-
-    it_should_behave_like 'a download repository'
-  end
-
-  context 'with a weird download repository' do
-    before(:each) do
-      @repository = RightScraper::Repositories::Base.from_hash(:display_name => 'test repo',
-                                                     :repo_type => :download,
-                                                     :url => "urn:a.site:stuff")
-    end
-
-    it_should_behave_like 'a download repository'
-  end
-
-  context 'with a malicious download repository' do
-    before(:each) do
-      @repository = RightScraper::Repositories::Base.from_hash(:display_name => 'test repo',
-                                                     :repo_type => :download,
-                                                     :url => "http://foo.bar.baz.quux/%20CBLAH",
-                                                     :first_credential => "foo:b/ar",
-                                                     :second_credential => "foo@bar")
-    end
-
-    it_should_behave_like 'a download repository'
-  end
-
-  context 'with a download repository that includes a query' do
-    before(:each) do
-      @repository = RightScraper::Repositories::Base.from_hash(:display_name => 'test repo',
-                                                     :repo_type => :download,
-                                                     :url => "http://foo.bar.baz.quux/stuff?q=bar")
-    end
-
-    it_should_behave_like 'a download repository'
-  end
-
-end