right_scraper 3.2.6 → 5.0.1

data/spec/scraper_helper.rb

@@ -1,88 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require File.expand_path(File.join(File.dirname(__FILE__), 'spec_helper'))
-require 'tmpdir'
-
-module RightScraper
-  module SpecHelpers
-    module FromScratchScraping
-      def FromScratchScraping.included(mod)
-        mod.module_eval do
-          before(:each) do
-            @basedir = Dir.mktmpdir
-            @retriever = @retriever_class.new(@repo,
-                                         :basedir => @basedir,
-                                         :max_bytes => 1024**2,
-                                         :max_seconds => 20)
-            @retriever.retrieve
-          end
-
-          after(:each) do
-            if @basedir && File.directory?(@basedir)
-              FileUtils.remove_entry_secure(@basedir)
-            end
-            @scraper = nil
-          end
-        end
-      end
-    end
-
-    module CookbookScraping
-      def CookbookScraping.included(mod)
-        mod.module_eval do
-          before(:each) do
-            @scraper = RightScraper::Scrapers::Base.scraper(:repo_dir        => @retriever.repo_dir,
-                                                            :kind            => :cookbook,
-                                                            :repository      => @retriever.repository,
-                                                            :ignorable_paths => @retriever.ignorable_paths)
-          end
-        end
-      end
-    end
-
-    module WorkflowScraping
-      def WorkflowScraping.included(mod)
-        mod.module_eval do
-          before(:each) do
-            @scraper = RightScraper::Scrapers::Base.scraper(:repo_dir        => @retriever.repo_dir,
-                                                            :kind            => :workflow,
-                                                            :repository      => @retriever.repository,
-                                                            :ignorable_paths => @retriever.ignorable_paths)
-          end
-        end
-      end
-    end
-
-  end
-  module ScraperHelper
-    def check_resource(resource, params={})
-      position = params[:position] || "."
-      resource.should_not == nil
-      resource.repository.should be_an_equal_repo @repo
-      resource.pos.should == position
-      resource.metadata.should == (params[:metadata] || @helper.repo_content)
-      resource.manifest.should == (params[:manifest] || @helper.manifest)
-    end
-  end
-end

data/spec/scraper_spec.rb

@@ -1,147 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require File.expand_path(File.join(File.dirname(__FILE__), 'spec_helper'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'download', 'download_retriever_spec_helper'))
-require 'tmpdir'
-require 'flexmock'
-
-describe RightScraper::Scraper do
-  include RightScraper::SpecHelpers::DevelopmentModeEnvironment
-
-  include RightScraper::SharedExamples
-
-  shared_examples_for 'scrapes to given base dir' do
-    before(:each) do
-      @scraper = RightScraper::Scraper.new(:basedir => @tmpdir, :kind => :cookbook)
-    end
-
-    after(:each) do
-      FileUtils.remove_entry_secure @tmpdir unless @tmpdir.nil?
-    end
-
-    it 'starts out successful' do
-      @scraper.succeeded?.should be_true
-      @scraper.errors.should == []
-    end
-
-    context 'given a legal download repository' do
-      before(:each) do
-        @helper = RightScraper::DownloadRetrieverSpecHelper.new
-        @repo = @helper.repo
-      end
-
-      after(:each) do
-        @helper.close
-      end
-
-      it_should_behave_like "a normal repository"
-
-      it 'should log correctly as it scrapes' do
-        callback = flexmock("callback")
-        callback.should_receive(:call).with(:begin, :retrieving, "from #{@repo}", nil).once.ordered
-        callback.should_receive(:call).with(:begin, :initialize, String, nil).once.ordered
-        callback.should_receive(:call).with(:commit, :initialize, String, nil).once.ordered
-        callback.should_receive(:call).with(:begin, :downloading, "", nil).once.ordered
-        callback.should_receive(:call).with(:begin, :running_command, String, nil).once.ordered
-        callback.should_receive(:call).with(:commit, :running_command, String, nil).once.ordered
-        callback.should_receive(:call).with(:commit, :downloading, "", nil).once.ordered
-        callback.should_receive(:call).with(:begin, :unpacking, "", nil).once.ordered
-        callback.should_receive(:call).with(:begin, :running_command, String, nil).once.ordered
-        callback.should_receive(:call).with(:commit, :running_command, String, nil).once.ordered
-        callback.should_receive(:call).with(:commit, :unpacking, "", nil).once.ordered
-        callback.should_receive(:call).with(:commit, :retrieving, "from #{@repo}", nil).once.ordered
-        callback.should_receive(:call).with(:begin, :scraping, String, nil).once.ordered
-        callback.should_receive(:call).with(:begin, :finding_next_cookbook, String, nil).once.ordered
-        callback.should_receive(:call).with(:begin, :reading_cookbook, String, nil).once.ordered
-        callback.should_receive(:call).with(:begin, :scanning_filesystem, String, nil).once.ordered
-        callback.should_receive(:call).with(:begin, :metadata_parsing, "", nil).once.ordered
-        callback.should_receive(:call).with(:commit, :metadata_parsing, "", nil).once.ordered
-        callback.should_receive(:call).with(:commit, :scanning_filesystem, String, nil).once.ordered
-        callback.should_receive(:call).with(:commit, :reading_cookbook, String, nil).once.ordered
-        callback.should_receive(:call).with(:commit, :finding_next_cookbook, String, nil).once.ordered
-        callback.should_receive(:call).with(:begin, :next, "", nil).once.ordered
-        callback.should_receive(:call).with(:begin, :searching, "", nil).once.ordered
-        callback.should_receive(:call).with(:commit, :searching, "", nil).once.ordered
-        callback.should_receive(:call).with(:commit, :next, "", nil).once.ordered
-        callback.should_receive(:call).with(:begin, :next, "", nil).once.ordered
-        callback.should_receive(:call).with(:commit, :next, "", nil).once.ordered
-        callback.should_receive(:call).with(:commit, :scraping, String, nil).once.ordered
-        @scraper.scrape(@repo) do |phase, operation, explanation, exception|
-          callback.call(phase, operation, explanation, exception)
-        end
-        @scraper.errors.should == []
-        @scraper.succeeded?.should be_true
-        @scraper.resources.size.should == 1
-      end
-    end
-
-    context 'given several repositories' do
-      it 'should continue to scrape even if errors occur' do
-        GC.start
-        repo = RightScraper::Repositories::Base.from_hash(:display_name => 'illegal repo',
-                                                          :repo_type    => :download,
-                                                          :url          => "http://example.com/foo")
-        @scraper.scrape(repo)
-        helpers = [RightScraper::DownloadRetrieverSpecHelper,
-                   RightScraper::DownloadRetrieverSpecHelper,
-                   RightScraper::DownloadRetrieverSpecHelper]
-        helpers.each do |klass|
-          helper = klass.new
-          @scraper.scrape(helper.repo)
-          helper.close
-        end
-        @scraper.succeeded?.should be_false
-        @scraper.resources.size.should == 3
-        @scraper.errors.size.should == 1
-      end
-    end
-
-    it 'catches normal logging behavior' do
-      logger = @scraper.instance_variable_get(:@logger)
-      logger.should_not be_nil
-      logger.info("foo")
-      logger.error("foo")
-      @scraper.succeeded?.should be_false
-      @scraper.errors.should == [[nil, :log, {:severity => Logger::ERROR,
-                                       :message => "foo",
-                                       :progname => nil}]]
-    end
-  end
-
-  context 'when base dir is not provided' do
-    before(:each) do
-      @tmpdir = nil
-    end
-
-    it_should_behave_like 'scrapes to given base dir'
-  end
-
-  context 'when base dir is provided' do
-    before(:each) do
-      @tmpdir = Dir.mktmpdir
-    end
-
-    it_should_behave_like 'scrapes to given base dir'
-  end
-end

data/spec/spec_helper.rb

@@ -1,185 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require 'rubygems'
-require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'right_scraper'))
-
-require 'flexmock'
-require 'rspec'
-require 'find'
-require 'json'
-
-RSpec.configure do |c|
-  c.mock_with(:flexmock)
-end
-
-ENV["DEVELOPMENT"] ||= "yes"
-
-# Helper module
-module RightScraper
-
-  module SpecHelpers
-    module DevelopmentModeEnvironment
-      def DevelopmentModeEnvironment.included(mod)
-        mod.module_eval do
-          before(:each) do
-            @oldtest = ENV['DEVELOPMENT']
-            ENV['DEVELOPMENT'] = "yes"
-          end
-          after(:each) do
-            if @oldtest.nil?
-              ENV.delete('DEVELOPMENT')
-            else
-              ENV['DEVELOPMENT'] = @oldtest
-            end
-          end
-        end
-      end
-    end
-    module ProductionModeEnvironment
-      def ProductionModeEnvironment.included(mod)
-        mod.module_eval do
-          before(:each) do
-            @oldtest = ENV['DEVELOPMENT']
-            ENV.delete('DEVELOPMENT')
-          end
-          after(:each) do
-            if @oldtest.nil?
-              ENV.delete('DEVELOPMENT')
-            else
-              ENV['DEVELOPMENT'] = @oldtest
-            end
-          end
-        end
-      end
-    end
-
-    # Set the 'verbose' environment variable for debugging a failing spec
-    VERBOSE='verbose'
-
-    # Execute given shell command and return output and exit code
-    # Allows centralizing logging/output
-    #
-    # === Parameters
-    # cmd(String):: Command to be run
-    #
-    # === Return
-    # res, process status(Array):: Pair whose first element is the output of the command
-    #                              and second element is the process exit status
-    def exec(cmd)
-      puts "+ [#{Dir.pwd}] #{cmd}" if ENV[VERBOSE]
-      res = `#{cmd} 2>&1`
-      puts res unless res.empty? if ENV[VERBOSE]
-      return res, $?
-    end
-
-    def create_cookbook(path, contents)
-      create_file_layout(path, contents)
-      File.open(File.join(path, 'metadata.json'), 'w') { |f| f.puts contents.to_json }
-    end
-
-    def create_workflow(path, name, definition, metadata)
-      File.open(File.join(path, "#{name}#{RightScraper::Resources::Workflow::DEFINITION_EXT}"), 'w') { |f| f.puts definition }
-      File.open(File.join(path, "#{name}#{RightScraper::Resources::Workflow::METADATA_EXT}"), 'w') { |f| f.puts metadata.to_json }
-    end
-
-    # Create file layout from given array
-    # Strings in array correspond to files while Hashes correspond to folders
-    # File content is equal to filename
-    #
-    # === Parameters
-    # Path(String):: Path where layout should be created
-    # layout(Array):: Describe the file layout to be created
-    #
-    # === Return
-    # true:: Always return true
-    def create_file_layout(path, layout)
-      FileUtils.mkdir_p(path)
-      layout.each do |elem|
-        if elem.is_a?(Hash)
-          elem.each do |k, v|
-            full_path = File.join(path, k)
-            FileUtils.mkdir_p(full_path)
-            create_file_layout(full_path, v)
-          end
-        else
-          File.open(File.join(path, elem.to_s), 'w') { |f| f.puts elem.to_s }
-        end
-      end
-      true
-    end
-
-    # Extract array representing file layout for given directory
-    #
-    # === Parameters
-    # path(String):: Path to directory whose layout is to be retrieved
-    # layout(Array):: Array being updated with layout, same as return value, empty array by default
-    # ignore(Array):: Optional: Name of files or directories that should be ignored
-    #
-    # === Return
-    # layout(Array):: Corresponding layout as used by 'create_file_layout'
-    def extract_file_layout(path, ignore=[])
-      return [] unless File.directory?(path)
-      dirs = []
-      files = []
-      ignore += [ '.', '..' ]
-      Dir.foreach(path) do |f|
-        next if ignore.include?(f)
-        full_path = File.join(path, f)
-        if File.directory?(full_path)
-          dirs << { f => extract_file_layout(full_path, ignore) }
-        else
-          files << f
-        end
-      end
-      dirs + files.sort
-    end
-
-    RSpec::Matchers.define :begin_with do |path|
-      match do |directory|
-        directory[0...path.length] == path
-      end
-    end
-  end
-
-  module SharedExamples
-    shared_examples_for "a normal repository" do
-      it 'should scrape' do
-        @scraper.scrape(@repo)
-        @scraper.succeeded?.should be_true
-        @scraper.resources.should_not == []
-        @scraper.resources.size.should == 1
-        @scraper.resources[0].manifest.should == {
-                "folder1/file3"=>"60b91f1875424d3b4322b0fdd0529d5d",
-                "file1"=>"5149d403009a139c7e085405ef762e1a",
-                "folder2/folder3/file4"=>"857c6673d7149465c8ced446769b523c",
-                "metadata.json"=>"7c72b234162002a96f4ba60f0db38601",
-                "folder1/file2"=>"3d709e89c8ce201e3c928eb917989aef"}
-        @scraper.resources[0].metadata.should == [{"folder1"=>["file2", "file3"]},
-                                                  {"folder2"=>[{"folder3"=>["file4"]}]},
-                                                  "file1"]
-      end
-    end
-  end
-
-end

data/spec/svn/cookbook_spec.rb

@@ -1,96 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec_helper'))
-require File.expand_path(File.join(File.dirname(__FILE__), '..', 'cookbook_helper'))
-
-describe RightScraper::Resources::Cookbook do
-  include RightScraper::SpecHelpers::DevelopmentModeEnvironment
-
-  include RightScraper::CookbookHelper
-
-  shared_examples_for 'svn repositories' do
-    it_should_behave_like 'a generic repository'
-
-    it 'should have the appropriate credentials' do
-      @repository.username.should == 'username'
-      @repository.password.should == 'password'
-    end
-  end
-
-  context 'with a SVN repository' do
-    before(:each) do
-      @repository = RightScraper::Repositories::Svn.from_hash(:display_name => 'test repo',
-                                                     :repo_type => :svn,
-                                                     :url => "http://a.site/foo/bar/baz",
-                                                     :first_credential => "username",
-                                                     :second_credential => "password")
-    end
-
-    it_should_behave_like 'svn repositories'
-
-    it 'should have a tag' do
-      @repository.tag.should == "HEAD"
-    end
-
-    it 'should have a cookbook hash' do
-      example_cookbook(@repository).resource_hash.should ==
-        Digest::MD5.hexdigest(RightScraper::Resources::Cookbook::EMPTY_MANIFEST_JSON)
-    end
-
-    it 'should have a cookbook hash invariant under credential changes' do
-      old_hash = example_cookbook(@repository).resource_hash
-      @repository.first_credential = "b-key"
-      example_cookbook(@repository).resource_hash.should == old_hash
-    end
-
-    it 'should have a cookbook hash invariant under tag changes' do
-      old_hash = example_cookbook(@repository).resource_hash
-      @repository.tag = "tag"
-      example_cookbook(@repository).resource_hash.should == old_hash
-    end
-
-    it 'should have a cookbook hash invariant under position changes' do
-      example_cookbook(@repository, "foo").resource_hash.should ==
-        example_cookbook(@repository, "bar").resource_hash
-    end
-  end
-
-  context 'with a SVN repository with a tag' do
-    before(:each) do
-      @repository = RightScraper::Repositories::Svn.from_hash(:display_name => 'test repo',
-                                                     :repo_type => :svn,
-                                                     :url => "http://a.site/foo/bar/baz",
-                                                     :tag => "foo",
-                                                     :first_credential => "username",
-                                                     :second_credential => "password")
-    end
-
-    it_should_behave_like 'svn repositories'
-
-    it 'should have a tag' do
-      @repository.tag.should == "foo"
-    end
-  end
-
-end