right_scraper 3.2.6 → 5.0.1

data/spec/logger_spec.rb

@@ -1,185 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require 'stringio'
-require File.expand_path(File.join(File.dirname(__FILE__), 'spec_helper'))
-require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'right_scraper', 'loggers', 'noisy'))
-
-describe RightScraper::Logger do
-  before(:each) do
-    @stream = StringIO.new()
-    @logger = RightScraper::Logger.new(@stream)
-    @logger.level = Logger::WARN
-  end
-
-  after(:each) do
-    @logger.close
-  end
-
-  def match_log_entry(entry, type, message)
-    shortform, longform = case type
-                          when Logger::DEBUG then ["D", "DEBUG"]
-                          when Logger::INFO then ["I", "INFO"]
-                          when Logger::WARN then ["W", "WARN"]
-                          when Logger::ERROR then ["E", "ERROR"]
-                          when Logger::FATAL then ["F", "FATAL"]
-                          when Logger::FATAL then ["U", "UNKNOWN"]
-                          end
-    datestamp = /[[:digit:]]{4}-[[:digit:]]{2}-[[:digit:]]{2}/
-    timestamp = /[[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}.[[:digit:]]+/
-    datetimestamp = /#{datestamp}T#{timestamp}/
-    entry.should =~ %r{^#{shortform}, \[#{datetimestamp} \#[[:digit:]]+\] #{longform} -- : #{message}\n$}
-  end
-
-  it 'should log to the stream' do
-    @logger.debug("foo")
-    @logger.info("first")
-    @logger.error("baz")
-    @logger.fatal("quux")
-    @stream.rewind
-    match_log_entry(@stream.readline, Logger::ERROR, "baz")
-    match_log_entry(@stream.readline, Logger::FATAL, "quux")
-    @stream.eof?.should be_true
-  end
-
-  it 'should note errors properly' do
-    @logger.note_error(Exception.new("foo!"), :baz)
-    match_log_entry(@stream.string, Logger::ERROR, "Saw foo! during baz")
-  end
-
-  it 'should note errors with an explanation properly' do
-    @logger.note_error(Exception.new("foo!"), :baz, "an explanation")
-    match_log_entry(@stream.string, Logger::ERROR, "Saw foo! during baz: an explanation")
-  end
-
-  it 'should pass values through the block' do
-    result = @logger.operation(:passthrough) { 4 }
-    result.should == 4
-  end
-end
-
-describe RightScraper::Loggers::NoisyLogger do
-  before(:each) do
-    @stream = StringIO.new()
-    @logger = RightScraper::Loggers::NoisyLogger.new(@stream)
-  end
-
-  after(:each) do
-    @logger.close
-  end
-
-  def match_log_entry(entry, type, message)
-    shortform, longform = case type
-                          when Logger::DEBUG then ["D", "DEBUG"]
-                          when Logger::INFO then ["I", "INFO"]
-                          when Logger::WARN then ["W", "WARN"]
-                          when Logger::ERROR then ["E", "ERROR"]
-                          when Logger::FATAL then ["F", "FATAL"]
-                          when Logger::FATAL then ["U", "UNKNOWN"]
-                          end
-    datestamp = /[[:digit:]]{4}-[[:digit:]]{2}-[[:digit:]]{2}/
-    timestamp = /[[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}.[[:digit:]]+/
-    datetimestamp = /#{datestamp}T#{timestamp}/
-    entry.should =~ %r{^#{shortform}, \[#{datetimestamp} \#[[:digit:]]+\] #{longform} -- : #{message}\n$}
-  end
-
-  context 'with the level set to WARN' do
-    before(:each) do
-      @logger.level = Logger::WARN
-    end
-
-    it 'should not log operations' do
-      @logger.operation(:foo) do
-        @logger.operation(:bar) do
-        end
-      end
-      @stream.string.should == ""
-    end
-
-    it 'should note errors with context' do
-      begin
-        @logger.operation(:foo) do
-          @logger.operation(:bar, "blah") do
-            @logger.note_error("an exception", :note, "bar")
-          end
-        end
-      rescue
-      end
-      @stream.rewind
-      match_log_entry(@stream.readline, Logger::ERROR, "Saw an exception during note: bar in bar: blah in foo")
-      @stream.eof?.should be_true
-    end
-
-    it 'should note errors when the level is set to WARN and an exception occurs' do
-      begin
-        @logger.operation(:foo) do
-          @logger.operation(:bar, "blah") do
-            raise "foo"
-          end
-        end
-      rescue
-      end
-      @stream.rewind
-      match_log_entry(@stream.readline, Logger::ERROR, "Saw foo during bar: blah in foo")
-      @stream.eof?.should be_true
-    end
-  end
-
-  context 'with the level set to DEBUG' do
-    before(:each) do
-      @logger.level = Logger::DEBUG
-    end
-
-    it 'should log begin/commit' do
-      @logger.operation(:foo) do
-        @logger.operation(:bar, "blah") do
-        end
-      end
-      @stream.rewind
-      match_log_entry(@stream.readline, Logger::DEBUG, "> begin foo")
-      match_log_entry(@stream.readline, Logger::DEBUG, ">> begin bar: blah")
-      match_log_entry(@stream.readline, Logger::DEBUG, ">> commit bar: blah")
-      match_log_entry(@stream.readline, Logger::DEBUG, "> commit foo")
-      @stream.eof?.should be_true
-    end
-
-    it 'should log begin/abort when an exception occurs' do
-      @logger.level = Logger::DEBUG
-      begin
-        @logger.operation(:foo) do
-          @logger.operation(:bar, "blah") do
-            raise "foo"
-          end
-        end
-      rescue
-      end
-      @stream.rewind
-      match_log_entry(@stream.readline, Logger::DEBUG, "> begin foo")
-      match_log_entry(@stream.readline, Logger::DEBUG, ">> begin bar: blah")
-      match_log_entry(@stream.readline, Logger::ERROR, "Saw foo during bar: blah in foo")
-      match_log_entry(@stream.readline, Logger::DEBUG, ">> abort bar: blah")
-      match_log_entry(@stream.readline, Logger::DEBUG, "> abort foo")
-      @stream.eof?.should be_true
-    end
-  end
-end

data/spec/repository_spec.rb

@@ -1,111 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require File.expand_path(File.join(File.dirname(__FILE__), 'spec_helper'))
-require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'right_scraper', 'repositories', 'mock'))
-
-describe RightScraper::Repositories::Base do
-  include RightScraper::SpecHelpers::DevelopmentModeEnvironment
-
-  context 'with a repository type that doesn\'t exist' do
-    it 'should throw a comprehensible error when you try to create it' do
-      lambda {
-        RightScraper::Repositories::Base.from_hash(:display_name      => 'display_name',
-                                           :repo_type         => :nonexistent,
-                                           :url               => 'url',
-                                           :tag               => 'tag',
-                                           :first_credential  => 'first_credential',
-                                           :second_credential => 'second_credential')
-      }.should raise_error(/Can't understand how to make nonexistent repos/)
-    end
-  end
-
-  context 'with a mock repository' do
-    before(:each) do
-      @repo = RightScraper::Repositories::Base.from_hash(:display_name      => 'display_name',
-                                                 :repo_type         => :mock,
-                                                 :url               => 'url',
-                                                 :tag               => 'tag',
-                                                 :first_credential  => 'first_credential',
-                                                 :second_credential => 'second_credential')
-    end
-
-    it 'should be initializable from a hash' do
-      @repo.should be_kind_of(RightScraper::Repositories::Base)
-      @repo.display_name.should      == 'display_name'
-      @repo.repo_type.should         == :mock
-      @repo.url.should               == 'url'
-      @repo.tag.should               == 'tag'
-      @repo.first_credential.should  == 'first_credential'
-      @repo.second_credential.should == 'second_credential'
-    end
-
-    it 'should know the SHA-1 of its root location' do
-      @repo.repository_hash.should ==
-        Digest::SHA1.hexdigest("1\000mock\000url")
-    end
-
-    it 'should know the SHA-1 of the identifier for this specific checkout' do
-      @repo.checkout_hash.should ==
-        Digest::SHA1.hexdigest("1\000mock\000url")
-    end
-  end
-end
-
-describe RightScraper::Repositories::Download do
-  include RightScraper::SpecHelpers::DevelopmentModeEnvironment
-
-  before(:each) do
-    @repo = RightScraper::Repositories::Base.from_hash(:display_name => 'test repo',
-                                               :repo_type => :download,
-                                               :url => "http://foo.bar.baz.quux/%20CBLAH",
-                                               :tag => "412530982323",
-                                               :first_credential => "foo:b/ar",
-                                               :second_credential => "foo@bar")
-  end
-
-  it 'should have a tag' do
-    @repo.tag.should == '412530982323'
-  end
-
-  it 'should include the tag in the checkout hash' do
-    @repo.checkout_hash.should_not == @repo.repository_hash
-    oldhash = @repo.checkout_hash
-    @repo.tag = "42398"
-    @repo.checkout_hash.should_not == oldhash
-  end
-
-  it 'should have the same repository hash with or without credentials' do
-    initial_hash = @repo.repository_hash
-    @repo.first_credential = nil
-    @repo.second_credential = nil
-    @repo.repository_hash.should == initial_hash
-  end
-
-  it 'should have the same checkout hash with or without credentials' do
-    initial_hash = @repo.checkout_hash
-    @repo.first_credential = nil
-    @repo.second_credential = nil
-    @repo.checkout_hash.should == initial_hash
-  end
-end

data/spec/retriever_spec_helper.rb

@@ -1,146 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require 'digest/sha1'
-require File.expand_path(File.join(File.dirname(__FILE__), 'spec_helper'))
-require 'tmpdir'
-
-module RightScraper
-
-  # Base class for all scrapers spec helpers
-  # Define helper methods used to manage repositories using each
-  # source control software
-  class RetrieverSpecHelper
-
-    include SpecHelpers
-
-    def initialize
-      @tmpdir = Dir.mktmpdir
-      FileUtils.mkdir(repo_path)
-      @repo_content = [ { 'folder1' => [ 'file2', 'file3' ] },
-                        { 'folder2' => [ { 'folder3' => [ 'file4' ] } ] },
-                        'file1' ]
-    end
-
-    def close
-      FileUtils.remove_entry_secure @tmpdir
-    end
-
-    # Path to test repository
-    #
-    # === Return
-    # repo_path(String):: Path to test repository
-    def repo_path
-      File.join(@tmpdir, "repository")
-    end
-
-    # Default test repo content
-    #
-    # === Return
-    # content(String):: Default test repo content
-    attr_reader :repo_content
-
-    def manifest
-      hash = {}
-      scan(@repo_content, hash, nil)
-      hash['metadata.json'] = Digest::MD5.hexdigest(@repo_content.to_json + "\n")
-      hash
-    end
-
-    def scan(contents, hash, position)
-      contents.each do |object|
-        if object.instance_of?(Hash)
-          object.each do |key, value|
-            relative_position = position ? File.join(position, key) : key
-            scan(value, hash, relative_position)
-          end
-        else
-          relative_position = position ? File.join(position, object) : object
-          hash[relative_position] = Digest::MD5.hexdigest(object + "\n")
-        end
-      end
-    end
-    private :scan
-
-    # Test branch content
-    #
-    # === Return
-    # content(String):: Branch content
-    def branch_content
-      content = [ { 'branch_folder' => [ 'bfile1', 'bfile2' ] }, 'bfile3' ]
-    end
-
-    # Additional content used to test incremental updates
-    #
-    # === Return
-    # content(String):: Additional content
-    def additional_content
-      content = [ { 'additional_folder' => [ 'afile1', 'afile2' ] }, 'afile3' ]
-    end
-
-    # Commit any non-commited changes of given directory
-    #
-    # === Parameters
-    # repo_path(String):: Path to directory where commit should be created
-    # commit_message(String):: Optional, commit message
-    #
-    # === Raise
-    # Exception:: If commit command fails
-    def commit_content(repo_path, commit_message='Initial commit')
-      raise 'Not implemented'
-    end
-
-    # Create a branch in given repository
-    # Optionally adds a new commit with given file layout
-    # Switch to branch so that next call to 'create_file_layout' will act
-    # on given branch
-    #
-    # === Parameters
-    # branch(String):: Name of branch that should be created
-    # new_content(Hash):: Layout of files to be added to branch as a single commit
-    #
-    # === Return
-    # true:: Always return true
-    #
-    # === Raise
-    # Exception:: If branch command fails
-    def setup_branch(branch, new_content=nil)
-      raise 'Not implemented'
-    end
-
-    # Commit id for commit in test repo
-    # i.e. git sha or svn rev
-    #
-    # === Parameters
-    # index_from_last(Integer):: Commit whose id should be returned:
-    #                              - 0 means last commit
-    #                              - 1 means 1 before last
-    #                              - etc.
-    #
-    # === Return
-    # commit_id(String):: Corresponding commit id
-    def commit_id(index_from_last=0)
-      raise 'Not implemented'
-    end
-  end
-end

data/spec/scanner_spec.rb

@@ -1,61 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require File.expand_path(File.join(File.dirname(__FILE__), 'spec_helper'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'download', 'download_retriever_spec_helper'))
-
-describe RightScraper::Scanners::Base do
-  include RightScraper::SpecHelpers::DevelopmentModeEnvironment
-
-  before(:each) do
-    @helper = RightScraper::DownloadRetrieverSpecHelper.new
-    @repo = @helper.repo
-  end
-
-  after(:each) do
-    @helper.close
-  end
-
-  it 'should be called correctly' do
-    scanner = flexmock("scanner")
-    scanner.should_receive(:new).with(Hash).once.and_return(scanner)
-    scanner.should_receive(:begin).with(RightScraper::Resources::Cookbook).once
-    scanner.should_receive(:notice_dir).with(nil).once.and_return(true)
-    scanner.should_receive(:notice).with("file1", Proc).once
-    scanner.should_receive(:notice_dir).with("folder1").once.and_return(true)
-    scanner.should_receive(:notice).with("folder1/file2", Proc).once
-    scanner.should_receive(:notice).with("folder1/file3", Proc).once
-    scanner.should_receive(:notice_dir).with("folder2").once.and_return(false)
-    scanner.should_receive(:notice).with("metadata.json", Proc).once
-    scanner.should_receive(:end).with(RightScraper::Resources::Cookbook).once
-    scanner.should_receive(:finish).with().once
-
-    @scraper = RightScraper::Scrapers::Base.scraper(:repo_dir   => @helper.download_repo_path,
-                                                    :kind       => :cookbook,
-                                                    :scanners   => [scanner],
-                                                    :repository => @repo)
-    @scraper.next_resource.should_not be_nil
-    @scraper.next_resource.should be_nil
-    @scraper.close
-  end
-end