right_scraper 1.0.26 → 3.0.0

data/spec/git/git_scraper_spec_helper.rb

@@ -1,72 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require File.expand_path(File.join(File.dirname(__FILE__), '..', 'scraper_spec_helper_base'))
-
-module RightScale
-
-  # Git implementation of scraper spec helper
-  # See parent class for methods headers comments
-  class GitScraperSpecHelper < ScraperSpecHelperBase
-
-    def setup_test_repo
-      FileUtils.rm_rf(repo_path)
-      FileUtils.mkdir_p(repo_path)
-      Dir.chdir(repo_path) do
-        res, status = exec('git init')
-        raise "Failed to initialize bare git repository: #{res}" unless status.success?
-      end
-      create_file_layout(repo_path, repo_content)
-      commit_content(repo_path)
-    end
-
-    def commit_content(commit_message='commit')
-      Dir.chdir(repo_path) do
-        res, status = exec('git add .')
-        res, status = exec("git commit --quiet -m \"#{commit_message}\"") if status.success?
-        raise "Failed to commit changes from #{repo_path}: #{res}" unless status.success?
-      end
-    end
-
-    def setup_branch(branch, new_content=nil)
-      Dir.chdir(repo_path) do
-        res, status = exec("git checkout -b #{branch}")
-        raise "Failed to setup branch #{branch}: #{res}" unless status.success?
-      end
-      unless new_content.nil?
-        create_file_layout(repo_path, new_content)
-        commit_content("Branch #{branch}")
-      end
-    end
-
-    def commit_id(index_from_last=0)
-      res = nil
-      Dir.chdir(repo_path) do
-        res, status = exec("git log --format=%H -#{index_from_last + 1}")
-        raise "Failed to retrieve commit sha #{index_from_last}: #{res}" unless status.success?
-      end
-      commit_id = res.split("\n").last
-    end
-
-  end
-end

data/spec/rcov.opts

@@ -1 +0,0 @@
---exclude "spec/*"~

data/spec/spec.opts

@@ -1,2 +0,0 @@
---colour
---format=nested

data/spec/svn/svn_scraper_spec.rb

@@ -1,148 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to 
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require File.expand_path(File.join(File.dirname(__FILE__), 'svn_scraper_spec_helper'))
-require 'scraper_base'
-require 'repository'
-require 'watcher'
-require File.join('scrapers', 'svn_scraper')
-require 'set'
-
-describe RightScale::SvnScraper do
-
-  context 'given a SVN repository' do
-
-    before(:each) do
-      @helper = RightScale::SvnScraperSpecHelper.new
-      @helper.setup_test_repo
-      @scrape_dir = File.expand_path(File.join(File.dirname(__FILE__), '__scrape'))
-      @scraper = RightScale::SvnScraper.new(@scrape_dir, max_bytes=1024**2, max_seconds=20)
-      @repo = RightScale::Repository.from_hash(:display_name => 'test repo',
-                                               :repo_type    => :svn,
-                                               :url          => @helper.repo_url)
-      FileUtils.rm_rf(RightScale::ScraperBase.repo_dir(@helper.repo_path, @repo))
-    end
-    
-    after(:each) do
-      @helper.delete_test_repo
-      FileUtils.rm_rf(@helper.svn_repo_path)
-      FileUtils.rm_rf(@scrape_dir)
-    end
-
-    it 'should scrape' do
-      messages = []
-      @scraper.scrape(@repo) { |m, progress| messages << m if progress }
-      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-      @scraper.succeeded?.should be_true
-      messages.size.should == 1
-      File.directory?(@scraper.current_repo_dir).should be_true
-      Set.new(@helper.extract_file_layout(@scraper.current_repo_dir, [ '.svn' ])).should ==
-        Set.new(@helper.repo_content)
-    end
-    
-    it 'should scrape incrementally' do
-	  pending "File URLs comparison on Windows is tricky" if RUBY_PLATFORM=~/mswin/
-      @scraper.scrape(@repo)
-      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-      @scraper.incremental_update?.should be_true
-      @helper.create_file_layout(@helper.repo_path, @helper.additional_content)
-      @helper.commit_content
-      messages = []
-      @scraper.scrape(@repo) { |m, progress| messages << m if progress }
-      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-      @scraper.succeeded?.should be_true
-      messages.size.should == 1
-      File.directory?(@scraper.current_repo_dir).should be_true
-      Set.new(@helper.extract_file_layout(@scraper.current_repo_dir, [ '.svn' ])).should ==
-        Set.new(@helper.repo_content + @helper.additional_content)
-    end
-
-    it 'should only scrape cookbooks directories' do
-      messages = []
-      @repo.cookbooks_path = [ 'folder1', File.join('folder2', 'folder3') ]
-      @scraper.scrape(@repo) { |m, progress| messages << m if progress }
-      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-      @scraper.succeeded?.should be_true
-      messages.size.should == 1
-      File.directory?(@scraper.current_repo_dir).should be_true
-      @helper.extract_file_layout(@scraper.current_repo_dir, [ '.svn' ]).should == [ { 'folder1' => [ 'file2', 'file3' ] }, { 'folder2' => ['folder3' => [ 'file4' ] ] } ]
-    end
-
-    it 'should only scrape cookbooks directories incrementally' do
-      pending "File URLs comparison on Windows is tricky" if RUBY_PLATFORM=~/mswin/
-      @repo.cookbooks_path = [ 'folder1', File.join('folder2', 'folder3') ]
-      @scraper.scrape(@repo)
-      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-      @scraper.incremental_update?.should be_true
-      messages = []
-      @scraper.scrape(@repo) { |m, progress| messages << m if progress }
-      puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-      @scraper.succeeded?.should be_true
-      messages.size.should == 1
-      File.directory?(@scraper.current_repo_dir).should be_true
-      @helper.extract_file_layout(@scraper.current_repo_dir, [ '.svn' ]).should == [ { 'folder1' => [ 'file2', 'file3' ] }, { 'folder2' => ['folder3' => [ 'file4' ] ] } ]
-    end
-
-    context 'and a revision' do
-
-      before(:each) do
-        @helper.create_file_layout(@helper.repo_path, @helper.branch_content)
-        @helper.commit_content
-        @rev_repo = RightScale::Repository.from_hash(:display_name => 'test repo',
-                                                     :repo_type    => :svn,
-                                                     :url          => @helper.repo_url,
-                                                     :tag          => @helper.commit_id(1))
-      end
-
-      it 'should scrape a revision' do
-        messages = []
-        @scraper.scrape(@rev_repo) { |m, progress| messages << m if progress }
-        puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-        @scraper.succeeded?.should be_true
-        messages.size.should == 1
-        File.directory?(@scraper.current_repo_dir).should be_true
-        Set.new(@helper.extract_file_layout(@scraper.current_repo_dir, [ '.svn' ])).should ==
-          Set.new(@helper.repo_content)
-      end
-
-      it 'should scrape a revision incrementally' do
-        @scraper.scrape(@rev_repo)
-        puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-        @scraper.incremental_update?.should be_true
-        @helper.create_file_layout(@helper.repo_path, @helper.additional_content)
-        @helper.commit_content
-        messages = []
-        @scraper.scrape(@rev_repo) { |m, progress| messages << m if progress }
-        puts "\n **ERRORS: #{@scraper.errors.join("\n")}\n" unless @scraper.succeeded?
-        @scraper.succeeded?.should be_true
-        messages.size.should == 1
-        @scraper.instance_variable_get(:@incremental).should == true
-        File.directory?(@scraper.current_repo_dir).should be_true
-        Set.new(@helper.extract_file_layout(@scraper.current_repo_dir, [ '.svn' ])).should ==
-          Set.new(@helper.repo_content)
-      end
-
-    end
-  end
-
-end

data/spec/watcher_spec.rb

@@ -1,74 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require File.join(File.dirname(__FILE__), 'spec_helper')
-require 'watcher'
-
-describe RightScale::Watcher do
-
-  before(:each) do
-    @dest_dir = File.join(File.dirname(__FILE__), '__destdir')
-    FileUtils.mkdir_p(@dest_dir)
-  end
-
-  after(:each) do
-    FileUtils.rm_rf(@dest_dir)
-  end
-
-  it 'should launch and watch well-behaved processes' do
-    watcher = RightScale::Watcher.new(max_bytes=1, max_seconds=5)
-    ruby = "trap('INT', 'IGNORE'); puts 42; exit 42"
-    status = watcher.launch_and_watch('ruby', ['-e', ruby], @dest_dir)
-    status.status.should == :success
-    status.exit_code.should == 42
-    status.output.should == "42\n"
-  end
-
-  it 'should report timeouts' do
-    watcher = RightScale::Watcher.new(max_bytes=1, max_seconds=2)
-    ruby = "trap('INT', 'IGNORE'); STDOUT.sync = true; puts 42; sleep 5"
-    status = watcher.launch_and_watch('ruby',  ['-e', ruby], @dest_dir)
-    status.status.should == :timeout
-    status.exit_code.should == -1
-    status.output.should == "42\n"
-  end
-
-  it 'should report size exceeded' do
-    watcher = RightScale::Watcher.new(max_bytes=1, max_seconds=5)
-    ruby = "trap('INT', 'IGNORE'); STDOUT.sync = true; puts 42; File.open(File.join('#{@dest_dir}', 'test'), 'w') { |f| f.puts 'MORE THAN 2 CHARS' }; sleep 5 rescue nil"
-    status = watcher.launch_and_watch('ruby', ['-e', ruby], @dest_dir)
-    status.status.should == :size_exceeded
-    status.exit_code.should == -1
-    status.output.should == "42\n"
-  end
-
-  it 'should allow infinite size and timeout' do
-    watcher = RightScale::Watcher.new(max_bytes=-1, max_seconds=-1)
-    ruby = "trap('INT', 'IGNORE'); STDOUT.sync = true; puts 42; File.open(File.join('#{@dest_dir}', 'test'), 'w') { |f| f.puts 'MORE THAN 2 CHARS' }; sleep 2 rescue nil"
-    status = watcher.launch_and_watch('ruby', ['-e', ruby], @dest_dir)
-    status.status.should == :success
-    status.exit_code.should == 0
-    status.output.should == "42\n"
-  end
-
-end