right_scraper 3.2.6 → 5.0.1

data/lib/right_scraper/scraper_logger.rb

@@ -1,66 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-module RightScraper
-
-  class ScraperLogger < Logger
-    attr_accessor :callback, :errors, :warnings
-
-    def add(severity, message=nil, progname=nil)
-      if severity >= (self.level || Logger::WARN)
-        if message.nil?
-          if block_given?
-            message = yield
-          else
-            message = progname
-            progname = self.progname
-          end
-        end
-        @errors << [nil, :log,
-          {:severity => severity,
-            :message => message,
-            :progname => progname}]
-      end
-    end
-
-    def initialize
-      @errors = []
-      @warnings = []
-    end
-
-    def note_phase(phase, type, explanation, exception=nil)
-      @callback.call(phase, type, explanation, exception) unless @callback.nil?
-      super
-    end
-
-    def note_error(exception, type, explanation="")
-      @errors << [exception, type, explanation]
-    end
-
-    def note_warning(message)
-      @warnings << message
-    end
-
-  end
-
-end

data/lib/right_scraper/svn_client.rb

@@ -1,164 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010-2013 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require 'right_popen'
-require 'right_popen/safe_output_buffer'
-
-module RightScraper
-  # Simplified interface to the process of creating SVN client
-  # contexts.
-  #
-  # SVN client contexts are needed for almost every nontrivial SVN
-  # operation, and the authentication procedure is truly baroque.
-  # Thus, when you need a client context, do something like this:
-  #   client = SvnClient.new(repository)
-  #   client.with_context do |ctx|
-  #     ...
-  #   end
-  module SvnClient
-
-    class SvnClientError < Exception; end
-
-    def calculate_version
-      unless @svn_version
-        begin
-          cmd = 'svn --version --quiet'
-          out = `#{cmd}`
-          if $?.success?
-            @svn_version = out.chomp.split('.').map {|e| e.to_i}
-          else
-            raise SvnClientError, "Unable to determine svn version: #{cmd.inspect} exited with #{$?.exitstatus}"
-          end
-        rescue Errno::ENOENT => e
-          raise SvnClientError, "Unable to determine svn version: #{e.message}"
-        end
-      end
-      @svn_version
-    end
-
-    def svn_arguments
-      version = calculate_version
-      case
-      when version[0] != 1
-        raise "SVN major revision is not 1, cannot be sure it will run properly."
-      when version[1] < 4
-        raise "SVN minor revision < 4; cannot be sure it will run properly."
-      when version[1] < 6
-        # --trust-server-cert is a 1.6ism
-        args = ["--no-auth-cache", "--non-interactive"]
-      else
-        args = ["--no-auth-cache", "--non-interactive", "--trust-server-cert"]
-      end
-      if repository.first_credential && repository.second_credential
-        args << "--username"
-        args << repository.first_credential
-        args << "--password"
-        args << repository.second_credential
-      end
-      args
-    end
-
-    def get_tag_argument
-      if repository.tag
-        tag_cmd = ["-r", get_tag.to_s]
-      else
-        tag_cmd = ["-r", "HEAD"]
-      end
-    end
-
-    def run_svn_no_chdir(*args)
-      run_svn_with(nil, :safe_output_svn_client, *args)
-    end
-
-    def run_svn(*args)
-      run_svn_with(repo_dir, :safe_output_svn_client, *args)
-    end
-
-    def run_svn_with_buffered_output(*args)
-      run_svn_with(repo_dir, :unsafe_output_svn_client, *args)
-    end
-
-    # runs svn client with safe buffering (by default).
-    #
-    # === Parameters
-    # @param [Array] args for svn client command line
-    #
-    # === Return
-    # @return [Array] lines of output or empty
-    def run_svn_with(initial_directory, output_handler, *args)
-      @output = ::RightScale::RightPopen::SafeOutputBuffer.new
-      @output_handler = output_handler
-      cmd = ['svn', args, svn_arguments].flatten
-      ::RightScale::RightPopen.popen3_sync(
-        cmd,
-        :target             => self,
-        :directory          => initial_directory,
-        :timeout_handler    => :timeout_svn_client,
-        :size_limit_handler => :size_limit_svn_client,
-        :exit_handler       => :exit_svn_client,
-        :stderr_handler     => output_handler,
-        :stdout_handler     => output_handler,
-        :inherit_io         => true,  # avoid killing any rails connection
-        :watch_directory    => repo_dir,
-        :size_limit_bytes   => @max_bytes,
-        :timeout_seconds    => @max_seconds)
-      @output.buffer
-    end
-
-    def safe_output_svn_client(data)
-      @output.safe_buffer_data(data)
-    end
-
-    def unsafe_output_svn_client(data)
-      @output.buffer << data.chomp
-    end
-
-    def timeout_svn_client
-      raise SvnClientError, "svn client timed out"
-    end
-
-    def size_limit_svn_client
-      raise SvnClientError, "svn client exceeded size limit"
-    end
-
-    def exit_svn_client(status)
-      unless status.success?
-        self.method(@output_handler).call("Exit code = #{status.exitstatus}")
-        raise SvnClientError, "svn client failed: #{@output.display_text}"
-      end
-      true
-    end
-
-    # Fetch the tag from the repository, or nil if one doesn't
-    # exist.  This is a separate method because the repo tag should
-    # be a number but is a string in the database.
-    def get_tag
-      case repository.tag
-      when Fixnum then repository.tag
-      when /^\d+$/ then repository.tag.to_i
-      else
-        repository.tag
-      end
-    end
-  end
-end

data/right_scraper.rconf

@@ -1,13 +0,0 @@
-# RConf configuration file, usage:
-#   gem install rconf
-#   rconf
-#
-ruby do
-  version  'ruby-1.9.2-p290'
-  rubygems '1.8.10'
-  gemset   'right_scraper'
-end
-bundler do
-  version     '1.2.1'
-  bundle_path File.join(ENV["HOME"], '.rightscale', 'right_scraper')
-end

data/spec/builder_spec.rb

@@ -1,50 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require File.expand_path(File.join(File.dirname(__FILE__), 'spec_helper'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'download', 'download_retriever_spec_helper'))
-
-describe RightScraper::Builders::Builder do
-
-  include RightScraper::SpecHelpers::DevelopmentModeEnvironment
-  
-  before(:each) do
-    @helper = RightScraper::DownloadRetrieverSpecHelper.new
-    @repo_dir = @helper.download_repo_path
-  end
-
-  it 'should be called correctly' do
-    builder = flexmock("builder")
-    builder.should_receive(:new).with(Hash).once.and_return(builder)
-    builder.should_receive(:go).with(String, RightScraper::Resources::Cookbook).once
-    builder.should_receive(:finish).with().once
-
-    @scraper = RightScraper::Scrapers::Cookbook.new(:repository => @helper.repo,
-                                                    :repo_dir   => @repo_dir,
-                                                    :builders   => [builder])
-    @scraper.next_resource.should_not be_nil
-    @scraper.next_resource.should be_nil
-    @scraper.close
-  end
-
-end

data/spec/cookbook_helper.rb

@@ -1,73 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require File.expand_path(File.join(File.dirname(__FILE__), 'retriever_spec_helper'))
-require 'uri'
-require 'cgi'
-
-module RightScraper
-  module CookbookHelper
-
-   def example_cookbook(repository, position=nil)
-      @helper = RightScraper::RetrieverSpecHelper.new if @helper.nil?
-      RightScraper::Resources::Cookbook.new(@repository, position)
-    end
-
-    def parse_url(repository)
-      uri = URI.parse(repository.url)
-      userinfo, query, tag = uri.select(:userinfo, :query, :fragment)
-      unless userinfo.nil?
-        username, password = userinfo.split(":", 2).map {|str| URI.unescape str}
-        uri.user = nil
-        uri.password = nil
-      end
-      unless query.nil?
-        hash = CGI::parse(query)
-        position = hash["p"][0]
-        hash.delete("p")
-        result = hash.map do |key, values|
-          values.map {|value| CGI::escape(key) + "=" + CGI::escape(value)}.join(";")
-        end.join(';')
- 
-        if result == ""
-          uri.query = nil
-        else
-          uri.query = result
-        end
-      end
-      uri.fragment = nil unless tag.nil?
-      { :url => uri.to_s,
-        :username => username,
-        :password => password,
-        :position => position,
-        :tag => tag}
-    end
-
-    shared_examples_for 'a generic repository' do
-      it 'should have the same url' do
-        parse_url(@repository)[:url].should == @repository.url
-      end
-    end
-
-  end
-end

data/spec/cookbook_manifest_spec.rb

@@ -1,93 +0,0 @@
-#--  -*- mode: ruby; encoding: utf-8 -*-
-# Copyright: Copyright (c) 2011 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require File.expand_path(File.join(File.dirname(__FILE__), 'spec_helper'))
-require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'right_scraper'))
-
-
-module RightScraper::Scanners
-  describe CookbookManifest do
-    it 'should accumulate files' do
-      resource = flexmock(:resource)
-      resource.should_receive(:manifest=).with({"foo" => Digest::MD5.hexdigest("bar"),
-                                                 "baz" => Digest::MD5.hexdigest("quux")
-                                               }).once
-      manifest = CookbookManifest.new
-      manifest.notice("foo") { "bar" }
-      manifest.notice("baz") { "quux" }
-      manifest.end(resource)
-    end
-    it 'should keep different resources separated' do
-      resource = flexmock(:resource)
-      resource.should_receive(:manifest=).with({"foo" => Digest::MD5.hexdigest("bar"),
-                                                 "baz" => Digest::MD5.hexdigest("quux")
-                                               }).once
-      resource.should_receive(:manifest=).with({"bar" => Digest::MD5.hexdigest("fred")
-                                               }).once
-      manifest = CookbookManifest.new
-      manifest.notice("foo") { "bar" }
-      manifest.notice("baz") { "quux" }
-      manifest.end(resource)
-      manifest.notice("bar") { "fred" }
-      manifest.end(resource)
-    end
-    it 'should hash to the same value despite resource order' do
-      cookbook1 = RightScraper::Resources::Cookbook.new('<empty>', '')
-      cookbook2 = RightScraper::Resources::Cookbook.new('<empty>', '')
-
-      manifest = CookbookManifest.new
-
-      manifest.notice("foo") {"bar"}
-      manifest.notice("baz") {"quux"}
-      manifest.notice("bar/none") {"fred"}
-      manifest.notice("bar/all") {"fred"}
-      manifest.notice("a/really/deep/directory/fox") {"stray"}
-      manifest.end(cookbook1)
-
-      manifest.notice("bar/none") {"fred"}
-      manifest.notice("a/really/deep/directory/fox") {"stray"}
-      manifest.notice("foo") {"bar"}
-      manifest.notice("baz") {"quux"}
-      manifest.notice("bar/all") {"fred"}
-      manifest.end(cookbook2)
-
-      cookbook1.resource_hash.should == cookbook2.resource_hash
-    end
-    it 'should hash to the same value despite repository location' do
-      cookbook1 = RightScraper::Resources::Cookbook.new('git://github.com/somerepo', '')
-      cookbook2 = RightScraper::Resources::Cookbook.new('svn://mycompany/rss/myrepo', '')
-
-      manifest = CookbookManifest.new
-
-      manifest.notice("foo") {"bar"}
-      manifest.notice("baz") {"quux"}
-      manifest.end(cookbook1)
-
-      manifest.notice("foo") {"bar"}
-      manifest.notice("baz") {"quux"}
-      manifest.end(cookbook2)
-
-      cookbook1.resource_hash.should == cookbook2.resource_hash
-    end
-  end
-end