right_scraper 3.2.6 → 5.0.1

data/Gemfile

@@ -1,15 +0,0 @@
-source 'https://rubygems.org'
-
-gemspec
-
-gem 'right_popen', :git => 'git@github.com:rightscale/right_popen.git',
-                   :branch => "master"
-
-gem 'right_support', '2.7' 
-
-group :development do
-  gem "rake", "0.8.7"
-  gem "ruby-debug", :platform=>"ruby_18"
-  gem "ruby-debug19", :platform=>"ruby_19"
-  gem "rdoc", "~> 2.4"
-end

data/Rakefile

@@ -1,89 +0,0 @@
-#-- -*-ruby-*-
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require 'rubygems'
-require 'rubygems/package_task'
-require 'bundler/setup'
-
-require 'fileutils'
-require 'rake'
-require 'rspec/core/rake_task'
-require 'rdoc/task'
-require 'rake/clean'
-
-task :default => 'spec'
-
-# == Gem packaging == #
-
-desc "Build right_scraper gem"
-Gem::PackageTask.new(Gem::Specification.load("right_scraper.gemspec")) do |package|
-  package.need_zip = true
-  package.need_tar = true
-end
-
-CLEAN.include('pkg')
-
-# == Unit Tests == #
-
-task :specs => :spec
-
-# == Unit Tests == #
-
-desc 'Run unit tests'
-RSpec::Core::RakeTask.new do |t|
-  t.pattern = 'spec/**/*_spec.rb'
-    t.rspec_opts = ["--color", "--format", "nested"]
-end
-
-namespace :spec do
-  desc "Run unit tests with RCov"
-  RSpec::Core::RakeTask.new(:rcov) do |t|
-    t.pattern = '*/spec/**/*_spec.rb'
-    t.rcov = true
-    t.rcov_opts = %q[--exclude "spec"]
-  end
-
-  desc "Print Specdoc for unit tests"
-  RSpec::Core::RakeTask.new(:doc) do |t|
-    t.pattern = '*/spec/**/*_spec.rb'
-    t.rspec_opts = ["--format", "documentation"]
-  end
-end
-
-# == Documentation == #
-
-desc "Generate API documentation to doc/rdocs/index.html"
-RDoc::Task.new do |rd|
-  rd.rdoc_dir = 'doc/rdocs'
-  rd.main = 'README.rdoc'
-  rd.rdoc_files.include 'README.rdoc', 'lib/**/*.rb'
-
-  rd.options << '--all'
-  rd.options << '--diagram'
-end
-
-# == Emacs integration == #
-desc "Rebuild TAGS file"
-task :tags do
-  sh "rtags -R */{lib,spec}"
-end

data/lib/right_scraper/logger.rb

@@ -1,107 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require 'logger'
-
-module RightScraper
-  # Very simplistic logger for scraper operations.
-  class Logger < ::Logger
-    # If no arguments, create a Logger that logs to nothing.
-    # Otherwise pass the arguments on to ::Logger.
-    def initialize(*args)
-      if args.empty?
-        is_windows = !!(RUBY_PLATFORM =~ /mswin|win32|dos|mingw|cygwin/)
-        super((is_windows ? 'nul' : '/dev/null'))
-        self.level = ::Logger::ERROR
-      else
-        super
-      end
-      @exceptional = false
-    end
-
-    # (RightScraper::Repositories::Base) Repository currently being examined.
-    attr_writer :repository
-
-    # Begin an operation that merits logging.  Will call #note_error
-    # if an exception is raised during the operation.
-    #
-    # === Parameters
-    # type(Symbol):: operation type identifier
-    # explanation(String):: optional explanation
-    def operation(type, explanation="")
-      begin
-        note_phase(:begin, type, explanation)
-        result = yield
-        note_phase(:commit, type, explanation)
-        result
-      rescue Exception => e
-        note_phase(:abort, type, explanation, e)
-        raise
-      end
-    end
-
-    # Note an event to the log.  In this base class this calls
-    # note_error when an error occurs, but subclasses will presumably
-    # want to override it.
-    #
-    # === Parameters
-    # phase(Symbol):: phase of operation; one of :begin, :commit, :abort
-    # type(Symbol):: operation type identifier
-    # explanation(String):: explanation of operation
-    # exception(Exception):: optional exception (only if +phase+ is :abort)
-    def note_phase(phase, type, explanation, exception=nil)
-      case phase
-      when :begin then @exceptional = false
-      when :abort then
-        unless @exceptional
-          note_error(exception, type, explanation)
-          @exceptional = true
-        end
-      end
-    end
-
-    # Log an error, with the given exception and explanation of what
-    # was going on.
-    #
-    # === Parameters
-    # exception(Exception):: exception raised
-    # type(Symbol):: operation type identifier
-    # explanation(String):: optional explanation
-    def note_error(exception, type, explanation="")
-      error("Saw #{exception} during #{type}#{maybe_explain(explanation)}")
-    end
-
-    def note_warning(message)
-      warn(message)
-    end
-
-    protected
-    def maybe_explain(explanation)
-      if explanation.nil? || explanation.empty?
-        ""
-      else
-        ": #{explanation}"
-      end
-    end
-  end
-end

data/lib/right_scraper/loggers/noisy.rb

@@ -1,85 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-require File.expand_path(File.join(File.dirname(__FILE__), '..', 'logger'))
-
-module RightScraper
-  module Loggers
-    # A very noisy logger, useful for debugging.
-    class NoisyLogger < Logger
-      # Initialize the logger, setting the current operation depth to 1.
-      def initialize(*args)
-        super
-        @pending = []
-      end
-
-      # Note an event to the log, including a visual indicator of how
-      # many nested operations are currently pending.
-      #
-      # === Parameters
-      # phase(Symbol):: phase of operation; one of :begin, :commit, :abort
-      # type(Symbol):: operation type identifier
-      # explanation(String):: explanation of operation
-      # exception(Exception):: optional exception (only if +phase+ is :abort)
-      def note_phase(phase, type, explanation, exception=nil)
-        if phase == :begin
-          @pending.push [type, explanation]
-        end
-        super
-        debug("#{depth_str} #{phase} #{immediate_context}")
-        unless phase == :begin
-          @pending.pop
-        end
-      end
-
-      def note_error(exception, type, explanation="")
-        recordedtype, recordedexplanation = @pending[-1]
-        if recordedtype != type || recordedexplanation != explanation
-          @pending.push [type, explanation]
-        end
-        error("Saw #{exception} during #{context}")
-        if recordedtype != type || recordedexplanation != explanation
-          @pending.pop
-        end
-      end
-
-      private
-      def depth_str
-        '>' * @pending.size
-      end
-
-      def context
-        @pending.reverse.map {|pair| contextify(pair)}.join(" in ")
-      end
-
-      def immediate_context
-        contextify(@pending[-1])
-      end
-
-      def contextify(pair)
-        type, explanation = pair
-        "#{type}#{maybe_explain(explanation)}"
-      end
-    end
-  end
-end

data/lib/right_scraper/repositories/mock.rb

@@ -1,70 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-require File.expand_path(File.join(File.dirname(__FILE__), 'base'))
-
-module RightScraper
-  module Repositories
-    # A "repository" that is just there for testing.  This class is not
-    # loaded by default.
-    class Mock < Base
-      # Create a new mock repository.
-      def initialize
-        @repo_type = :mock
-      end
-      # (String) Type of the repository, here 'download'.
-      attr_accessor :repo_type
-
-      # (String) Optional, tag or branch of repository that should be downloaded
-      attr_accessor :tag
-
-      # (String) Optional, username
-      attr_accessor :first_credential
-
-      # (String) Optional, password
-      attr_accessor :second_credential
-
-      # Unique representation for this repo, should resolve to the same string
-      # for repos that should be cloned in same directory
-      #
-      # === Returns
-      # res(String):: Unique representation for this repo
-      def to_s
-        res = "mock #{url}:#{tag}"
-      end
-
-      # (Base class) Appropriate class for scraping this sort of
-      # repository.
-      def scraper
-        @@scraper || raise("Scraper for mocks isn't defined yet")
-      end
-
-      # Set the correct sort of scraper to use for mock repositories.
-      def self.scraper=(scraper)
-        @@scraper = scraper
-      end
-
-      # Add this repository to the list of available types.
-      @@types[:mock] = RightScraper::Repositories::Mock
-    end
-  end
-end

data/lib/right_scraper/retrievers/checkout.rb

@@ -1,79 +0,0 @@
-#--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# 'Software'), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#++
-
-module RightScraper
-  module Retrievers
-
-    # Base class for retrievers that want to do version control operations
-    # (CVS, SVN, etc.). Subclasses can get away with implementing only
-    # Retrievers::Base#available? and #do_checkout but to support incremental
-    # operation need to implement #exists? and #do_update, in addition to
-    # Retrievers::Base#ignorable_paths.
-    class CheckoutBasedRetriever < Base
-
-      # Check out repository into the directory.  Occurs between
-      # variable initialization and beginning scraping.
-      def retrieve
-        raise RetrieverError.new("retriever is unavailable") unless available?
-        if exists?
-          begin
-            @logger.operation(:updating) do
-              do_update
-            end
-          rescue Exception => e
-            FileUtils.remove_entry_secure @basedir
-            @logger.operation(:checkout, "switching to using checkout") do
-              do_checkout
-            end
-          end
-        else
-          @logger.operation(:checkout) do
-            do_checkout
-          end
-        end
-      end
-
-      # Return true if a checkout exists.
-      #
-      # === Returns
-      # Boolean:: true if the checkout already exists (and thus
-      #           incremental updating can occur).
-      def exists?
-        false
-      end
-
-      # Perform an incremental update of the checkout.  Subclasses that
-      # want to handle incremental updating need to override this.
-      def do_update
-        do_checkout
-      end
-
-      # Perform a de novo full checkout of the repository.  Subclasses
-      # must override this to do anything useful.
-      def do_checkout
-        FileUtils.mkdir_p(@repo_dir)
-      end
-
-    end
-  end
-end