right_scraper 3.2.6 → 5.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ac45d2b026fadbb61775b0b93c9739b163ff8fcb
+  data.tar.gz: 14b1d686bdb7ca42b6026e4b40352e65990dac3b
+SHA512:
+  metadata.gz: 45e307c5376f9fe07522f44540a49016ff1166747327b87f6b3a807408a42135a29250b560b6add6dfbfbfa4421dce95fd2dfad3a784b9e85609377cc294e455
+  data.tar.gz: 60f022f6c81364948e14d49066b988266db2a23c97ead1fc8c3dc8a3bcd993086decfda1430c506bc0961e60661e569fff33304a139aa4b8505a064a21a73958

data/lib/right_scraper.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2013 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -21,36 +21,18 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
-# Explicitly list required files to make IDEs happy
-require 'fileutils'
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'builders', 'base'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'builders', 'filesystem'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'builders', 'union'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'logger'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'processes', 'ssh'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'repositories', 'base'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'repositories', 'download'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'repositories', 'git'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'repositories', 'svn'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'resources', 'base'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'resources', 'cookbook'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'resources', 'workflow'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'retrievers', 'base'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'retrievers', 'checkout'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'retrievers', 'download'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'retrievers', 'git'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'retrievers', 'svn'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scanners', 'base'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scanners', 'cookbook_manifest'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scanners', 'cookbook_metadata'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scanners', 'cookbook_s3_upload'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scanners', 'union'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scanners', 'workflow_manifest'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scanners', 'workflow_metadata'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scraper'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scraper_logger'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scrapers', 'base'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scrapers', 'cookbook'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scrapers', 'workflow'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'svn_client'))
-require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'version'))
+require 'right_scraper/version'
+
+# Autoload everything possible
+module RightScraper
+  autoload :Builders, 'right_scraper/builders'
+  autoload :Loggers, 'right_scraper/loggers'
+  autoload :Main, 'right_scraper/main'
+  autoload :Processes, 'right_scraper/processes'
+  autoload :RegisteredBase, 'right_scraper/registered_base'
+  autoload :Repositories, 'right_scraper/repositories'
+  autoload :Resources, 'right_scraper/resources'
+  autoload :Retrievers, 'right_scraper/retrievers'
+  autoload :Scanners, 'right_scraper/scanners'
+  autoload :Scrapers, 'right_scraper/scrapers'
+end

data/lib/right_scraper/builders.rb

@@ -0,0 +1,32 @@
+#
+# Copyright (c) 2013 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+# ancestor
+require 'right_scraper'
+
+module RightScraper
+  module Builders
+    autoload :Base, 'right_scraper/builders/base'
+    autoload :Filesystem, 'right_scraper/builders/filesystem'
+    autoload :Union, 'right_scraper/builders/union'
+  end
+end

data/lib/right_scraper/builders/base.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2013 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -21,43 +21,42 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
-require File.expand_path(File.join(File.dirname(__FILE__), '..', 'logger'))
+# ancestor
+require 'right_scraper/builders'
 
 module RightScraper
   module Builders
+
     # Base class for building additional metadata from filesystem
-    # based checkouts.  Subclasses should override #go, and possibly
+    # based checkouts. Subclasses should override #go, and possibly
     # #new if they require additional arguments.
     #
     # The lifecycle for a builder is as follows:
     # - builder = Builder.new (once)
     # - builder.go(dir, resource) (many times)
     # - builder.finish (once)
-    class Builder
-      # Create a new Builder.  Recognizes options as given.  Some
-      # options may be required, others optional.  This class recognizes
-      # only :logger.
-      #
-      # === Options
-      # <tt>:logger</tt>:: Optional.  Logger currently being used
-      #
-      # === Parameters
-      # options(Hash):: builder options
-      def initialize(options={})
-        @logger = options.fetch(:logger, Logger.new)
+    class Base
+      # @param [Hash] options for builder
+      # @option options [Logger] :logger for builder
+      def initialize(options = {})
+        @logger = options[:logger] || ::RightScraper::Loggers::Default.new
       end
 
       # Run builder for this resource.
       #
-      # === Parameters
-      # dir(String):: directory resource exists at
-      # resource(Object):: resource instance being built
+      # @param [String] dir for resource
+      # @param [Object] resource to build
+      #
+      # @return [TrueClass] always true
       def go(dir, resource)
+        raise NotImplementedError
       end
 
-      # Notification that all scans for this repository have
-      # completed.
+      # Notification that all scans for this repository have completed.
+      #
+      # @return [TrueClass] always true
       def finish
+        true
       end
     end
   end

data/lib/right_scraper/builders/filesystem.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2013 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -21,12 +21,13 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
-require File.expand_path(File.join(File.dirname(__FILE__), 'base'))
+# ancestor
+require 'right_scraper/builders'
 
 module RightScraper
   module Builders
     # Build metadata by scanning the filesystem.
-    class Filesystem < Builder
+    class Filesystem < ::RightScraper::Builders::Base
 
       # Create a new filesystem scanner.  In addition to the options
       # recognized by Builder, this class recognizes <tt>:retriever</tt> and
@@ -85,9 +86,10 @@ module RightScraper
           if File.directory?(fullpath)
             maybe_scan(Dir.new(fullpath), relative_position)
           else
-            @scanner.notice(relative_position) do
-              open(fullpath) {|f| f.read}
-            end
+            # bind the temporary parameters to the callback in case it is not
+            # invoked immediately.
+            bind_now = lambda{|file| lambda{ open(file) {|f| f.read} } }.call(fullpath)
+            @scanner.notice(relative_position, &bind_now)
           end
         end
       end

data/lib/right_scraper/builders/union.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2013 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -21,6 +21,9 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
+# ancestor
+require 'right_scraper/builders'
+
 module RightScraper
   module Builders
     # Union builder, to permit running multiple builders in sequence

data/lib/right_scraper/loggers.rb

@@ -0,0 +1,31 @@
+#
+# Copyright (c) 2013 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+# ancestor
+require 'right_scraper'
+
+module RightScraper
+  module Loggers
+    autoload :Base, 'right_scraper/loggers/base'
+    autoload :Default, 'right_scraper/loggers/default'
+  end
+end

data/lib/right_scraper/loggers/base.rb

@@ -0,0 +1,113 @@
+#--
+# Copyright: Copyright (c) 2013 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+# ancestor
+require 'right_scraper/loggers'
+
+require 'logger'
+
+module RightScraper
+  module Loggers
+    class Base < ::Logger
+
+      attr_accessor :callback, :errors, :warnings
+
+      def initialize(*args)
+        super(*args)
+        @abort_seen = false
+        @callback = nil
+        @errors = []
+        @warnings = []
+      end
+
+      # Encapsulates an operation that merits logging.
+      #
+      # @param [Symbol] type of operation
+      # @param [String] explanation of operation or empty
+      #
+      # @yield [] operational callback (required)
+      #
+      # @return [Object] result of callback
+      def operation(type, explanation = '')
+        begin
+          note_phase(:begin, type, explanation)
+          result = yield
+          note_phase(:commit, type, explanation)
+          result
+        rescue Exception => e
+          note_phase(:abort, type, explanation, e)
+          raise
+        end
+      end
+
+      # Note a phase change within an operation.
+      #
+      # @param [Symbol] phase of operation; one of :begin, :commit, :abort
+      # @param [Symbol] type of operation
+      # @param [String] explanation of operation or nil
+      # @param [Exception] exception or nil
+      #
+      # @return [TrueClass] always true
+      def note_phase(phase, type, explanation, exception = nil)
+        @callback.call(phase, type, explanation, exception) if @callback
+        case phase
+        when :begin
+          @abort_seen = false
+        when :commit
+          # do nothing
+        when :abort
+          unless @abort_seen
+            note_error(exception, type, explanation)
+            @abort_seen = true
+          end
+        else
+          fail 'Unknown phase'
+        end
+      end
+
+      # Note an error, with the given exception and explanation of what
+      # was going on.
+      #
+      # @param [Exception] exception or nil
+      # @param [Symbol] type of operation
+      # @param [String] explanation of operation or nil
+      #
+      # @return [TrueClass] always true
+      def note_error(exception, type, explanation = nil)
+        raise NotImplementedError
+      end
+
+      # Note a warning for current operation.
+      #
+      # @param [Exception] exception or nil
+      # @param [Symbol] type of operation
+      # @param [String] explanation of operation or nil
+      #
+      # @return [TrueClass] always true
+      def note_warning(message)
+        raise NotImplementedError
+      end
+
+    end
+  end
+end

data/lib/right_scraper/loggers/default.rb

@@ -0,0 +1,98 @@
+#--
+# Copyright: Copyright (c) 2010-2013 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+# ancestor
+require 'right_scraper/loggers'
+
+module RightScraper::Loggers
+
+  # provides a default scraper logger implementation that accumulates errors and
+  # warnings but otherwise is a null logger.
+  class Default < ::RightScraper::Loggers::Base
+
+    def initialize(*args)
+      if args.empty?
+        is_windows = !!(RUBY_PLATFORM =~ /mswin|win32|dos|mingw|cygwin/)
+        super(is_windows ? 'nul' : '/dev/null')
+      else
+        super(*args)
+      end
+      self.level = ::Logger::ERROR
+      @recording_messages = true
+    end
+
+    # implements Interface#note_error
+    def note_error(exception, type, explanation = '')
+      without_recording_messages do
+        explanation = explanation.to_s.strip
+        message = "Saw #{exception ? exception.message : 'error'} during #{type}"
+        message += ": #{explanation}" unless explanation.empty?
+        error(message)
+      end
+      @errors << [exception, type, explanation]
+    end
+
+    # implements Interface#note_warning
+    def note_warning(message)
+      without_recording_messages { warn(message) }
+      @warnings << message
+    end
+
+    # overrides ::Logger#add in order to record errors and warnings logged via
+    # the normal logger interface.
+    def add(severity, message = nil, progname = nil)
+      if severity >= self.level
+        # super logger.
+        if message.nil?
+          if block_given?
+            message = yield
+          else
+            message = progname
+            progname = self.progname
+          end
+        end
+        super(severity, message, progname)
+
+        # record errors (with detail) and warnings.
+        if @recording_messages
+          if severity >= Logger::ERROR
+            @errors << [nil, :log, message]
+          elsif severity == ::Logger::WARN
+            @warnings << message
+          end
+        end
+      end
+      true
+    end
+
+    protected
+
+    def without_recording_messages
+      old_recording_messages = @recording_messages
+      @recording_messages = false
+      yield
+    ensure
+      @recording_messages = old_recording_messages
+    end
+  end
+end