right_scraper 5.0.1 → 5.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ac45d2b026fadbb61775b0b93c9739b163ff8fcb
-  data.tar.gz: 14b1d686bdb7ca42b6026e4b40352e65990dac3b
+  metadata.gz: aeafef3534344761cfb4b2bc01dc8a8aec848390
+  data.tar.gz: ac21c7c9c6121c4470f8d90b6a8b16a994e963e0
 SHA512:
-  metadata.gz: 45e307c5376f9fe07522f44540a49016ff1166747327b87f6b3a807408a42135a29250b560b6add6dfbfbfa4421dce95fd2dfad3a784b9e85609377cc294e455
-  data.tar.gz: 60f022f6c81364948e14d49066b988266db2a23c97ead1fc8c3dc8a3bcd993086decfda1430c506bc0961e60661e569fff33304a139aa4b8505a064a21a73958
+  metadata.gz: 158ca23e31b79c3b9e57f09cce5f1abf95a28132412f6fd1ae5f91d03bc84874c2cc536399ceb465942180d2cc764dbebf6046362a211c9c1759c4a5cbda2752
+  data.tar.gz: 318501b488f55fe9f4ac2701f0d3bad1ee2d26b7e2ab7645818d38f8d1eadd66720436c3a80b80c204b09014d75c4ff140718e2fb813680455822c3c90e2dc21

data/README.rdoc

@@ -10,7 +10,7 @@ may specify only the functionality (and required libraries and gems) you require
 This gem depends on all available RightScraper modules, enabling full support at the
 cost of requiring some systems administration work external to Ruby.
 
-Maintained by the RightScale Teal Team
+Maintained by the RightScale Sapphire Team
 
 == USAGE
 
@@ -42,7 +42,7 @@ The build can be tested using the RSpec gem.
 
 <b>RightScraper</b>
 
-Copyright:: Copyright (c) 2010 RightScale, Inc.
+Copyright:: Copyright (c) 2010-2016 RightScale, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/bin/right_scraper-retrieve

@@ -0,0 +1,76 @@
+#!/usr/bin/env ruby
+#--
+# Copyright: Copyright (c) 2010-2016 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+def warn(*args)
+  # eliminate ruby/gem warnings from output
+end
+
+require 'json'
+require 'right_git'
+require File.expand_path('../../lib/right_scraper', __FILE__)
+
+if ::ARGV.size != 2
+  $stderr.puts "Usage: #{::File.basename(__FILE__)} <options.json> <repository.json>"
+  exit 1
+end
+
+main = nil
+begin
+  options = ::JSON.load(::File.read(::ARGV.shift))
+  repository = ::JSON.load(::File.read(::ARGV.shift))
+  main = ::RightScraper::Main.new(options)
+  if retrieved = main.retrieve(repository)
+    # remove any credentials from repository after retrieval; not needed in
+    # order to run scanners subsequently.
+    %w(first_credential second_credential).each { |k| repository.delete(k) }
+    retrieved[:repository] = repository
+    retrieved[:warnings] = main.warnings unless main.warnings.empty?
+    $stdout.puts(::JSON.generate(retrieved))
+  else
+    result = { errors: main.errors }
+    $stderr.puts(::JSON.generate(result))
+    exit 2
+  end
+  exit 0
+rescue ::RightScraper::Error, ::RightGit::Shell::ShellError => e
+  # all explicitly raised scraper errors derive from these errors.
+  result = { errors: [e.message] }
+  $stderr.puts(::JSON.generate(result))
+  exit 3
+rescue ::SystemExit => e
+  exit e.status
+rescue ::Exception => e
+  result = {
+    errors: ['internal error'],
+    unhandled_exception: {
+      class: e.class,
+      message: e.message,
+      backtrace: (e.backtrace || []).join("\n")
+    }
+  }
+  $stderr.puts(::JSON.generate(result))
+  exit 4
+ensure
+  main.cleanup rescue nil if main
+end

data/bin/right_scraper-scan

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+#--
+# Copyright: Copyright (c) 2010-2016 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+def warn(*args)
+  # eliminate ruby/gem warnings from output
+end
+
+require 'json'
+require 'right_git'
+require File.expand_path('../../lib/right_scraper', __FILE__)
+
+if ::ARGV.size != 2
+  $stderr.puts "Usage: #{::File.basename(__FILE__)} <options.json> <retrieved.json>"
+  exit 1
+end
+
+main = nil
+begin
+  options = ::JSON.load(::File.read(::ARGV.shift))
+  retrieved = ::JSON.load(::File.read(::ARGV.shift))
+  main = ::RightScraper::Main.new(options)
+
+  # cleanup any leftover errors/warnings in retrieved hash.
+  retrieved.delete('errors')
+  retrieved.delete('warnings')
+  if main.scan(retrieved)
+    retrieved[:warnings] = main.warnings unless main.warnings.empty?
+    retrieved[:resources] = main.resources.map do |r|
+      # remove repository from each returned cookbook due to redundancy.
+      h = r.to_hash
+      h.delete(:repository)
+      h
+    end
+    $stdout.puts(::JSON.generate(retrieved))
+  else
+    result = { errors: main.errors }
+    $stderr.puts(::JSON.generate(result))
+    exit 1
+  end
+  exit 0
+rescue ::RightScraper::Error, ::RightGit::Shell::ShellError => e
+  # all explicitly raised scraper errors derive from these errors.
+  result = { errors: [e.message] }
+  $stderr.puts(::JSON.generate(result))
+  exit 3
+rescue ::SystemExit => e
+  exit e.status
+rescue ::Exception => e
+  result = {
+    errors: ['internal error'],
+    unhandled_exception: {
+      class: e.class,
+      message: e.message,
+      backtrace: (e.backtrace || []).join("\n")
+    }
+  }
+  $stderr.puts(::JSON.generate(result))
+  exit 4
+ensure
+  main.cleanup rescue nil if main
+end

data/lib/right_scraper.rb

@@ -25,6 +25,9 @@ require 'right_scraper/version'
 
 # Autoload everything possible
 module RightScraper
+  # base error class
+  class Error < ::StandardError; end
+
   autoload :Builders, 'right_scraper/builders'
   autoload :Loggers, 'right_scraper/loggers'
   autoload :Main, 'right_scraper/main'

data/lib/right_scraper/main.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright: Copyright (c) 2010-2013 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2016 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -24,6 +24,7 @@
 # ancestor
 require 'right_scraper'
 
+require 'right_support'
 require 'fileutils'
 
 module RightScraper
@@ -48,32 +49,43 @@ module RightScraper
     # <tt>:max_bytes</tt>:: Maximum number of bytes to read from remote repo, unlimited if nil
     # <tt>:max_seconds</tt>:: Maximum number of seconds to spend reading from remote repo, unlimited if nil
     def initialize(options={})
-      options = {
+      options = ::RightSupport::Data::Mash.new(
         :kind        => nil,
         :basedir     => nil,
         :max_bytes   => nil,
         :max_seconds => nil,
-        :callback    => nil,
         :logger      => nil,
         :s3_key      => nil,
         :s3_secret   => nil,
         :s3_bucket   => nil,
-        :errors      => nil,
-        :warnings    => nil,
         :scanners    => nil,
         :builders    => nil,
-      }.merge(options)
+      ).merge(options)
+      @old_logger_callback = nil
       @temporary = !options.has_key?(:basedir)
       options[:basedir] ||= Dir.mktmpdir
       options[:logger] ||= ::RightScraper::Loggers::Default.new
       @logger = options[:logger]
       @resources = []
+      options[:errors] = @logger.errors
+      options[:warnings] = @logger.warnings
+
+      # load classes from scanners and builders options, if necessary.
+      [:scanners, :builders].each do |k|
+        list = options[k] || []
+        list.each_with_index do |clazz, index|
+          unless clazz.kind_of?(::Class)
+            list[index] = ::Object.const_get(clazz)
+          end
+        end
+      end
       @options = options
     end
 
-    # Scrape given repository, depositing files into the scrape
-    # directory.  Update content of unique directory incrementally
-    # when possible with further calls.
+    # Scrapes and scans a given repository.
+    #
+    # @deprecated the newer methodology will perform these operations in stages
+    # controlled externally instead of calling this all-in-one method.
     #
     # === Parameters
     # repo(Hash|RightScraper::Repositories::Base):: Repository to be scraped
@@ -98,59 +110,89 @@ module RightScraper
     # === Raise
     # 'Invalid repository type':: If repository type is not known
     def scrape(repo, incremental=true, &callback)
-      errorlen = errors.size
-      repo = RightScraper::Repositories::Base.from_hash(repo) if repo.is_a?(Hash)
+      @old_logger_callback = @logger.callback
       @logger.callback = callback
+      errorlen = errors.size
       begin
-        # 1. Retrieve the files
-        retriever = nil
-        repo_dir_changed = false
-        @logger.operation(:retrieving, "from #{repo}") do
-          # note that the retriever type may be unavailable but allow the
-          # retrieve method to raise any such error.
-          retriever = repo.retriever(@options)
-          repo_dir_changed = retriever.retrieve
-        end
-
-        # TEAL FIX: Note that retrieve will now return true iff there has been
-        # a change to the last scraped repository directory for efficiency
-        # reasons and only for retreiver types that support this behavior.
-        #
-        # Even if the retrieval is skipped due to already having the data on
-        # disk we still need to scrape its resources only because of the case
-        # of the metadata scraper daemon, which updates multiple repositories
-        # of similar criteria.
-        #
-        # The issue is that a new repo can appear later with the same criteria
-        # as an already-scraped repo and will need it's own copy of the
-        # scraped resources. The easiest (but not most efficient) way to
-        # deliver these is to rescrape the already-seen resources. This
-        # becomes more expensive as we rely on generating "metadata.json" from
-        # "metadata.rb" for cookbooks but is likely not expensive enough to
-        # need to improve this logic.
-
-
-        # 2. Now scrape if there is a scraper in the options
-        @logger.operation(:scraping, retriever.repo_dir) do
-          if @options[:kind]
-            options = @options.merge({:ignorable_paths => retriever.ignorable_paths,
-                                      :repo_dir        => retriever.repo_dir,
-                                      :repository      => retriever.repository})
-            scraper = RightScraper::Scrapers::Base.scraper(options)
-            @resources += scraper.scrape
-          end
+        if retrieved = retrieve(repo, &callback)
+          scan(retrieved, &callback)
         end
       rescue Exception
-        # logger handles communication with the end user and appending
-        # to our error list, we just need to keep going.
+        # legacy logger handles communication with the end user and appending
+        # to our error list; we just need to keep going. the new methodology
+        # has no such guaranteed communication so the caller will decide how to
+        # handle errors, etc.
       ensure
-        # ensure basedir is always removed if temporary (even with errors).
-        ::FileUtils.remove_entry_secure(@options[:basedir]) rescue nil if @temporary
+        cleanup
       end
-      @logger.callback = nil
       errors.size == errorlen
     end
 
+    # Retrieves the given repository. See #scrape for details.
+    def retrieve(repo)
+      errorlen = errors.size
+      unless repo.kind_of?(::RightScraper::Repositories::Base)
+        repo = RightScraper::Repositories::Base.from_hash(::RightSupport::Data::Mash.new(repo))
+      end
+      retriever = nil
+
+      # 1. Retrieve the files
+      @logger.operation(:retrieving, "from #{repo}") do
+        # note that the retriever type may be unavailable but allow the
+        # retrieve method to raise any such error.
+        retriever = repo.retriever(@options)
+        retriever.retrieve
+      end
+
+      if errors.size == errorlen
+        # create the freed directory with world-writable permission for
+        # subsequent scan output for less-privileged child processes.
+        freed_base_path = freed_dir(repo)
+        ::FileUtils.rm_rf(freed_base_path) if ::File.exist?(freed_base_path)
+        ::FileUtils.mkdir_p(freed_base_path)
+        ::File.chmod(0777, freed_base_path)
+
+        # the following hash is needed for running any subsequent scanners.
+        {
+          ignorable_paths: retriever.ignorable_paths,
+          repo_dir: retriever.repo_dir,
+          freed_dir: freed_base_path,
+          repository: retriever.repository
+        }
+      else
+        nil
+      end
+    end
+
+    # Scans a local directory. See #scrape for details.
+    def scan(retrieved)
+      errorlen = errors.size
+      old_callback = @logger.callback
+      options = ::RightSupport::Data::Mash.new(@options).merge(retrieved)
+      repo = options[:repository]
+      unless repo.kind_of?(::RightScraper::Repositories::Base)
+        repo = RightScraper::Repositories::Base.from_hash(::RightSupport::Data::Mash.new(repo))
+        options[:repository] = repo
+      end
+      @logger.operation(:scraping, options[:repo_dir]) do
+        scraper = ::RightScraper::Scrapers::Base.scraper(options)
+        @resources += scraper.scrape
+      end
+      errors.size == errorlen
+    end
+
+    # base directory for any file operations.
+    def base_dir
+      @options[:basedir]
+    end
+
+    # cleans up temporary files, etc.
+    def cleanup
+      @logger.callback = @old_logger_callback
+      @old_logger_callback = nil
+      ::FileUtils.remove_entry_secure(base_dir) rescue nil if @temporary
+    end
+
     # Path to directory where given repo should be or was downloaded
     #
     # === Parameters
@@ -159,7 +201,14 @@ module RightScraper
     # === Return
     # String:: Path to local directory that corresponds to given repository
     def repo_dir(repo)
-      RightScraper::Retrievers::Base.repo_dir(@options[:basedir], repo)
+      RightScraper::Retrievers::Base.repo_dir(base_dir, repo)
+    end
+
+    # Path to directory where scanned artifacts can by copied out of containment
+    # due to lack of permissions to write to other directories. the freed files
+    # can then be reused by subsequent scanners, etc.
+    def freed_dir(repo)
+      ::File.expand_path('../freed', repo_dir(repo))
     end
 
     # (Array):: Error messages in case of failure
@@ -172,7 +221,7 @@ module RightScraper
       @logger.warnings
     end
 
-   # Was scraping successful?
+    # Was scraping successful?
     # Call errors to get error messages if false
     #
     # === Return

data/lib/right_scraper/processes.rb

@@ -28,6 +28,5 @@ module RightScraper
     autoload :Shell, 'right_scraper/processes/shell'
     autoload :SSHAgent, 'right_scraper/processes/ssh_agent'
     autoload :SvnClient, 'right_scraper/processes/svn_client'
-    autoload :Warden, 'right_scraper/processes/warden'
   end
 end

data/lib/right_scraper/processes/shell.rb

@@ -37,7 +37,7 @@ module RightScraper
       include ::RightGit::Shell::Interface
 
       # exceptions.
-      class LimitError < ::RightGit::Shell::ShellError; end
+      class LimitError < ::RightScraper::Error; end
 
       class SizeLimitError < LimitError; end
       class TimeLimitError < LimitError; end
@@ -149,7 +149,7 @@ module RightScraper
         @exit_code = status.exitstatus
         if @raise_on_failure && !status.success?
           @output.buffer << "Exit code = #{@exit_code}"
-          raise ::RightGit::Shell::ShellError, "Execution failed: #{@output.display_text}"
+          raise ::RightScraper::Error, "Execution failed: #{@output.display_text}"
         end
         true
       end