right_scraper 3.2.6 → 5.0.1

data/lib/right_scraper/retrievers/svn.rb

@@ -20,81 +20,121 @@
 # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
-require File.join(File.dirname(__FILE__), '..', 'svn_client')
-
-module RightScraper
-  module Retrievers
-    # Retriever for svn repositories
-    class Svn < CheckoutBasedRetriever
-
-      include RightScraper::SvnClient
-
-      @@available = false
-
-      # Determines if svn is available.
-      def available?
-        unless @@available
-          begin
-            calculate_version
-            @@available = true
-          rescue SvnClientError => e
-            @logger.note_error(e, :available, "svn retriever is unavailable")
-          end
+
+# ancestor
+require 'right_scraper/retrievers'
+
+module RightScraper::Retrievers
+
+  # Retriever for svn repositories
+  class Svn < ::RightScraper::Retrievers::CheckoutBase
+
+    SVN_CLIENT = ::RightScraper::Processes::SvnClient
+
+    @@available = false
+
+    # Determines if svn is available.
+    def available?
+      unless @@available
+        begin
+          SVN_CLIENT.calculate_version
+          @@available = true
+        rescue SVN_CLIENT::SvnClientError => e
+          @logger.note_error(e, :available, 'svn retriever is unavailable')
         end
-        @@available
       end
+      @@available
+    end
 
-      # Return true if a checkout exists.  Currently tests for .svn in
-      # the checkout.
-      #
-      # === Returns
-      # Boolean:: true if the checkout already exists (and thus
-      #           incremental updating can occur).
-      def exists?
-        File.exists?(File.join(@repo_dir, '.svn'))
-      end
+    # Return true if a checkout exists.  Currently tests for .svn in
+    # the checkout.
+    #
+    # === Returns
+    # Boolean:: true if the checkout already exists (and thus
+    #           incremental updating can occur).
+    def exists?
+      ::File.exists?(::File.join(@repo_dir, '.svn'))
+    end
 
-      # Incrementally update the checkout.  The operations are as follows:
-      # * update to #tag
-      # In theory if #tag is a revision number that already exists no
-      # update is necessary.  It's not clear if the SVN client libraries
-      # are bright enough to notice this.
-      def do_update
-        @logger.operation(:update) do
-          run_svn("update", get_tag_argument)
-        end
+    # Ignore .svn directories.
+    def ignorable_paths
+      ['.svn']
+    end
+
+    # Implements CheckoutBase#do_checkout
+    def do_checkout
+      @logger.operation(:checkout_revision) do
+        revision = resolve_revision
+        svn_args = ['checkout', @repository.url, @repo_dir]
+        svn_args += ['--revision', revision] if revision
+        svn_args << '--force'
+        svn_client.execute(svn_args)
         do_update_tag
       end
+    end
 
-      # Update our idea of what the head of the repository is.  We
-      # would like to use svn info, but that doesn't do the right
-      # thing all the time; the right thing to do is to run log and
-      # pick out the first tag.
-      def do_update_tag
-        @repository = @repository.clone
-        lines = run_svn_with_buffered_output("log", "-r", 'HEAD')
-        lines.each do |line|
-          if line =~ /^r(\d+)/
-            @repository.tag = $1
-            break
-          end
-        end
+    # Implements CheckoutBase#do_update
+    def do_update
+      @logger.operation(:update) do
+        revision = resolve_revision
+        svn_client.execute('update', '--revision', revision, '--force')
+        do_update_tag
       end
+    end
 
-      # Check out the remote repository.  The operations are as follows:
-      # * checkout repository at #tag to @repo_dir
-      def do_checkout
-        super
-        @logger.operation(:checkout_revision) do
-          run_svn_no_chdir("checkout", @repository.url, @repo_dir, get_tag_argument)
+    # Implements CheckoutBase#do_update_tag
+    def do_update_tag
+      # query latest count=1 log entry for latest revision; don't attempt to
+      # specify revision on the assumption that the requested revision is
+      # already checked out. the --revision argument appears to expect a
+      # revision from-to range or else a start date or date range or else a
+      # specific revision number. it prints nothing when HEAD is specified by
+      # itself.
+      @repository = @repository.clone
+      svn_args = ['log', '--limit', '1']
+      svn_client.output_for(svn_args).lines.each do |line|
+        if matched = SVN_LOG_REGEX.match(line)
+          @repository.tag = matched[1]
+          break
         end
-        do_update_tag
       end
+    end
 
-      # Ignore .svn directories.
-      def ignorable_paths
-        ['.svn']
+    private
+
+    # http://svnbook.red-bean.com/en/1.7/svn.tour.revs.specifiers.html#svn.tour.revs.keywords
+    #
+    # Example: HEAD | <revision number> | {<datetime>}
+    #
+    # {2010-12-06T19:11:25} === {2010-12-06 19:11:25 +0000}
+    SVN_REVISION_REGEX = /^(HEAD|\d+|\{[0-9: T+\-]+\})$/
+
+    # Example:
+    # r12 | ira | 2006-11-27 12:31:51 -0600 (Mon, 27 Nov 2006) | 6 lines
+    SVN_LOG_REGEX = /^r(\d+)/  # ignoring additional info after revision
+
+    def resolve_revision
+      revision = @repository.tag.to_s.strip
+      if revision.empty?
+        revision = nil
+      elsif (revision =~ SVN_REVISION_REGEX).nil?
+        raise RetrieverError, "Revision reference contained illegal characters: #{revision.inspect}"
       end
+      # timestamps can contain spaces; surround them with double quotes.
+      revision = revision.inspect if revision.index(' ')
+      revision
     end
+
+    def svn_client
+      @svn_client ||= SVN_CLIENT.new(
+        @repository,
+        @logger,
+        ::RightScraper::Processes::Shell.new(
+          :initial_directory => self.repo_dir,
+          :max_bytes         => self.max_bytes,
+          :max_seconds       => self.max_seconds,
+          :watch_directory   => self.repo_dir))
+    end
+
   end
 end

data/lib/right_scraper/scanners.rb

@@ -0,0 +1,37 @@
+#
+# Copyright (c) 2013 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+# ancestor
+require 'right_scraper'
+
+module RightScraper
+  module Scanners
+    autoload :Base, 'right_scraper/scanners/base'
+    autoload :CookbookManifest, 'right_scraper/scanners/cookbook_manifest'
+    autoload :CookbookMetadata, 'right_scraper/scanners/cookbook_metadata'
+    autoload :CookbookS3Upload, 'right_scraper/scanners/cookbook_s3_upload'
+    autoload :Union, 'right_scraper/scanners/union'
+    autoload :WorkflowManifest, 'right_scraper/scanners/workflow_manifest'
+    autoload :WorkflowMetadata, 'right_scraper/scanners/workflow_metadata'
+    autoload :WorkflowS3Upload, 'right_scraper/scanners/workflow_s3_upload'
+  end
+end

data/lib/right_scraper/scanners/base.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2013 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -21,91 +21,88 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
-module RightScraper
-  module Scanners
-    # Base class for scanning filesystems.  Subclasses should override
-    # #notice and may override #new, #begin, #end and
-    # #notice_dir.
-    #
-    # Overriding #new is useful for getting
-    # additional arguments.  Overriding #begin allows you to do
-    # processing before the scan of a given resource begins;
-    # overriding #end allows you to do processing after it completes.
-    #
-    # Most processing will occur in #notice, which notifies you that a
-    # file has been detected, and in #notice_dir.  In #notice you are
-    # handed the relative position of the file from the start of the
-    # resource; so if you were scanning <tt>/a/resource</tt> and
-    # noticed a file <tt>b/c</tt>, #notice would be called with
-    # <tt>"b/c"</tt>, even though the full pathname is
-    # <tt>/a/resource/b/c</tt>.  If you decide you need the actual
-    # data, #notice takes a block which will return that data to you
-    # if you +yield+.
-    #
-    # In #notice_dir you are handed the relative position of a
-    # directory.  The return value determines whether you find the
-    # directory worth recursing into, or not--as an example, when
-    # looking for the <tt>metadata.json</tt> file it is never
-    # necessary to descend past the topmost directory of the resource,
-    # but the same is not true when building a manifest.
-    class Base
-      # Create a new Scanner.  Recognizes options as given.  Some
-      # options may be required, others optional.  This class recognizes
-      # only _:logger_.
-      #
-      # === Options ===
-      # _:logger_:: Optional.  Logger currently being used
-      #
-      # === Parameters ===
-      # options(Hash):: scanner options
-      def initialize(options={})
-        @logger = options.fetch(:logger, RightScraper::Logger.new)
-      end
+# ancestor
+require 'right_scraper/scanners'
 
-      # Notification that all scans for this repository have
-      # completed.
-      def finish
-      end
+module RightScraper::Scanners
 
-      # Begin a scan for the given resource.
-      #
-      # === Parameters ===
-      # resource(RightScraper::Resource::Base):: resource to scan
-      def begin(resource)
-      end
+  # Base class for scanning filesystems.  Subclasses should override
+  # #notice and may override #new, #begin, #end and
+  # #notice_dir.
+  #
+  # Overriding #new is useful for getting
+  # additional arguments.  Overriding #begin allows you to do
+  # processing before the scan of a given resource begins;
+  # overriding #end allows you to do processing after it completes.
+  #
+  # Most processing will occur in #notice, which notifies you that a
+  # file has been detected, and in #notice_dir.  In #notice you are
+  # handed the relative position of the file from the start of the
+  # resource; so if you were scanning <tt>/a/resource</tt> and
+  # noticed a file <tt>b/c</tt>, #notice would be called with
+  # <tt>"b/c"</tt>, even though the full pathname is
+  # <tt>/a/resource/b/c</tt>.  If you decide you need the actual
+  # data, #notice takes a block which will return that data to you
+  # if you +yield+.
+  #
+  # In #notice_dir you are handed the relative position of a
+  # directory.  The return value determines whether you find the
+  # directory worth recursing into, or not--as an example, when
+  # looking for the <tt>metadata.json</tt> file it is never
+  # necessary to descend past the topmost directory of the resource,
+  # but the same is not true when building a manifest.
+  class Base
 
-      # Finish a scan for the given resource.
-      #
-      # === Parameters ===
-      # resource(RightScraper::Resource::Base):: resource that just finished
-      #                                  scanning
-      def end(resource)
+    # @param [Hash] options for scanner
+    def initialize(options={})
+      unless @logger = options[:logger]
+        raise ::ArgumentError, ':logger is required'
       end
+    end
 
-      # Notice a file during scanning.
-      #
-      # === Block ===
-      # Return the data for this file.  We use a block because it may
-      # not always be necessary to read the data.
-      #
-      # === Parameters ===
-      # relative_position(String):: relative pathname for _pathname_
-      #                             from root of resource
-      def notice(relative_position)
-      end
+    # Notification that all scans for this repository have
+    # completed.
+    def finish
+    end
 
-      # Notice a directory during scanning.  Returns true if the scanner
-      # should recurse into the directory (the default behavior)
-      #
-      # === Parameters ===
-      # relative_position(String):: relative pathname for the directory
-      #                             from root of resource
-      #
-      # === Returns ===
-      # Boolean:: should the scanning recurse into the directory
-      def notice_dir(relative_position)
-        true
-      end
+    # Begin a scan for the given resource.
+    #
+    # === Parameters ===
+    # resource(RightScraper::Resource::Base):: resource to scan
+    def begin(resource)
+    end
+
+    # Finish a scan for the given resource.
+    #
+    # === Parameters ===
+    # resource(RightScraper::Resource::Base):: resource that just finished
+    #                                  scanning
+    def end(resource)
+    end
+
+    # Notice a file during scanning.
+    #
+    # === Block ===
+    # Return the data for this file.  We use a block because it may
+    # not always be necessary to read the data.
+    #
+    # === Parameters ===
+    # relative_position(String):: relative pathname for _pathname_
+    #                             from root of resource
+    def notice(relative_position)
+    end
+
+    # Notice a directory during scanning.  Returns true if the scanner
+    # should recurse into the directory (the default behavior)
+    #
+    # === Parameters ===
+    # relative_position(String):: relative pathname for the directory
+    #                             from root of resource
+    #
+    # === Returns ===
+    # Boolean:: should the scanning recurse into the directory
+    def notice_dir(relative_position)
+      true
     end
   end
 end

data/lib/right_scraper/scanners/cookbook_manifest.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2013 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -21,39 +21,40 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
-require File.expand_path(File.join(File.dirname(__FILE__), 'base'))
+# ancestor
+require 'right_scraper/scanners'
+
 require 'digest/md5'
 
-module RightScraper
-  module Scanners
-    # Build manifests from a filesystem.
-    class CookbookManifest < Base
-      # Create a new manifest scanner.  Does not accept any new arguments.
-      def initialize(*args)
-        super
-        @manifest = {}
-      end
+module RightScraper::Scanners
+
+  # Build manifests from a filesystem.
+  class CookbookManifest < ::RightScraper::Scanners::Base
+    # Create a new manifest scanner.  Does not accept any new arguments.
+    def initialize(*args)
+      super
+      @manifest = {}
+    end
 
-      # Complete a scan for the given resource.
-      #
-      # === Parameters ===
-      # resource(RightScraper::Resources::Base):: resource to scan
-      def end(resource)
-        resource.manifest = @manifest
-        @manifest = {}
-      end
+    # Complete a scan for the given resource.
+    #
+    # === Parameters ===
+    # resource(RightScraper::Resources::Base):: resource to scan
+    def end(resource)
+      resource.manifest = @manifest
+      @manifest = {}
+    end
 
-      # Notice a file during scanning.
-      #
-      # === Block ===
-      # Return the data for this file.  We use a block because it may
-      # not always be necessary to read the data.
-      #
-      # === Parameters ===
-      # relative_position(String):: relative pathname for file from root of resource
-      def notice(relative_position)
-        @manifest[relative_position] = Digest::MD5.hexdigest(yield)
-      end
+    # Notice a file during scanning.
+    #
+    # === Block ===
+    # Return the data for this file.  We use a block because it may
+    # not always be necessary to read the data.
+    #
+    # === Parameters ===
+    # relative_position(String):: relative pathname for file from root of resource
+    def notice(relative_position)
+      @manifest[relative_position] = Digest::MD5.hexdigest(yield)
     end
   end
 end