right_scraper 3.2.6 → 5.0.1

data/lib/right_scraper/repositories/download.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2013 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -21,12 +21,16 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
+# ancestor
+require 'right_scraper/repositories'
+
 module RightScraper
   module Repositories
+
     # A repository that is just an archive file hanging off a
     # web server somewhere.  This version uses a command line curl to
     # download the archive, and command line tar to extract it.
-    class Download < Base
+    class Download < ::RightScraper::Repositories::Base
       # (String) Type of the repository, here 'download'.
       def repo_type
         :download
@@ -81,10 +85,8 @@ module RightScraper
         RightScraper::Retrievers::Download.new(self, options)
       end
 
-
-      # Add this repository to the list of available types.
-      @@types[:download] = RightScraper::Repositories::Download
-
+      # self-register
+      register_self
     end
   end
 end

data/lib/right_scraper/repositories/git.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2013 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -21,10 +21,13 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
+# ancestor
+require 'right_scraper/repositories'
+
 module RightScraper
   module Repositories
     # A Git repository.
-    class Git < Base
+    class Git < ::RightScraper::Repositories::Base
 
       # (String) Optional, tag or branch of repository that should be downloaded
       attr_accessor :tag
@@ -80,13 +83,9 @@ module RightScraper
         RightScraper::Retrievers::Git.new(self, options)
       end
 
-      # Add this repository to the list of available types.
-      @@types[:git] = RightScraper::Repositories::Git
-
-      # Add git URL schemas to the list of okay schemas.
-      @@okay_schemes << "git"
-      @@okay_schemes << "git+ssh"
-      @@okay_schemes << "ssh"
+      # self-register
+      register_self
+      register_url_schemas('git', 'git+ssh', 'ssh')
     end
   end
 end

data/lib/right_scraper/repositories/svn.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2013 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -21,10 +21,13 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
+# ancestor
+require 'right_scraper/repositories'
+
 module RightScraper
   module Repositories
     # A repository that is stored in a Subversion server.
-    class Svn < Base
+    class Svn < ::RightScraper::Repositories::Base
 
       # (String) Optional, tag or branch of repository that should be downloaded
       attr_accessor :tag
@@ -85,12 +88,9 @@ module RightScraper
         RightScraper::Retrievers::Svn.new(self, options)
       end
 
-      # Add this repository to the list of available types.
-      @@types[:svn] = RightScraper::Repositories::Svn
-
-      # Add git URL schemas to the list of okay schemas.
-      @@okay_schemes << "svn"
-      @@okay_schemes << "svn+ssh"
+      # self-register
+      register_self
+      register_url_schemas('svn', 'svn+ssh')
     end
   end
 end

data/lib/right_scraper/resources.rb

@@ -0,0 +1,32 @@
+#
+# Copyright (c) 2013 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+# ancestor
+require 'right_scraper'
+
+module RightScraper
+  module Resources
+    autoload :Base, 'right_scraper/resources/base'
+    autoload :Cookbook, 'right_scraper/resources/cookbook'
+    autoload :Workflow, 'right_scraper/resources/workflow'
+  end
+end

data/lib/right_scraper/resources/base.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2013 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -20,6 +20,10 @@
 # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
+
+# ancestor
+require 'right_scraper/resources'
+
 require 'digest/sha1'
 require 'uri'
 

data/lib/right_scraper/resources/cookbook.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright: Copyright (c) 2010-2012 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2013 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -21,45 +21,52 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
+# ancestor
+require 'right_scraper/resources'
+
 require 'digest/md5'
 require 'json'
 require 'right_support'
 
-module RightScraper
+module RightScraper::Resources
+  class Cookbook < ::RightScraper::Resources::Base
 
-  module Resources
+    EMPTY_MANIFEST_JSON = ::JSON.dump(:manifest => {}).freeze
 
-    class Cookbook < Base
+    # @return [String] repo_dir as local repo root dir (sans relative cookbook pos path)
+    attr_reader :repo_dir
 
-      EMPTY_MANIFEST_JSON = ::JSON.dump(:manifest => {}).freeze
+    def initialize(repo, pos, repo_dir)
+      super(repo, pos)
+      @repo_dir = repo_dir
+    end
 
-      def manifest=(value)
-        @manifest_json = nil
-        @resource_hash = nil
-        @manifest      = value
-      end
+    def manifest=(value)
+      @manifest_json = nil
+      @resource_hash = nil
+      @manifest      = value
+    end
 
-      def manifest_json
-        unless @manifest_json
-          if manifest && !manifest.empty?
-            # note that we are preserving the :manifest key at the root only to
-            # avoid having to change how the manifest is interpreted by Repose.
-            manifest_hash = { :manifest => manifest }
-            @manifest_json = ::RightSupport::Data::HashTools.deep_sorted_json(manifest_hash, pretty=true).freeze
-          else
-            @manifest_json = EMPTY_MANIFEST_JSON
-          end
+    def manifest_json
+      unless @manifest_json
+        if manifest && !manifest.empty?
+          # note that we are preserving the :manifest key at the root only to
+          # avoid having to change how the manifest is interpreted by Repose.
+          manifest_hash = { :manifest => manifest }
+          @manifest_json = ::RightSupport::Data::HashTools.deep_sorted_json(manifest_hash, pretty=true).freeze
+        else
+          @manifest_json = EMPTY_MANIFEST_JSON
         end
-        @manifest_json
       end
+      @manifest_json
+    end
 
-      def resource_hash
-        unless @resource_hash
-          @resource_hash = ::Digest::MD5.hexdigest(manifest_json).freeze
-        end
-        @resource_hash
+    def resource_hash
+      unless @resource_hash
+        @resource_hash = ::Digest::MD5.hexdigest(manifest_json).freeze
       end
-
+      @resource_hash
     end
+
   end
 end

data/lib/right_scraper/resources/workflow.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2013 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -21,35 +21,34 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
-module RightScraper
-
-  module Resources
-
-    class Workflow < Base
-
-      METADATA_EXT = '.meta'
-      DEFINITION_EXT = '.def'
- 
-      # Relative path to definition file
-      # @pos must be set before this can be called
-      #
-      # === Return
-      # path(String):: Path to definition file
-      def definition_path
-        path = @pos
-      end
-
-      # Relative path to metadata file
-      # @pos must be set before this can be called
-      #
-      # === Return
-      # path(String):: Path to metadata file
-      def metadata_path
-        path = @pos.chomp(File.extname(@pos)) + METADATA_EXT if @pos
-      end
+# ancestor
+require 'right_scraper/resources'
 
+module RightScraper::Resources
+
+  class Workflow < ::RightScraper::Resources::Base
+
+    METADATA_EXT = '.meta'
+    DEFINITION_EXT = '.def'
+
+    # Relative path to definition file
+    # @pos must be set before this can be called
+    #
+    # === Return
+    # path(String):: Path to definition file
+    def definition_path
+      path = @pos
+    end
+
+    # Relative path to metadata file
+    # @pos must be set before this can be called
+    #
+    # === Return
+    # path(String):: Path to metadata file
+    def metadata_path
+      path = @pos.chomp(File.extname(@pos)) + METADATA_EXT if @pos
     end
 
   end
-end
 
+end

data/lib/right_scraper/retrievers.rb

@@ -0,0 +1,34 @@
+#
+# Copyright (c) 2013 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+# ancestor
+require 'right_scraper'
+
+module RightScraper
+  module Retrievers
+    autoload :Base, 'right_scraper/retrievers/base'
+    autoload :CheckoutBase, 'right_scraper/retrievers/checkout_base'
+    autoload :Download, 'right_scraper/retrievers/download'
+    autoload :Git, 'right_scraper/retrievers/git'
+    autoload :Svn, 'right_scraper/retrievers/svn'
+  end
+end

data/lib/right_scraper/retrievers/base.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2013 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -21,102 +21,98 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
-module RightScraper
-  module Retrievers
-    # Base class for all retrievers.
-    #
-    # Retrievers fetch remote repositories into a given path
-    # They will attempt to fetch incrementally when possible (e.g. leveraging
-    # the underlying source control management system incremental capabilities) 
-    class Base
+# ancestor
+require 'right_scraper/retrievers'
 
-      # Integer:: optional maximum size permitted for repositories
-      attr_accessor :max_bytes
+require 'fileutils'
 
-      # Integer:: optional maximum number of seconds for any single
-      # retrieve operation.
-      attr_accessor :max_seconds
+module RightScraper::Retrievers
 
-      # RightScraper::Repositories::Base:: repository currently being retrieved
-      attr_reader :repository
+  # Base class for all retrievers.
+  #
+  # Retrievers fetch remote repositories into a given path
+  # They will attempt to fetch incrementally when possible (e.g. leveraging
+  # the underlying source control management system incremental capabilities)
+  class Base
 
-      # String:: Path to directory where files are retrieved
-      attr_reader :repo_dir
+    attr_accessor :max_bytes, :max_seconds
 
-      # exceptions
-      class RetrieverError < Exception; end
+    attr_reader :logger, :repository, :repo_dir
 
-      # Create a new retriever for the given repository.  This class
-      # recognizes several options, and subclasses may recognize
-      # additional options.  Options may never be required.
-      #
-      # === Options
-      # <tt>:basedir</tt>:: Required, base directory where all files should be retrieved
-      # <tt>:max_bytes</tt>:: Maximum number of bytes to read
-      # <tt>:max_seconds</tt>:: Maximum number of seconds to spend reading
-      # <tt>:logger</tt>:: Logger to use
-      #
-      # === Parameters
-      # repository(RightScraper::Repositories::Base):: repository to scrape
-      # options(Hash):: retriever options
-      #
-      # === Raise
-      # 'Missing base directory':: if :basedir option is missing
-      def initialize(repository, options={})
-        raise 'Missing base directory' unless options[:basedir]
-        @repository = repository
-        @max_bytes = options[:max_bytes] || nil
-        @max_seconds = options[:max_seconds] || nil
-        @basedir = options[:basedir]
-        @repo_dir = RightScraper::Retrievers::Base.repo_dir(@basedir, repository)
-        @logger = options[:logger] || RightScraper::Logger.new
-        @logger.repository = repository
-        @logger.operation(:initialize, "setting up in #{@repo_dir}") do
-          FileUtils.mkdir_p(@repo_dir)
-        end
-      end
+    # exceptions
+    class RetrieverError < Exception; end
 
-      # Determines if retriever is available (has required CLI tools, etc.)
-      def available?
-        raise NotImplementedError
-      end
-
-      # Paths to ignore when traversing the filesystem.  Mostly used for
-      # things like Git and Subversion version control directories.
-      #
-      # === Return
-      # list(Array):: list of filenames to ignore.
-      def ignorable_paths
-        []
+    # Create a new retriever for the given repository.  This class
+    # recognizes several options, and subclasses may recognize
+    # additional options.  Options may never be required.
+    #
+    # === Options
+    # <tt>:basedir</tt>:: Required, base directory where all files should be retrieved
+    # <tt>:max_bytes</tt>:: Maximum number of bytes to read
+    # <tt>:max_seconds</tt>:: Maximum number of seconds to spend reading
+    # <tt>:logger</tt>:: Logger to use
+    #
+    # === Parameters
+    # repository(RightScraper::Repositories::Base):: repository to scrape
+    # options(Hash):: retriever options
+    #
+    # === Raise
+    # 'Missing base directory':: if :basedir option is missing
+    def initialize(repository, options={})
+      raise 'Missing base directory' unless options[:basedir]
+      @repository = repository
+      @max_bytes = options[:max_bytes] || nil
+      @max_seconds = options[:max_seconds] || nil
+      @basedir = options[:basedir]
+      @repo_dir = RightScraper::Retrievers::Base.repo_dir(@basedir, repository)
+      unless @logger = options[:logger]
+        raise ::ArgumentError, ':logger is required'
       end
-
-      # Retrieve repository, overridden in heirs
-      def retrieve
-        raise NotImplementedError
+      @logger.operation(:initialize, "setting up in #{@repo_dir}") do
+        ::FileUtils.mkdir_p(@repo_dir)
       end
+    end
 
-      # Path to directory where given repo should be or was downloaded
-      #
-      # === Parameters
-      # root_dir(String):: Path to directory containing all scraped repositories
-      # repo(Hash|RightScraper::Repositories::Base):: Remote repository corresponding to local directory
-      #
-      # === Return
-      # String:: Path to local directory that corresponds to given repository
-      def self.repo_dir(root_dir, repo)
-        repo = RightScraper::Repositories::Base.from_hash(repo) if repo.is_a?(Hash)
-        dir_name  = repo.repository_hash
-        dir_path  = File.join(root_dir, dir_name)
-        "#{dir_path}/repo"
-      end
+    # Determines if retriever is available (has required CLI tools, etc.)
+    def available?
+      raise ::NotImplementedError
+    end
 
-      protected
+    # Paths to ignore when traversing the filesystem.  Mostly used for
+    # things like Git and Subversion version control directories.
+    #
+    # === Return
+    # list(Array):: list of filenames to ignore.
+    def ignorable_paths
+      []
+    end
 
-      # (Hash) Lookup table from textual description of scraper type
-      # ('cookbook' or 'workflow' currently) to the class that
-      # represents that scraper.
-      @@types = {} unless class_variable_defined?(:@@types)
+    # Retrieve repository, overridden in heirs
+    def retrieve
+      raise ::NotImplementedError
+    end
 
+    # Path to directory where given repo should be or was downloaded
+    #
+    # === Parameters
+    # root_dir(String):: Path to directory containing all scraped repositories
+    # repo(Hash|RightScraper::Repositories::Base):: Remote repository corresponding to local directory
+    #
+    # === Return
+    # String:: Path to local directory that corresponds to given repository
+    def self.repo_dir(root_dir, repo)
+      repo = ::RightScraper::Repositories::Base.from_hash(repo) if repo.is_a?(Hash)
+      dir_name  = repo.repository_hash
+      dir_path  = ::File.join(root_dir, dir_name)
+      "#{dir_path}/repo"
     end
+
+    protected
+
+    # (Hash) Lookup table from textual description of scraper type
+    # ('cookbook' or 'workflow' currently) to the class that
+    # represents that scraper.
+    @@types = {} unless class_variable_defined?(:@@types)
+
   end
 end