right_scraper 1.0.26 → 3.0.0

data/lib/right_scraper/retrievers/git.rb

@@ -0,0 +1,140 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+require 'git'
+
+module RightScraper
+  module Retrievers
+    # Retriever for resources stored in a git repository.
+    class Git < CheckoutBasedRetriever
+      # In addition to normal retriever initialization, if the
+      # underlying repository has a credential we need to initialize a
+      # fresh SSHAgent and add the credential to it.
+      def retrieve
+        RightScraper::Processes::SSHAgent.with do |agent|
+          agent.add_key(@repository.first_credential) unless
+            @repository.first_credential.nil?
+          super
+        end
+      end
+
+      # Return true if a checkout exists.  Currently tests for .git in
+      # the checkout.
+      #
+      # === Returns ===
+      # Boolean:: true if the checkout already exists (and thus
+      #           incremental updating can occur).
+      def exists?
+        File.exists?(File.join(@repo_dir, '.git'))
+      end
+
+      def do_fetch(git)
+        @logger.operation(:fetch) do
+          git.tags.each {|tag| git.lib.tag(['-d', tag.name])}
+          git.fetch(['--all', '--prune', '--tags'])
+        end
+      end
+
+      # Incrementally update the checkout.  The operations are as follows:
+      # * checkout #tag
+      # * if #tag is the head of a branch:
+      #   * find that branch's remote
+      #   * fetch it
+      #   * merge changes
+      #   * update @repository#tag
+      # Note that if #tag is a SHA revision or a tag that exists in the
+      # current repository, no fetching is done.
+      def do_update
+        git = ::Git.open(@repo_dir)
+        do_fetch(git)
+        git.reset_hard
+        do_checkout_revision(git)
+        do_update_tag(git)
+      end
+
+      def do_update_tag(git)
+        @repository = @repository.clone
+        @repository.tag = git.gtree("HEAD").sha
+      end
+
+      # Clone the remote repository.  The operations are as follows:
+      # * clone repository to @repo_dir
+      # * checkout #tag
+      # * update @repository#tag
+      def do_checkout
+        super
+        git = @logger.operation(:cloning, "to #{@repo_dir}") do
+          ::Git.clone(@repository.url, @repo_dir)
+        end
+        do_fetch(git)
+        do_checkout_revision(git)
+        do_update_tag git
+      end
+
+      def do_checkout_revision(git)
+        @logger.operation(:checkout_revision) do
+          case
+          when tag?(git, repo_tag) && branch?(git, repo_tag) then
+            raise "Ambiguous reference: '#{repo_tag}' denotes both a branch and a tag"
+          when branch = find_remote_branch(git, repo_tag) then
+            branch.checkout
+          when branch = find_local_branch(git, repo_tag) then
+            branch.checkout
+          else
+            git.checkout(repo_tag)
+          end
+        end if repo_tag
+      end
+
+      def tag?(git, name)
+        git.tags.find {|t| t.name == name}
+      end
+
+      def branch?(git, name)
+        git.branches.find {|t| t.name == name}
+      end
+
+      def repo_tag
+        name = (@repository.tag || "master").chomp
+        name = "master" if name.empty?
+        name
+      end
+
+      def find_branch(git, tag)
+        find_local_branch(git, tag) || find_remote_branch(git, tag)
+      end
+
+      def find_local_branch(git, name)
+        git.branches.local.find {|b| b.name == name}
+      end
+
+      def find_remote_branch(git, name)
+        git.branches.remote.find {|b| b.name == name}
+      end
+
+      # Ignore .git directories.
+      def ignorable_paths
+        ['.git']
+      end
+    end
+  end
+end

data/lib/right_scraper/retrievers/svn.rb

@@ -0,0 +1,87 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+require File.join(File.dirname(__FILE__), '..', 'svn_client')
+
+require 'process_watcher'
+
+module RightScraper
+  module Retrievers
+    # Retriever for svn repositories
+    class Svn < CheckoutBasedRetriever
+
+      include RightScraper::SvnClient
+
+      # Return true if a checkout exists.  Currently tests for .svn in
+      # the checkout.
+      #
+      # === Returns
+      # Boolean:: true if the checkout already exists (and thus
+      #           incremental updating can occur).
+      def exists?
+        File.exists?(File.join(@repo_dir, '.svn'))
+      end
+
+      # Incrementally update the checkout.  The operations are as follows:
+      # * update to #tag
+      # In theory if #tag is a revision number that already exists no
+      # update is necessary.  It's not clear if the SVN client libraries
+      # are bright enough to notice this.
+      def do_update
+        @logger.operation(:update) do
+          run_svn("update", get_tag_argument)
+        end
+        do_update_tag
+      end
+
+      # Update our idea of what the head of the repository is.  We
+      # would like to use svn info, but that doesn't do the right
+      # thing all the time; the right thing to do is to run log and
+      # pick out the first tag.
+      def do_update_tag
+        @repository = @repository.clone
+        log = run_svn("log", "-r", 'HEAD')
+        log.split(/\n/).each do |line|
+          if line =~ /^r(\d+)/
+            @repository.tag = $1
+            break
+          end
+        end
+      end
+
+      # Check out the remote repository.  The operations are as follows:
+      # * checkout repository at #tag to @repo_dir
+      def do_checkout
+        super
+        @logger.operation(:checkout_revision) do
+          run_svn_no_chdir("checkout", @repository.url, @repo_dir, get_tag_argument)
+        end
+        do_update_tag
+      end
+
+      # Ignore .svn directories.
+      def ignorable_paths
+        ['.svn']
+      end
+    end
+  end
+end

data/lib/right_scraper/scanners/base.rb

@@ -0,0 +1,111 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+module RightScraper
+  module Scanners
+    # Base class for scanning filesystems.  Subclasses should override
+    # #notice and may override #new, #begin, #end and
+    # #notice_dir.
+    #
+    # Overriding #new is useful for getting
+    # additional arguments.  Overriding #begin allows you to do
+    # processing before the scan of a given resource begins;
+    # overriding #end allows you to do processing after it completes.
+    #
+    # Most processing will occur in #notice, which notifies you that a
+    # file has been detected, and in #notice_dir.  In #notice you are
+    # handed the relative position of the file from the start of the
+    # resource; so if you were scanning <tt>/a/resource</tt> and
+    # noticed a file <tt>b/c</tt>, #notice would be called with
+    # <tt>"b/c"</tt>, even though the full pathname is
+    # <tt>/a/resource/b/c</tt>.  If you decide you need the actual
+    # data, #notice takes a block which will return that data to you
+    # if you +yield+.
+    #
+    # In #notice_dir you are handed the relative position of a
+    # directory.  The return value determines whether you find the
+    # directory worth recursing into, or not--as an example, when
+    # looking for the <tt>metadata.json</tt> file it is never
+    # necessary to descend past the topmost directory of the resource,
+    # but the same is not true when building a manifest.
+    class Base
+      # Create a new Scanner.  Recognizes options as given.  Some
+      # options may be required, others optional.  This class recognizes
+      # only _:logger_.
+      #
+      # === Options ===
+      # _:logger_:: Optional.  Logger currently being used
+      #
+      # === Parameters ===
+      # options(Hash):: scanner options
+      def initialize(options={})
+        @logger = options.fetch(:logger, RightScraper::Logger.new)
+      end
+
+      # Notification that all scans for this repository have
+      # completed.
+      def finish
+      end
+
+      # Begin a scan for the given resource.
+      #
+      # === Parameters ===
+      # resource(RightScraper::Resource::Base):: resource to scan
+      def begin(resource)
+      end
+
+      # Finish a scan for the given resource.
+      #
+      # === Parameters ===
+      # resource(RightScraper::Resource::Base):: resource that just finished
+      #                                  scanning
+      def end(resource)
+      end
+
+      # Notice a file during scanning.
+      #
+      # === Block ===
+      # Return the data for this file.  We use a block because it may
+      # not always be necessary to read the data.
+      #
+      # === Parameters ===
+      # relative_position(String):: relative pathname for _pathname_
+      #                             from root of resource
+      def notice(relative_position)
+      end
+
+      # Notice a directory during scanning.  Returns true if the scanner
+      # should recurse into the directory (the default behavior)
+      #
+      # === Parameters ===
+      # relative_position(String):: relative pathname for the directory
+      #                             from root of resource
+      #
+      # === Returns ===
+      # Boolean:: should the scanning recurse into the directory
+      def notice_dir(relative_position)
+        true
+      end
+    end
+  end
+end

data/lib/right_scraper/scanners/cookbook_manifest.rb

@@ -0,0 +1,59 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require File.expand_path(File.join(File.dirname(__FILE__), 'base'))
+require 'digest/sha1'
+
+module RightScraper
+  module Scanners
+    # Build manifests from a filesystem.
+    class CookbookManifest < Base
+      # Create a new manifest scanner.  Does not accept any new arguments.
+      def initialize(*args)
+        super
+        @manifest = {}
+      end
+
+      # Complete a scan for the given resource.
+      #
+      # === Parameters ===
+      # resource(RightScraper::Resources::Base):: resource to scan
+      def end(resource)
+        resource.manifest = @manifest
+        @manifest = {}
+      end
+
+      # Notice a file during scanning.
+      #
+      # === Block ===
+      # Return the data for this file.  We use a block because it may
+      # not always be necessary to read the data.
+      #
+      # === Parameters ===
+      # relative_position(String):: relative pathname for file from root of resource
+      def notice(relative_position)
+        @manifest[relative_position] = Digest::SHA1.hexdigest(yield)
+      end
+    end
+  end
+end

data/lib/right_scraper/scanners/cookbook_metadata.rb

@@ -0,0 +1,69 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'json'
+
+module RightScraper
+  module Scanners
+    # Load cookbook metadata from a filesystem.
+    class CookbookMetadata < Base
+      # Begin a scan for the given cookbook.
+      #
+      # === Parameters
+      # cookbook(RightScraper::Resources::Cookbook):: cookbook to scan
+      def begin(cookbook)
+        @cookbook = cookbook
+      end
+
+      # Notice a file during scanning.
+      #
+      # === Block
+      # Return the data for this file.  We use a block because it may
+      # not always be necessary to read the data.
+      #
+      # === Parameters
+      # relative_position(String):: relative pathname for the file from root of cookbook
+      def notice(relative_position)
+        if relative_position == "metadata.json"
+          @logger.operation(:metadata_parsing) do
+            @cookbook.metadata = JSON.parse(yield)
+          end
+        end
+      end
+
+      # Notice a directory during scanning.  Since metadata.json is by
+      # definition only in the root directory we don't need to recurse,
+      # but we do need to go into the first directory (identified by
+      # +relative_position+ being +nil+).
+      #
+      # === Parameters
+      # relative_position(String):: relative pathname for the directory from root of cookbook
+      #
+      # === Returns
+      # Boolean:: should the scanning recurse into the directory
+      def notice_dir(relative_position)
+        relative_position == nil
+      end
+    end
+  end
+end

data/lib/right_scraper/scanners/cookbook_s3_upload.rb

@@ -0,0 +1,84 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+require 'right_aws'
+require 'json'
+
+module RightScraper
+  module Scanners
+    # Upload scanned files to an S3 bucket.
+    class CookbookS3Upload < Base
+      # Create a new S3Upload.  In addition to the options recognized
+      # by Scanner, this class recognizes <tt>:s3_key</tt>,
+      # <tt>:s3_secret</tt>, and <tt>:s3_bucket</tt> and requires all
+      # of those.
+      #
+      # === Options
+      # <tt>:s3_key</tt>:: Required.  S3 access key.
+      # <tt>:s3_secret</tt>:: Required.  S3 secret key.
+      # <tt>:s3_bucket</tt>:: Required.  Bucket to upload cookbooks to.
+      #
+      # === Parameters
+      # options(Hash):: scanner options
+      def initialize(options={})
+        super
+        s3_key = options.fetch(:s3_key)
+        s3_secret = options.fetch(:s3_secret)
+        s3 = RightAws::S3.new(aws_access_key_id=s3_key,
+                              aws_secret_access_key=s3_secret,
+                              :logger => Logger.new)
+        @bucket = s3.bucket(options.fetch(:s3_bucket))
+        raise "Need an actual, existing S3 bucket!" if @bucket.nil?
+      end
+
+      # Upon ending a scan for a cookbook, upload the cookbook
+      # contents to S3.
+      #
+      # === Parameters
+      # cookbook(RightScraper::Cookbook):: cookbook to scan
+      def end(cookbook)
+        @bucket.put(File.join('Cooks', cookbook.resource_hash),
+                    {
+                      :metadata => cookbook.metadata,
+                      :manifest => cookbook.manifest
+                    }.to_json)
+      end
+
+      # Upload a file during scanning.
+      #
+      # === Block
+      # Return the data for this file.  We use a block because it may
+      # not always be necessary to read the data.
+      #
+      # === Parameters
+      # relative_position(String):: relative pathname for file from root of cookbook
+      def notice(relative_position)
+        contents = yield
+        name = Digest::SHA1.hexdigest(contents)
+        path = File.join('Files', name)
+        unless @bucket.key(path).exists?
+          @bucket.put(path, contents)
+        end
+      end
+    end
+  end
+end

data/lib/right_scraper/scanners/union.rb

@@ -0,0 +1,89 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+module RightScraper
+  module Scanners
+    # Union scanner, to permit running multiple scanners while only
+    # walking the fs once.
+    class Union
+      # Create a new union scanner.  Recognizes no new options.
+      #
+      # === Parameters
+      # classes(List):: List of Scanner classes to run
+      # options(Hash):: scanner options
+      def initialize(classes, options={})
+        @subscanners = classes.map {|klass| klass.new(options)}
+      end
+
+      # Notify subscanners that all scans have completed.
+      def finish
+        @subscanners.each {|scanner| scanner.finish}
+      end
+
+      # Begin a scan for the given resource.
+      #
+      # === Parameters
+      # resource(RightScraper::Resource::Base):: resource to scan
+      def begin(resource)
+        @subscanners.each {|scanner| scanner.begin(resource)}
+      end
+
+      # Finish a scan for the given resource.
+      #
+      # === Parameters
+      # resource(RightScraper::Resource::Base):: resource that just finished scanning
+      def end(resource)
+        @subscanners.each {|scanner| scanner.end(resource)}
+      end
+
+      # Notice a file during scanning.
+      #
+      # === Block
+      # Return the data for this file.  We use a block because it may
+      # not always be necessary to read the data.
+      #
+      # === Parameters
+      # relative_position(String):: relative pathname for the file from the root of resource
+      def notice(relative_position)
+        data = nil
+        @subscanners.each {|scanner| scanner.notice(relative_position) {
+            data = yield if data.nil?
+            data
+          }
+        }
+      end
+
+      # Notice a directory during scanning.  Returns true if any of the
+      # subscanners report that they should recurse into the directory.
+      #
+      # === Parameters
+      # relative_position(String):: relative pathname for directory from root of resource
+      #
+      # === Returns
+      # Boolean:: should the scanning recurse into the directory
+      def notice_dir(relative_position)
+        @subscanners.any? {|scanner| scanner.notice_dir(relative_position)}
+      end
+    end
+  end
+end