right_scraper 1.0.26 → 3.0.0

data/lib/right_scraper/scanners/workflow_manifest.rb

@@ -0,0 +1,86 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require File.expand_path(File.join(File.dirname(__FILE__), 'base'))
+require 'digest/sha1'
+
+module RightScraper
+  module Scanners
+    # Build manifests from a filesystem.
+    class WorkflowManifest < Base
+      # Create a new manifest scanner.  Does not accept any new arguments.
+      def initialize(*args)
+        super
+        @manifest = {}
+      end
+
+      # Retrieve relative workflow files positions
+      #
+      # === Parameters
+      # workflow(Resources::Workflow):: Workflow whose manifest is being built
+      def begin(workflow)
+        @workflow = workflow
+        @metadata_filename = File.basename(@workflow.metadata_path)
+        @definition_filename = File.basename(@workflow.definition_path)
+      end
+
+      # Complete a scan for the given resource.
+      #
+      # === Parameters ===
+      # resource(RightScraper::Resources::Base):: resource to scan
+      def end(resource)
+        resource.manifest = @manifest
+        @manifest = {}
+      end
+
+      # Notice a file during scanning.
+      #
+      # === Block ===
+      # Return the data for this file.  We use a block because it may
+      # not always be necessary to read the data.
+      #
+      # === Parameters ===
+      # relative_position(String):: relative pathname for file from root of resource
+      def notice(relative_position)
+        if [ @metadata_filename, @definition_filename ].include?(relative_position)
+          @manifest[relative_position] = Digest::SHA1.hexdigest(yield)
+        end
+      end
+
+      # Notice a directory during scanning.  Since the workflow definition and
+      # metadata live in the root directory we don't need to recurse,
+      # but we do need to go into the first directory (identified by
+      # +relative_position+ being +nil+).
+      #
+      # === Parameters
+      # relative_position(String):: relative pathname for the directory from root of workflow
+      #
+      # === Returns
+      # Boolean:: should the scanning recurse into the directory
+      def notice_dir(relative_position)
+        relative_position == nil
+      end
+ 
+    end
+  end
+end

data/lib/right_scraper/scanners/workflow_metadata.rb

@@ -0,0 +1,70 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'json'
+
+module RightScraper
+  module Scanners
+    # Load workflow metadata from a filesystem.
+    class WorkflowMetadata < Base
+      # Begin a scan for the given workflow.
+      #
+      # === Parameters
+      # workflow(RightScraper::Resources::Workflow):: workflow to scan
+      def begin(workflow)
+        @workflow = workflow
+        @metadata_filename = File.basename(workflow.metadata_path)
+      end
+
+      # Notice a file during scanning.
+      #
+      # === Block
+      # Return the data for this file.  We use a block because it may
+      # not always be necessary to read the data.
+      #
+      # === Parameters
+      # relative_position(String):: relative pathname for the file from root of workflow
+      def notice(relative_position)
+        if relative_position == @metadata_filename
+          @logger.operation(:metadata_parsing) do
+            @workflow.metadata = JSON.parse(yield)
+          end
+        end
+      end
+
+      # Notice a directory during scanning.  Since the workflow definition and
+      # metadata live in the root directory we don't need to recurse,
+      # but we do need to go into the first directory (identified by
+      # +relative_position+ being +nil+).
+      #
+      # === Parameters
+      # relative_position(String):: relative pathname for the directory from root of workflow
+      #
+      # === Returns
+      # Boolean:: should the scanning recurse into the directory
+      def notice_dir(relative_position)
+        relative_position == nil
+      end
+    end
+  end
+end

data/lib/right_scraper/scanners/workflow_s3_upload.rb

@@ -0,0 +1,85 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+require 'right_aws'
+require 'json'
+
+module RightScraper
+  module Scanners
+    # Upload workflow definition and metadata to an S3 bucket.
+    class WorkflowS3Upload < Base
+      # Create a new S3Upload.  In addition to the options recognized
+      # by Scanner, this class recognizes <tt>:s3_key</tt>,
+      # <tt>:s3_secret</tt>, and <tt>:s3_bucket</tt> and requires all
+      # of those.
+      #
+      # === Options
+      # <tt>:s3_key</tt>:: Required.  S3 access key.
+      # <tt>:s3_secret</tt>:: Required.  S3 secret key.
+      # <tt>:s3_bucket</tt>:: Required.  Bucket to upload workflows to.
+      #
+      # === Parameters
+      # options(Hash):: scanner options
+      def initialize(options={})
+        super
+        s3_key = options.fetch(:s3_key)
+        s3_secret = options.fetch(:s3_secret)
+        s3 = RightAws::S3.new(aws_access_key_id=s3_key,
+                              aws_secret_access_key=s3_secret,
+                              :logger => Logger.new)
+        @bucket = s3.bucket(options.fetch(:s3_bucket))
+        raise "Need an actual, existing S3 bucket!" if @bucket.nil?
+      end
+
+      # Upon ending a scan for a workflows, upload the workflows
+      # contents to S3.
+      #
+      # === Parameters
+      # workflows(RightScraper::Workflows):: Workflow to scan
+      def end(workflow)
+        @bucket.put(File.join('Workflows', workflow.resource_hash),
+                    {
+                      :metadata => workflow.metadata,
+                      :manifest => workflow.manifest
+                    }.to_json)
+      end
+
+      # Upload a file during scanning.
+      #
+      # === Block
+      # Return the data for this file.  We use a block because it may
+      # not always be necessary to read the data.
+      #
+      # === Parameters
+      # relative_position(String):: relative pathname for file from root of cookbook
+      def notice(relative_position)
+        # TBD: Only uplad definition and metadata, will there be more files?
+        contents = yield
+        name = Digest::SHA1.hexdigest(contents)
+        path = File.join('Files', name)
+        unless @bucket.key(path).exists?
+          @bucket.put(path, contents)
+        end
+      end
+    end
+  end
+end

data/lib/right_scraper/scraper.rb

@@ -1,18 +1,18 @@
 #--
-# Copyright: Copyright (c) 2010 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
 # 'Software'), to deal in the Software without restriction, including
 # without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to 
+# distribute, sublicense, and/or sell copies of the Software, and to
 # permit persons to whom the Software is furnished to do so, subject to
 # the following conditions:
 #
 # The above copyright notice and this permission notice shall be
 # included in all copies or substantial portions of the Software.
 #
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
 # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
@@ -20,94 +20,118 @@
 # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
+require File.expand_path(File.join(File.dirname(__FILE__), 'logger'))
 
-module RightScale
-
-  # Hash of repository types associated with corresponding scraper class
-  SCRAPERS = { 'git'      => RightScale::GitScraper,
-               'svn'      => RightScale::SvnScraper,
-               'download' => RightScale::DownloadScraper }
+module RightScraper
 
   # Library main entry point. Instantiate this class and call the scrape
-  # method to download or update a remote repository to the local disk.
+  # method to download or update a remote repository to the local disk and
+  # run a scraper on the resulting files.
   class Scraper
 
-    # (String) Path to directory where remote repository was downloaded
-    # Note: This will be a subfolder of the scrape directory (directory given to initializer)
-    attr_reader :last_repo_dir
-    
+    # (Array):: Scraped resources 
+    attr_reader :resources
+ 
     # Initialize scrape destination directory
     #
-    # === Parameters
-    # scrape_dir(String):: Scrape destination directory
-    # max_bytes(Integer):: Maximum size allowed for repos, -1 for no limit (default)
-    # max_seconds(Integer):: Maximum number of seconds a single scrape operation should take, -1 for no limit (default)
-    def initialize(scrape_dir, max_bytes = -1, max_seconds = -1)
-      @scrape_dir = scrape_dir
-      @max_bytes = max_bytes
-      @max_seconds = max_seconds
-      @scrapers = {}
+    # === Options
+    # <tt>:kind</tt>:: Type of scraper that will traverse directory for resources, one of :cookbook or :workflow
+    # <tt>:basedir</tt>:: Local directory where files are retrieved and scraped, use temporary directory if nil
+    # <tt>:max_bytes</tt>:: Maximum number of bytes to read from remote repo, unlimited if nil
+    # <tt>:max_seconds</tt>:: Maximum number of seconds to spend reading from remote repo, unlimited if nil
+    def initialize(options={})
+      @temporary = !options.has_key?(:basedir)
+      options[:basedir] ||= Dir.mktmpdir
+      @logger = ScraperLogger.new
+      @options = options.merge({:logger => @logger})
+      @resources = []
     end
 
-    # Scrape given repository.
-    # Create unique directory inside scrape directory when called for the first time.
-    # Update content of unique directory incrementally when possible with further calls.
+    # Scrape given repository, depositing files into the scrape
+    # directory.  Update content of unique directory incrementally
+    # when possible with further calls.
     #
     # === Parameters
-    # repo(Hash|RightScale::Repository):: Repository to be scraped
-    # Note: repo can either be a Hash or a RightScale::Repo instance.
-    # See the RightScale::Repo class for valid Hash keys.
-    # incremental(FalseClass|TrueClass):: Whether scrape should be incremental if possible (true by default)
+    # repo(Hash|RightScraper::Repositories::Base):: Repository to be scraped
+    #   Note: repo can either be a Hash or a RightScraper::Repositories::Base instance.
+    #         See the RightScraper::Repositories::Base class for valid Hash keys.
     #
     # === Block
     # If a block is given, it will be called back with progress information
-    # the block should take two arguments:
-    # - first argument is the string containing the info
-    # - second argument is a boolean indicating whether to increment progress
-    # The block is called exactly once with the increment flag set to true
+    # the block should take four arguments:
+    # - first argument is one of <tt>:begin</tt>, <tt>:commit</tt>,
+    #   <tt>:abort</tt> which signifies what
+    #   the scraper is trying to do and where it is when it does it
+    # - second argument is a symbol describing the operation being performed
+    #   in an easy-to-match way
+    # - third argument is optional further explanation
+    # - fourth argument is the exception pending (only relevant for <tt>:abort</tt>)
     #
     # === Return
     # true:: If scrape was successful
-    # false:: If scrape failed, call error_message for information on failure
+    # false:: If scrape failed, call errors for information on failure
     #
     # === Raise
     # 'Invalid repository type':: If repository type is not known
     def scrape(repo, incremental=true, &callback)
-      repo = RightScale::Repository.from_hash(repo) if repo.is_a?(Hash)
-      repo.repo_type = repo.repo_type.to_s # In case it's a symbol
-      raise "Invalid repository type" unless SCRAPERS.include?(repo.repo_type)
-      @scraper = @scrapers[repo.repo_type] ||= SCRAPERS[repo.repo_type].new(@scrape_dir, @max_bytes, @max_seconds)
-      @scraper.scrape(repo, incremental, &callback)
-      @last_repo_dir = @scraper.current_repo_dir
-      @scraper.succeeded?
+      errorlen = errors.size
+      repo = RightScraper::Repositories::Base.from_hash(repo) if repo.is_a?(Hash)
+      @logger.callback = callback
+      begin
+        # 1. Retrieve the files
+        retriever = nil
+        @logger.operation(:retrieving, "from #{repo}") do
+          retriever = repo.retriever(@options)
+          retriever.retrieve
+        end
+
+        # 2. Now scrape if there is a scraper in the options
+        @logger.operation(:scraping, retriever.repo_dir) do
+          if @options[:kind]
+            options = @options.merge({:ignorable_paths => retriever.ignorable_paths,
+                                      :repo_dir        => retriever.repo_dir,
+                                      :repository      => retriever.repository})
+            scraper = RightScraper::Scrapers::Base.scraper(options)
+            @resources += scraper.scrape
+          end
+        end
+
+        # 3. Cleanup if temporary
+        FileUtils.remove_entry_secure(@options[:basedir]) if @temporary
+      rescue
+        # logger handles communication with the end user and appending
+        # to our error list, we just need to keep going.
+      end
+      @logger.callback = nil
+      errors.size == errorlen
     end
-    
-    # Retrieve directory path where repo was or would be downloaded
+
+    # Path to directory where given repo should be or was downloaded
     #
     # === Parameters
-    # repo(Hash|RightScale::Repository):: Remote repository corresponding to local directory
+    # repo(Hash|RightScraper::Repositories::Base):: Remote repository corresponding to local directory
     #
-    # === Return 
-    # repo_dir(String):: Path to local directory that corresponds to given repository
+    # === Return
+    # String:: Path to local directory that corresponds to given repository
     def repo_dir(repo)
-      repo_dir = RightScale::ScraperBase.repo_dir(scrape_dir, repo)
+      RightScraper::Retrievers::Base.repo_dir(@options[:basedir], repo)
     end
 
-    # Error messages in case of failure
-    #
-    # === Return
-    # errors(Array):: Error messages or empty array if no error
+    # (Array):: Error messages in case of failure
     def errors
-      errors = @scraper && @scraper.errors || []
+      @logger.errors
     end
 
-    # Was scraping successful?
-    # Call error_message to get error messages if false
+   # Was scraping successful?
+    # Call errors to get error messages if false
     #
     # === Return
-    # succeeded(Boolean):: true if scrape finished with no error, false otherwise.
+    # Boolean:: true if scrape finished with no error, false otherwise.
     def succeeded?
-      succeeded = errors.size == 0
+      errors.empty?
     end
+    alias_method :successful?, :succeeded?
+
   end
 end
+

data/lib/right_scraper/scraper_logger.rb

@@ -0,0 +1,61 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+module RightScraper
+
+  class ScraperLogger < Logger
+    attr_accessor :errors
+    attr_accessor :callback
+
+    def add(severity, message=nil, progname=nil)
+      if severity >= (self.level || Logger::WARN)
+        if message.nil?
+          if block_given?
+            message = yield
+          else
+            message = progname
+            progname = self.progname
+          end
+        end
+        @errors << [nil, :log,
+          {:severity => severity,
+            :message => message,
+            :progname => progname}]
+      end
+    end
+
+    def initialize
+      @errors = []
+    end
+
+    def note_phase(phase, type, explanation, exception=nil)
+      @callback.call(phase, type, explanation, exception) unless @callback.nil?
+      super
+    end
+
+    def note_error(exception, type, explanation="")
+      @errors << [exception, type, explanation]
+    end
+  end
+
+end