right_scraper 1.0.26 → 3.0.0

data/Gemfile

@@ -0,0 +1,16 @@
+source "http://rubygems.org"
+
+gem "json", "~> 1.4.5"
+gem "blackwinter-git", "~> 1.2.7"
+gem "libarchive", "~> 0.1.2"
+gem "right_aws", "~> 2.0"
+gem "process_watcher", "~> 0.3"
+
+group :development do
+  gem "rspec", "~> 2.3"
+  gem "rake"
+  gem "flexmock"
+  gem "ruby-debug", :platform=>"ruby_18"
+  gem "ruby-debug19", :platform=>"ruby_19"
+  gem "rdoc", "~> 2.4"
+end

data/README.rdoc

@@ -4,24 +4,11 @@
 
 === Synopsis
 
-RightScraper provides a simple interface to download and keep repositories up-to-date 
-using various protocols.
-
-The supported protocols include:
-- *git*: RightScraper will clone then pull repos from git
-- *SVN*: RightScraper will checkout then update SVN repositories
-- *tarballs*: Includes uncompressed (.tar), gzip (.tgz, .gzip) and bzip (.bzip, .bzip2) tar files.
-
-The scraper first inspects the local directory to see if the repo has already been scraped
-and if so runs some basic checks before it tries to update it. Incremental updates are not
-supported with tar files.
-
-=== Rationale
-
-The idea is to have many repos that need to be downloaded/kept up-to-date in a central
-place. Point the scraper to this central place and it will take care of creating unique
-local directories for each remote repository and keep that mapping to download changes
-incrementally upon request.
+RightScraper provides a simple interface to download and keep repositories up-to-date
+using various protocols.  It has been decomposed into various modules so that you
+may specify only the functionality (and required libraries and gems) you require.
+This gem depends on all available RightScraper modules, enabling full support at the
+cost of requiring some systems administration work external to Ruby.
 
 == USAGE
 
@@ -30,22 +17,16 @@ incrementally upon request.
  require 'rubygems'
  require 'right_scraper'
 
- scraper = RightScale::Scraper.new('/tmp')
+ scraper = RightScale::Scraper.new(:basedir => '/tmp', :kind => :cookbook)
  scraper.scrape(:type => :git, :url => 'git://github.com/rightscale/right_scraper.git')
 
 == INSTALLATION
 
-RightScraper can be installed by entering the following at the command prompt:
+This module for RightScraper can be installed by entering the following at the command
+prompt:
 
  gem install right_scraper
 
-== DEPENDENCIES
-
-RightScraper relies on the following tools:
-- git
-- svn
-- curl
-
 == TESTING
 
 Install the following RubyGems required for testing:
@@ -59,7 +40,7 @@ The build can be tested using the RSpec gem.
 
 <b>RightScraper</b>
 
-Copyright:: Copyright (c) 2010 RightScale, Inc. 
+Copyright:: Copyright (c) 2010 RightScale, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/Rakefile

@@ -1,18 +1,18 @@
-#--
-# Copyright: Copyright (c) 2010 RightScale, Inc.
+#-- -*-ruby-*-
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
 # 'Software'), to deal in the Software without restriction, including
 # without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to 
+# distribute, sublicense, and/or sell copies of the Software, and to
 # permit persons to whom the Software is furnished to do so, subject to
 # the following conditions:
 #
 # The above copyright notice and this permission notice shall be
 # included in all copies or substantial portions of the Software.
 #
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
 # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
@@ -22,56 +22,68 @@
 #++
 
 require 'rubygems'
+require 'rubygems/package_task'
+require 'bundler/setup'
+
 require 'fileutils'
 require 'rake'
-require 'spec/rake/spectask'
+require 'rspec/core/rake_task'
+require 'rdoc/task'
+require 'rake/clean'
 
 task :default => 'spec'
 
+# == Gem packaging == #
+
+desc "Build right_scraper gem"
+Gem::PackageTask.new(Gem::Specification.load("right_scraper.gemspec")) do |package|
+  package.need_zip = true
+  package.need_tar = true
+end
+
+CLEAN.include('pkg')
+
 # == Unit Tests == #
 
-desc "Run unit tests"
-Spec::Rake::SpecTask.new do |t|
-  t.spec_files = Dir['**/*_spec.rb']
-  t.spec_opts = lambda do
-    IO.readlines(File.join(File.dirname(__FILE__), 'spec', 'spec.opts')).map {|l| l.chomp.split " "}.flatten
-  end
+task :specs => :spec
+
+# == Unit Tests == #
+
+desc 'Run unit tests'
+RSpec::Core::RakeTask.new do |t|
+  t.pattern = 'spec/**/*_spec.rb'
+    t.rspec_opts = ["--color", "--format", "nested"]
 end
 
-desc "Run unit tests with RCov"
-Spec::Rake::SpecTask.new(:rcov) do |t|
-  t.spec_files = Dir['**/*_spec.rb']
-  t.rcov = true
-  t.rcov_opts = lambda do
-    IO.readlines(File.join(File.dirname(__FILE__), 'spec', 'rcov.opts')).map {|l| l.chomp.split " "}.flatten
+namespace :spec do
+  desc "Run unit tests with RCov"
+  RSpec::Core::RakeTask.new(:rcov) do |t|
+    t.pattern = '*/spec/**/*_spec.rb'
+    t.rcov = true
+    t.rcov_opts = %q[--exclude "spec"]
   end
-end
 
-desc "Print Specdoc for unit tests"
-Spec::Rake::SpecTask.new(:doc) do |t|
-   t.spec_opts = ["--format", "specdoc", "--dry-run"]
-   t.spec_files = Dir['**/*_spec.rb']
+  desc "Print Specdoc for unit tests"
+  RSpec::Core::RakeTask.new(:doc) do |t|
+    t.pattern = '*/spec/**/*_spec.rb'
+    t.rspec_opts = ["--format", "documentation"]
+  end
 end
 
-# == Gem Management == #
+# == Documentation == #
 
-desc "Build right_scraper gem"
-task :gem do
-   ruby 'right_scraper.gemspec'
-   pkg_dir = File.join(File.dirname(__FILE__), 'pkg')
-   FileUtils.mkdir_p(pkg_dir)
-   FileUtils.mv(Dir.glob(File.join(File.dirname(__FILE__), '*.gem')), pkg_dir)
-end
+desc "Generate API documentation to doc/rdocs/index.html"
+RDoc::Task.new do |rd|
+  rd.rdoc_dir = 'doc/rdocs'
+  rd.main = 'README.rdoc'
+  rd.rdoc_files.include 'README.rdoc', 'lib/**/*.rb'
 
-desc 'Install the right_scraper library as a gem'
-task :install => [:gem] do
-   file = Dir["pkg/*.gem"].last
-   sh "gem install #{file}"
+  rd.options << '--all'
+  rd.options << '--diagram'
 end
 
-desc 'Uninstalls and reinstalls the right_scraper library as a gem'
-task :reinstall do
-   sh "gem uninstall right_scraper"
-   sh "rake install"
+# == Emacs integration == #
+desc "Rebuild TAGS file"
+task :tags do
+  sh "rtags -R */{lib,spec}"
 end
-

data/lib/right_scraper/builders/base.rb

@@ -0,0 +1,64 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'logger'))
+
+module RightScraper
+  module Builders
+    # Base class for building additional metadata from filesystem
+    # based checkouts.  Subclasses should override #go, and possibly
+    # #new if they require additional arguments.
+    #
+    # The lifecycle for a builder is as follows:
+    # - builder = Builder.new (once)
+    # - builder.go(dir, resource) (many times)
+    # - builder.finish (once)
+    class Builder
+      # Create a new Builder.  Recognizes options as given.  Some
+      # options may be required, others optional.  This class recognizes
+      # only :logger.
+      #
+      # === Options
+      # <tt>:logger</tt>:: Optional.  Logger currently being used
+      #
+      # === Parameters
+      # options(Hash):: builder options
+      def initialize(options={})
+        @logger = options.fetch(:logger, Logger.new)
+      end
+
+      # Run builder for this resource.
+      #
+      # === Parameters
+      # dir(String):: directory resource exists at
+      # resource(Object):: resource instance being built
+      def go(dir, resource)
+      end
+
+      # Notification that all scans for this repository have
+      # completed.
+      def finish
+      end
+    end
+  end
+end

data/lib/right_scraper/builders/filesystem.rb

@@ -0,0 +1,96 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require File.expand_path(File.join(File.dirname(__FILE__), 'base'))
+
+module RightScraper
+  module Builders
+    # Build metadata by scanning the filesystem.
+    class Filesystem < Builder
+
+      # Create a new filesystem scanner.  In addition to the options
+      # recognized by Builder, this class recognizes <tt>:retriever</tt> and
+      # <tt>:scanner</tt>.
+      #
+      # === Options
+      # <tt>:scanner</tt>:: Required.  Scanner currently being used
+      # <tt>:ignorable_paths</tt>:: Ignore directories whose name belong to this list
+      #
+      # === Parameters
+      # options(Hash):: scraper options
+      def initialize(options={})
+        super
+        @scanner = options.fetch(:scanner)
+        @ignorable_paths = options[:ignorable_paths]
+      end
+
+      # Tell the scanner we're done.
+      def finish
+        super
+        @scanner.finish
+      end
+
+      # Run builder for this resource.
+      #
+      # === Parameters
+      # dir(String):: directory resource exists at
+      # resource(Object):: resource instance being built
+      def go(dir, resource)
+        @logger.operation(:scanning_filesystem, "rooted at #{dir}") do
+          @scanner.begin(resource)
+          maybe_scan(Dir.new(dir), nil)
+          @scanner.end(resource)
+        end
+      end
+
+      def maybe_scan(directory, position)
+        if @scanner.notice_dir(position)
+          scan(directory, position)
+        end
+      end
+
+      # Scan the contents of directory.
+      #
+      # === Parameters
+      # directory(Dir):: directory to scan
+      # position(String):: relative pathname for _directory_ from root of resource
+      def scan(directory, position)
+        directory.each do |entry|
+          next if entry == '.' || entry == '..'
+          next if @ignorable_paths && @ignorable_paths.include?(entry)
+
+          fullpath = File.join(directory.path, entry)
+          relative_position = position ? File.join(position, entry) : entry
+
+          if File.directory?(fullpath)
+            maybe_scan(Dir.new(fullpath), relative_position)
+          else
+            @scanner.notice(relative_position) do
+              open(fullpath) {|f| f.read}
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/right_scraper/builders/union.rb

@@ -0,0 +1,57 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+module RightScraper
+  module Builders
+    # Union builder, to permit running multiple builders in sequence
+    # with the same interface as running one.
+    class Union
+      # (Array) subcomponents of this union
+      attr_reader :subbuilders
+
+      # Create a new union builder.  Recognizes no new options.
+      #
+      # === Parameters
+      # classes(List):: List of Builder classes to run
+      # options(Hash):: options to initialize each Builder with
+      def initialize(classes, options={})
+        @subbuilders = classes.map {|klass| klass.new(options)}
+      end
+
+      # Run each builder for this resource.
+      #
+      # === Parameters
+      # dir(String):: directory resource exists at
+      # resource(RightScraper::Resources::Base):: resource instance being built
+      def go(dir, resource)
+        @subbuilders.each {|builder| builder.go(dir, resource)}
+      end
+
+      # Notify subbuilders that all scans for this repository have
+      # completed.
+      def finish
+        @subbuilders.each {|builder| builder.finish}
+      end
+    end
+  end
+end

data/lib/right_scraper/logger.rb

@@ -0,0 +1,102 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'logger'
+
+module RightScraper
+  # Very simplistic logger for scraper operations.
+  class Logger < ::Logger
+    # If no arguments, create a Logger that logs to nothing.
+    # Otherwise pass the arguments on to ::Logger.
+    def initialize(*args)
+      if args.empty?
+        super('/dev/null')
+        self.level = ::Logger::ERROR
+      else
+        super
+      end
+      @exceptional = false
+    end
+
+    # (RightScraper::Repositories::Base) Repository currently being examined.
+    attr_writer :repository
+
+    # Begin an operation that merits logging.  Will call #note_error
+    # if an exception is raised during the operation.
+    #
+    # === Parameters
+    # type(Symbol):: operation type identifier
+    # explanation(String):: optional explanation
+    def operation(type, explanation="")
+      begin
+        note_phase(:begin, type, explanation)
+        result = yield
+        note_phase(:commit, type, explanation)
+        result
+      rescue
+        note_phase(:abort, type, explanation, $!)
+        raise
+      end
+    end
+
+    # Note an event to the log.  In this base class this calls
+    # note_error when an error occurs, but subclasses will presumably
+    # want to override it.
+    #
+    # === Parameters
+    # phase(Symbol):: phase of operation; one of :begin, :commit, :abort
+    # type(Symbol):: operation type identifier
+    # explanation(String):: explanation of operation
+    # exception(Exception):: optional exception (only if +phase+ is :abort)
+    def note_phase(phase, type, explanation, exception=nil)
+      case phase
+      when :begin then @exceptional = false
+      when :abort then
+        unless @exceptional
+          note_error(exception, type, explanation)
+          @exceptional = true
+        end
+      end
+    end
+
+    # Log an error, with the given exception and explanation of what
+    # was going on.
+    #
+    # === Parameters
+    # exception(Exception):: exception raised
+    # type(Symbol):: operation type identifier
+    # explanation(String):: optional explanation
+    def note_error(exception, type, explanation="")
+      error("Saw #{exception} during #{type}#{maybe_explain(explanation)}")
+    end
+
+    protected
+    def maybe_explain(explanation)
+      if explanation.nil? || explanation.empty?
+        ""
+      else
+        ": #{explanation}"
+      end
+    end
+  end
+end

data/lib/right_scraper/loggers/noisy.rb

@@ -0,0 +1,85 @@
+#--
+# Copyright: Copyright (c) 2010 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'logger'))
+
+module RightScraper
+  module Loggers
+    # A very noisy logger, useful for debugging.
+    class NoisyLogger < Logger
+      # Initialize the logger, setting the current operation depth to 1.
+      def initialize(*args)
+        super
+        @pending = []
+      end
+
+      # Note an event to the log, including a visual indicator of how
+      # many nested operations are currently pending.
+      #
+      # === Parameters
+      # phase(Symbol):: phase of operation; one of :begin, :commit, :abort
+      # type(Symbol):: operation type identifier
+      # explanation(String):: explanation of operation
+      # exception(Exception):: optional exception (only if +phase+ is :abort)
+      def note_phase(phase, type, explanation, exception=nil)
+        if phase == :begin
+          @pending.push [type, explanation]
+        end
+        super
+        debug("#{depth_str} #{phase} #{immediate_context}")
+        unless phase == :begin
+          @pending.pop
+        end
+      end
+
+      def note_error(exception, type, explanation="")
+        recordedtype, recordedexplanation = @pending[-1]
+        if recordedtype != type || recordedexplanation != explanation
+          @pending.push [type, explanation]
+        end
+        error("Saw #{exception} during #{context}")
+        if recordedtype != type || recordedexplanation != explanation
+          @pending.pop
+        end
+      end
+
+      private
+      def depth_str
+        '>' * @pending.size
+      end
+
+      def context
+        @pending.reverse.map {|pair| contextify(pair)}.join(" in ")
+      end
+
+      def immediate_context
+        contextify(@pending[-1])
+      end
+
+      def contextify(pair)
+        type, explanation = pair
+        "#{type}#{maybe_explain(explanation)}"
+      end
+    end
+  end
+end