right_scraper 1.0.26 → 3.0.0

data/lib/right_scraper/scrapers/base.rb

@@ -0,0 +1,262 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+module RightScraper
+  module Scrapers
+
+    # Base class for all scrapers. Subclasses should override 
+    # #find_next which instantiates the resource from the file system.
+    class Base
+      
+      # Scraped resources
+      attr_reader :resources
+ 
+      # Initialize scraper
+      #
+      # === Options
+      # <tt>:kind</tt>:: Scraper type, one of :cookbook or :workflow
+      # <tt>:repo_dir</tt>:: Required, path to directory containing files
+      #   to be scraped
+      # <tt>:ignorable_paths</tt>:: List of directory names that should
+      #   be ignored by scraper
+      # <tt>:scanners</tt>:: List of Scanner classes to use, optional
+      # <tt>:builders</tt>:: List of Builder classes to use, optional
+      #
+      # === Return
+      # scraper(Scrapers::Base):: Corresponding scraper instance
+      def self.scraper(options)
+        scraper_kind = options.delete(:kind)
+        scraper_class = @@types[scraper_kind]
+        raise "Can't understand how to build scraper #{scraper_kind}" if scraper_class.nil?
+        scraper = scraper_class.new(options)
+      end
+
+      # Do the scrape!
+      # Extract all resources from directory
+      # Call this method or call 'next_resource' to retrieve
+      # resources one by one (you must then call 'close' yourself)
+      # Fill @resources
+      #
+      # === Return
+      # resources<Array>:: List of all scraped resources
+      def scrape
+        @resources = []
+        begin
+          resource = next_resource
+          until resource.nil?
+            @resources << resource
+            resource = next_resource
+          end
+        ensure
+          close
+        end
+        @resources
+      end
+
+      # Return the next resource in the filesystem, or nil if none.  As
+      # a part of building the resources, invokes the builders.
+      # A resource can be a cookbook, a workflow, a RightScript etc.
+      #
+      # === Returns
+      # Object:: next resource in filesystem, or nil if none.
+      def next_resource
+        @logger.operation(:next) do
+          next nil if @next.nil?
+
+          value = @next
+          @next = search_dirs
+          while @next.nil? && !@queue.empty?
+            pop_queue
+          end
+          value
+        end
+      end
+
+      # Close any opened file descriptor
+      #
+      # === Return
+      # true:: Always return true
+      def close
+        @builder.finish
+        if @stack && !@stack.empty?
+          @stack.each {|s| s.close}
+          @stack = []
+        end
+        true
+      end
+
+      protected
+
+      # Directory containing files to be scraped
+      attr_reader :repo_dir
+
+      # (Hash) Lookup table from textual description of scraper type
+      # ('cookbook' or 'workflow' currently) to the class that
+      # represents that scraper.
+      @@types = {} unless class_variable_defined?(:@@types)
+
+      # Initialize scraper
+      #
+      # === Options
+      # <tt>:repository</tt>:: Required, original repository containing scraped
+      #   files
+       # <tt>:repo_dir</tt>:: Required, path to directory containing files
+      #   to be scraped
+      # <tt>:ignorable_paths</tt>:: List of directory names that should
+      #   be ignored by scraper
+      # <tt>:scanners</tt>:: List of Scanner classes to use, defaulting
+      #   to RightScraper::Scanners::ResourceManifest and
+      #   RightScraper::Scanners::CookbookMetadata
+      # <tt>:builders</tt>:: List of Builder classes to use, defaulting to
+      #   RightScaper::Builders::Filesystem
+      #
+      def initialize(options)
+        raise "Repository required when initializing a scraper" unless options[:repository]
+        raise "Repository directory required when initializing a scraper" unless options[:repo_dir]
+        @repository = options[:repository]
+        @logger = options[:logger] || ScraperLogger.new
+        @repo_dir = options[:repo_dir]
+        @ignorable_paths = options[:ignorable_paths]
+        @stack = []
+        @queue = (@repository.resources_path || [""]).reverse
+        @resources = []
+        scanners = options[:scanners] || default_scanners 
+        @scanner = RightScraper::Scanners::Union.new(scanners, options)
+        builders = options[:builders] || default_builders
+        @builder = RightScraper::Builders::Union.new(builders, :ignorable_paths => @ignorable_paths,
+                                                               :scanner         => @scanner,
+                                                               :logger          => @logger,
+                                                               :max_bytes       => @max_bytes,
+                                                               :max_seconds     => @max_seconds)
+        pop_queue # Initialize @next
+      end
+
+      # List of default scanners for this scaper
+      #
+      # === Return
+      # Array<Scanner>:: Default scanners
+      def default_scanners
+      end
+
+      # List of default builders for this scaper
+      #
+      # === Return
+      # Array<Builder>:: Default builders
+      def default_brokers
+      end
+
+      # Find the interesting item in given directory
+      # Override in actual scraper implementation
+      #
+      # === Parameters
+      # dir(Dir):: directory to begin search in
+      def find_next(dir)
+        raise NotImplementedError
+      end
+
+      # Return the position of the scraper.  Here, the position is the
+      # path relative from the top of the temporary directory.  Akin to
+      # IO#pos or IO#tell.
+      def pos
+        strip_repo_dir(@stack.last.path)
+      end
+      alias_method :tell, :pos
+
+      # Turn path from an absolute filesystem location to a relative
+      # file location from #repo_dir.
+      #
+      # === Parameters
+      # path(String):: absolute path to relativize
+      #
+      # === Returns
+      # res(String):: relative pathname for path
+      def strip_repo_dir(path)
+        res = path[repo_dir.length+1..-1]
+        if res == nil || res == ""
+          "."
+        else
+          res
+        end
+      end
+      private :strip_repo_dir
+
+      # Test if the entry given is ignorable.  By default just uses
+      # #ignorable_paths
+      #
+      # === Parameters
+      # entry(String):: file name to check
+      #
+      # === Returns
+      # Boolean:: true if the entry should be ignored
+      def ignorable?(entry)
+        @ignorable_paths.include?(entry)
+      end
+
+      # Initialize @next with the next resource
+      #
+      # === Returns
+      # @next(Resources::Base):: Next resource
+      def pop_queue
+        until @queue.empty?
+          nextdir = @queue.pop
+          if File.directory?(File.join(repo_dir, nextdir))
+            @next = find_next(Dir.new(File.join(repo_dir, nextdir)))
+            return @next
+          else
+            @logger.warn("When processing in #{@repository}, no such path #{nextdir}")
+          end
+        end
+        @next = nil
+      end
+
+      # Search the directory stack looking for the next resource.
+      def search_dirs
+        @logger.operation(:searching) do
+          until @stack.empty?
+            dir = @stack.last
+            entry = dir.read
+            if entry == nil
+              dir.close
+              @stack.pop
+              next
+            end
+
+            next if entry == '.' || entry == '..'
+            next if ignorable?(entry)
+
+            fullpath = File.join(dir.path, entry)
+
+            if File.directory?(fullpath)
+              result = find_next(Dir.new(fullpath))
+              break
+            end
+          end
+          result
+        end
+      end
+      private :search_dirs
+
+    end
+  end
+end
+

data/lib/right_scraper/scrapers/cookbook.rb

@@ -0,0 +1,73 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require File.expand_path(File.join(File.dirname(__FILE__), 'base'))
+
+module RightScraper
+  module Scrapers
+
+    # Chef cookbook scraper
+    class Cookbook < Base
+
+      # Find the next cookbook, starting in dir.
+      #
+      # === Parameters
+      # dir(Dir):: directory to begin search in
+      def find_next(dir)
+        @logger.operation(:finding_next_cookbook, "in #{dir.path}") do
+          if File.exists?(File.join(dir.path, 'metadata.json'))
+            @logger.operation(:reading_cookbook, "from #{dir.path}") do
+              cookbook = RightScraper::Resources::Cookbook.new(@repository, strip_repo_dir(dir.path))
+              @builder.go(dir.path, cookbook)
+              cookbook
+            end
+          else
+            @stack << dir
+            search_dirs
+          end
+        end
+      end
+
+      # List of default scanners for this scaper
+      #
+      # === Return
+      # Array<Scanner>:: Default scanners
+      def default_scanners
+        [RightScraper::Scanners::CookbookMetadata,
+          RightScraper::Scanners::CookbookManifest]
+      end
+
+      # List of default builders for this scaper
+      #
+      # === Return
+      # Array<Builder>:: Default builders
+      def default_builders
+        [RightScraper::Builders::Filesystem]
+      end
+
+      # Add this scraper to the list of available types.
+      @@types[:cookbook] = RightScraper::Scrapers::Cookbook
+ 
+    end
+  end
+end

data/lib/right_scraper/scrapers/workflow.rb

@@ -0,0 +1,88 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require File.expand_path(File.join(File.dirname(__FILE__), 'base'))
+
+module RightScraper
+  module Scrapers
+
+    # Workflow scraper
+    class Workflow < Base
+
+     # Initialize list of known workflows on top of
+      def initialize(options)
+        @known_workflows = []
+        super(options)
+      end
+
+      # Find the next workflows, starting in dir.
+      #
+      # === Parameters
+      # dir(Dir):: directory to begin search in
+      def find_next(dir)
+        @logger.operation(:finding_next_workflow, "in #{dir.path}") do
+          def_ext = RightScraper::Resources::Workflow::DEFINITION_EXT
+          meta_ext = RightScraper::Resources::Workflow::METADATA_EXT
+          potentials = Dir[File.join(dir.path, "*#{def_ext}")]
+          potentials.reject! { |wdef| !File.exists?(wdef.chomp(File.extname(wdef)) + meta_ext) }
+          potentials.reject! { |wdef| @known_workflows.include?(wdef) }
+          unless potentials.empty?
+            wdef = potentials.first
+            relative_def = strip_repo_dir(wdef)
+            @logger.operation(:reading_workflow, "#{relative_def}") do
+              workflow = RightScraper::Resources::Workflow.new(@repository, relative_def)
+              @builder.go(File.dirname(wdef), workflow)
+              @known_workflows << wdef
+              workflow
+            end
+          else
+            @stack << dir
+            search_dirs
+          end
+        end
+      end
+
+      # List of default scanners for this scaper
+      #
+      # === Return
+      # Array<Scanner>:: Default scanners
+      def default_scanners
+        [RightScraper::Scanners::WorkflowMetadata,
+          RightScraper::Scanners::WorkflowManifest]
+      end
+
+      # List of default builders for this scaper
+      #
+      # === Return
+      # Array<Builder>:: Default builders
+      def default_builders
+        [RightScraper::Builders::Filesystem]
+      end
+ 
+      # Add this scraper to the list of available types.
+      @@types[:workflow] = RightScraper::Scrapers::Workflow
+
+    end
+  end
+end
+

data/lib/right_scraper/svn_client.rb

@@ -0,0 +1,101 @@
+#--
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+require 'process_watcher'
+
+module RightScraper
+  # Simplified interface to the process of creating SVN client
+  # contexts.
+  #
+  # SVN client contexts are needed for almost every nontrivial SVN
+  # operation, and the authentication procedure is truly baroque.
+  # Thus, when you need a client context, do something like this:
+  #   client = SvnClient.new(repository)
+  #   client.with_context do |ctx|
+  #     ...
+  #   end
+  module SvnClient
+    def calculate_version
+      unless @svn_version
+        out = ProcessWatcher.watch("svn", ["--version", "--quiet"],
+                                   repo_dir, @max_bytes || -1, @max_seconds || -1)
+        @svn_version = out.chomp.split(".").map {|e| e.to_i}
+      end
+      @svn_version
+    end
+
+    def svn_arguments
+      version = calculate_version
+      case
+      when version[0] != 1
+        raise "SVN major revision is not 1, cannot be sure it will run properly."
+      when version[1] < 4
+        raise "SVN minor revision < 4; cannot be sure it will run properly."
+      when version[1] < 6
+        # --trust-server-cert is a 1.6ism
+        args = ["--no-auth-cache", "--non-interactive"]
+      else
+        args = ["--no-auth-cache", "--non-interactive", "--trust-server-cert"]
+      end
+      if repository.first_credential && repository.second_credential
+        args << "--username"
+        args << repository.first_credential
+        args << "--password"
+        args << repository.second_credential
+      end
+      args
+    end
+
+    def get_tag_argument
+      if repository.tag
+        tag_cmd = ["-r", get_tag.to_s]
+      else
+        tag_cmd = ["-r", "HEAD"]
+      end
+    end
+
+    def run_svn_no_chdir(*args)
+      ProcessWatcher.watch("svn", [args, svn_arguments].flatten,
+                           repo_dir, @max_bytes || -1, @max_seconds || -1) do |phase, operation, exception|
+        #$stderr.puts "#{phase} #{operation} #{exception}"
+      end
+    end
+
+    def run_svn(*args)
+      Dir.chdir(repo_dir) do
+        run_svn_no_chdir(*args)
+      end
+    end
+
+    # Fetch the tag from the repository, or nil if one doesn't
+    # exist.  This is a separate method because the repo tag should
+    # be a number but is a string in the database.
+    def get_tag
+      case repository.tag
+      when Fixnum then repository.tag
+      when /^\d+$/ then repository.tag.to_i
+      else
+        repository.tag
+      end
+    end
+  end
+end

data/lib/right_scraper/version.rb

@@ -0,0 +1,28 @@
+#--  -*- mode: ruby; encoding: utf-8 -*-
+# Copyright: Copyright (c) 2011 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+module RightScraper
+  # (Fixnum) protocol versioning scheme; prepended to hashes to
+  # prevent collisions.
+  PROTOCOL_VERSION = 1
+end

data/lib/right_scraper.rb

@@ -1,18 +1,18 @@
 #--
-# Copyright: Copyright (c) 2010 RightScale, Inc.
+# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
 # 'Software'), to deal in the Software without restriction, including
 # without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to 
+# distribute, sublicense, and/or sell copies of the Software, and to
 # permit persons to whom the Software is furnished to do so, subject to
 # the following conditions:
 #
 # The above copyright notice and this permission notice shall be
 # included in all copies or substantial portions of the Software.
 #
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
 # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
@@ -21,12 +21,36 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
-# Explicitely list required files to make IDEs happy
+# Explicitly list required files to make IDEs happy
 require 'fileutils'
-require File.join(File.dirname(__FILE__), 'right_scraper', 'scraper_base')
-require File.join(File.dirname(__FILE__), 'right_scraper', 'scrapers', 'download_scraper')
-require File.join(File.dirname(__FILE__), 'right_scraper', 'scrapers', 'git_scraper')
-require File.join(File.dirname(__FILE__), 'right_scraper', 'scrapers', 'svn_scraper')
-require File.join(File.dirname(__FILE__), 'right_scraper', 'repository')
-require File.join(File.dirname(__FILE__), 'right_scraper', 'scraper')
-require File.join(File.dirname(__FILE__), 'right_scraper', 'watcher')
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'builders', 'base'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'builders', 'filesystem'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'builders', 'union'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'logger'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'processes', 'ssh'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'repositories', 'base'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'repositories', 'download'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'repositories', 'git'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'repositories', 'svn'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'resources', 'base'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'resources', 'cookbook'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'resources', 'workflow'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'retrievers', 'base'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'retrievers', 'checkout'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'retrievers', 'download'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'retrievers', 'git'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'retrievers', 'svn'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scanners', 'base'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scanners', 'cookbook_manifest'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scanners', 'cookbook_metadata'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scanners', 'cookbook_s3_upload'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scanners', 'union'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scanners', 'workflow_manifest'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scanners', 'workflow_metadata'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scraper'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scraper_logger'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scrapers', 'base'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scrapers', 'cookbook'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'scrapers', 'workflow'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'svn_client'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_scraper', 'version'))