imw 0.2.18 → 0.3.0

data/lib/imw/runner.rb

@@ -1,118 +0,0 @@
-require 'imw'
-require 'optparse'
-
-module IMW
-
-  RunnerError = Class.new(IMW::Error)
-  
-  class Runner
-
-    DEFAULT_OPTIONS = {
-      :requires  => [],
-      :selectors => [],
-      :dry_run   => false
-    }
-
-    attr_reader :args, :options
-
-    def initialize *args
-      @args    = args
-      @options = DEFAULT_OPTIONS.dup
-      parser.parse!(args)
-    end
-
-    def parser
-      OptionParser.new do |opts|
-        opts.banner = "usage: imw [OPTIONS] TASK"
-        opts.separator <<EOF
-
-  Run TASK for all datasets in the repository.  IMW will read any
-  *.imw files in the current directory by default.
-
-  Options include
-
-EOF
-
-        opts.on('-v', '--verbose', "Print verbose output") do
-          IMW.verbose = true      # class level, see IMW::Runner.verbose?
-        end
-
-        opts.on('-d', '--skip-dependencies', "Execute given tasks without invoking dependencies first") do
-          options[:execute] = true
-        end
-
-        opts.on('-l', '--list', "List datasets in repository") do
-          options[:list] = true
-        end
-
-        opts.on('-s', '--selector SELECTOR', "Filter datasets by regexp SELECTOR.  Can be given more than once.") do |selector|
-          options[:selectors] << selector
-        end
-        
-        opts.on('-r', '--require PATH', "Require PATH.  Can be given more than once.") do |path|
-          options[:requires] << path
-        end
-
-      end
-    end
-
-    def require_files
-      Dir['*.imw'].each { |path| load File.expand_path(path) }
-      Dir['*.rb'].each { |path| require path.gsub(/\.rb$/,'') }
-      options[:requires].each do |path|
-        IMW.open(path) do |requireable|
-          if requireable.directory?
-            requireable["**/*.rb"].each  { |file| require file }
-            requireable["**/*.imw"].each { |file| load    file }
-          else
-            require requireable.path
-          end
-        end
-      end
-    end
-
-    def task
-      args.first
-    end
-
-    def handles
-      if options[:selectors].blank?
-        IMW.repository.keys.sort
-      else
-        IMW.repository.handles.map do |handle|
-          handle if options[:selectors].all? { |selector| handle.to_s =~ Regexp.new(selector) }
-        end.compact.sort
-      end
-    end
-
-    def datasets
-      handles.map { |handle| IMW.repository[handle] }
-    end
-
-    def list!
-      puts handles
-      exit
-    end
-
-    def run_task!
-      datasets.each do |dataset|
-        dataset[task].send(options[:execute] ? :execute : :invoke)
-      end
-      exit
-    end
-      
-    def run!
-      require_files
-      case
-      when options[:list]
-        list!
-      when task.blank?
-        puts parser
-        exit 1
-      else
-        run_task!
-      end
-    end
-  end
-end
-

data/lib/imw/schemes.rb

@@ -1,23 +0,0 @@
-module IMW
-  module Schemes
-    autoload :Local,  'imw/schemes/local'
-    autoload :Remote, 'imw/schemes/remote'
-    autoload :S3,     'imw/schemes/s3'
-    autoload :HTTP,   'imw/schemes/http'
-    autoload :HTTPS,  'imw/schemes/http'
-    autoload :HDFS,   'imw/schemes/hdfs'
-    autoload :SQL,    'imw/schemes/sql'
-    
-    HANDLERS = [
-                ["Schemes::Local::Base",  Proc.new { |resource| resource.scheme == 'file' || resource.scheme.blank?    } ],
-                ["Schemes::Remote::Base", Proc.new { |resource| resource.scheme != 'file' && resource.scheme.present?  } ],
-                ["Schemes::S3",        %r{^s3://}i    ],
-                ["Schemes::HTTP",      %r{^http://}i  ],
-                ["Schemes::HTTPS",     %r{^https://}i ],
-                ["Schemes::HDFS",      %r{^hdfs://}i  ],
-                ["Schemes::SQL::Base", %r{^\w+sql://}i ]
-               ]
-  end
-end
-
-

data/lib/imw/schemes/ftp.rb

@@ -1,142 +0,0 @@
-module IMW
-  module Schemes
-
-    # Defines methods for reading and writing data from an FTP server.
-    #
-    #   IMW.open('ftp://user:pass@my_bucket/path/to/some/file.csv')
-    #
-    # Learn more about {Amazon Web Services}[http://aws.amazon.com].
-    module FTP
-
-      module Base
-
-      # Is this resource an FTP resource?
-      #
-      # @return [true, false] 
-      def on_ftp?
-        true
-      end
-      alias_method :is_ftp?, :on_ftp?
-
-      # Copy this resource to the +new_uri+.
-      #
-      # @param [String, IMW::Resource] new_uri
-      # @return [IMW::Resource] the new resource
-      def cp new_uri
-        local_obj = IMW.open(new_uri)
-        raise IMW::PathError.new("FTP resources (#{self}) can only be downloaded to a local path") unless local_obj.is_local?
-        local_obj.dir.should_exist!
-        FTP.open(host, user, password) do |ftp|
-          ftp.get(path, local_obj.path)
-        end
-        local_obj
-      end
-
-      # Does this resource exist on S3?
-      #
-      # @return [true, false]
-      def exist?
-        s3_object.exists?
-      end
-      alias_method :exists?, :exist?
-
-      # Remove this resource from S3.
-      #
-      # @return [IMW::Resource] the deleted object
-      def rm
-        s3_object.delete
-      end
-      alias_method :rm!, :rm
-
-      # Return the S3N URL for this S3 object
-      #
-      #   resource = IMW.open('s3://my_bucket/path/to/some/obj')
-      #   resource.s3n_url
-      #   => 's3n://my_bucket/path/to/some/obj'
-      #
-      # @return [String]
-      def s3n_url
-        uri.to_s.gsub(/^s3:/, 's3n:')
-      end
-
-      # Return the contents of this S3 object.
-      #
-      # @return [String]
-      def read
-        s3_object.value
-      end
-
-      # Store +source+ into +destination+.
-      # 
-      # @param [String, IMW::Resource, #io] source
-      # @param [String, IMW::Resource, #path, #bucket] destination
-      # @return [IMW::Resource] the new S3 object
-      def self.put source, destination
-        source       = IMW.open(source)
-        destintation = IMW.open(destination)
-        raise IMW::ArgumentError.new("destination must be on S3 -- #{destination} given") unless destination.on_s3?
-        make_connection!
-        AWS::S3::S3Object.store(destination.path, source.io, destination.bucket)
-        destination
-      end
-
-      # Download +source+ from S3 into +destination+.
-      #
-      # @param [String, IMW::Resource, #path, #bucket] source
-      # @param [String, IMW::Resource, #write] destination
-      # @return [IMW::Resource] the new resource
-      def self.get source, destination
-        source      = IMW.open(source)
-        destination = IMW.open!(destination)
-        raise IMW::ArgumentError.new("source must be on S3 -- #{source} given") unless source.on_s3?
-        make_connection!
-        AWS::S3::S3Object.stream(source.path, source.bucket) do |chunk|
-          destination.write(chunk)
-        end
-        destination.close
-        destination.reopen
-      end
-
-      # Copy S3 resource +source+ to +destination+.
-      #
-      # @param [String, IMW::Resource, #path, #bucket] source
-      # @param [String, IMW::Resource, #path, #bucket] destination
-      # @return [IMW::Resource] the new resource
-      def self.copy source, destination
-        source      = IMW.open(source)
-        destination = IMW.open(destination)
-        raise IMW::PathError.new("Bucket names must be non-blank and match to 'copy'") unless source.bucket.present? && destination.bucket.present? && source.bucket == destination.bucket
-        make_connection!
-        AWS::S3::Object.copy(source.path, destination.path, destination.bucket)
-        destination
-      end
-
-      # Return the resource at the base path of this resource joined
-      # to +path+.
-      #
-      #   IMW.open('s3:://bucket/path/to/dir').join('subdir')
-      #   #=> IMW::Resource at 's3://bucket/path/to/dir/subdir'
-      #
-      # @param [Array<String>] paths
-      # @return [IMW::Resource]
-      def join *paths
-        IMW.open(File.join(stripped_uri.to_s, *paths))
-      end
-
-      protected
-      # Make an S3 connection.
-      #
-      # Uses settings defined in IMW::AWS_CREDENTIALS.
-      #
-      # @return [AWS
-      def self.make_connection!
-        return @connection if @connection
-        raise IMW::Error.new("Must define a constant IMW::AWS_CREDENTIALS with an :access_key_id and a :secret_access_key before using S3 resources") unless defined?(IMW::AWS_CREDENTIALS)
-        require 'aws/s3'
-        @connection = AWS::S3::Base.establish_connection!(IMW::AWS_CREDENTIALS)
-      end
-      
-    end
-  end
-end
-

data/lib/imw/schemes/hdfs.rb

@@ -1,251 +0,0 @@
-module IMW
-  module Schemes
-
-    # Defines methods for reading and writing data to/from an
-    # HDFS[http://hadoop.apache.org/common/docs/current/hdfs_design.html]]
-    #
-    # Learn more about Hadoop[http://hadoop.apache.org] and the
-    # {Hadoop Distributed
-    # Filesystem}[http://hadoop.apache.org/common/docs/current/hdfs_design.html].
-    module HDFS
-
-      # Checks to see if this is a file or directory
-      def self.extended obj
-        obj.extend(obj.is_directory? ? HDFSDirectory : HDFSFile)
-      end
-
-      # Is this resource an HDFS resource?
-      #
-      # @return [true, false]
-      def on_hdfs?
-        true
-      end
-      alias_method :is_hdfs?, :on_hdfs?
-
-      # Copy this resource to the +new_uri+.
-      #
-      # @param [String, IMW::Resource] new_uri
-      # @return [IMW::Resource] the new resource
-      def cp new_uri
-        IMW::Tools::Transferer.new(:cp, self, new_uri).transfer!
-      end
-
-      # Move this resource to the +new_uri+.
-      #
-      # @param [String, IMW::Resource] new_uri
-      # @return [IMW::Resource] the new resource
-      def mv new_uri
-        IMW::Tools::Transferer.new(:mv, self, new_uri).transfer!
-      end
-
-      # Delete this resource from the HDFS.
-      #
-      # @option options [true,false] :skip_trash
-      def rm options={}
-        should_exist!("Cannot delete.")
-        args = [:rm]
-        args << '-skipTrash' if options[:skip] || options[:skip_trash] || options[:skipTrash]
-        args << path
-        HDFS.fs(*args)
-        self
-      end
-      alias_method :rm!, :rm
-      
-      
-      # Does this path exist on the HDFS?
-      #
-      # @return [true, false]
-      def exist?
-        return @exist unless @exist.nil?
-        refresh!
-        @exist
-      end
-      alias_method :exists?, :exist?
-
-
-      # Return the size (in bytes) of this resource on the HDFS.
-      #
-      # This value is cached.  Call +refresh+ to refresh the cache
-      # manually.
-      #
-      # @return [Fixnum]
-      def size
-        return @size unless @size.nil?
-        refresh!
-        should_exist!("Cannot report size")
-        @size
-      end
-
-      # Return the number of directories contained at or below this
-      # path on the HDFS.
-      #
-      # This value is cached.  Call +refresh+ to refresh the cache
-      # manually.
-      #
-      # @return [Fixnum]
-      def num_dirs
-        return @num_dirs unless @num_dirs.nil?
-        refresh!
-        should_exist!("Cannot report number of directories.")
-        @num_dirs
-      end
-
-      # Return the number of files contained at or below this path
-      # on the HDFS.
-      #
-      # This value is cached.  Call +refresh+ to refresh the cache
-      # manually.
-      #
-      # @return [Fixnum]
-      def num_files
-        return @num_files unless @num_files.nil?
-        refresh!
-        should_exist!("Cannot report number of files.")
-        @num_files
-      end
-
-      # Is this resource an HDFS directory?
-      #
-      # @return [true, false]
-      def is_directory?
-        exist? && num_dirs > 0
-      end
-
-      # Refresh the cached file properties.
-      #
-      # @return [IMW::Resource] this resource
-      def refresh!
-        response = HDFS.fs(:count, path)
-        if response.blank? || response =~ /^Can not find listing for/
-          @exist = false
-          @num_dirs, @num_files, @size, @hdfs_path = false, false, false, false
-        else
-          @exist = true
-          parts = response.split
-          @num_dirs, @num_files, @size = parts[0..2].map(&:to_i)
-          @hdfs_path = parts.last
-        end
-        self
-      end
-
-      # Execute +command+ with +args+ on the Hadoop Distributed
-      # Filesystem (HDFS).
-      #
-      # If passed a block, yield each line of the output from the
-      # command, else just return the output.
-      #
-      # Try running `hadoop fs -help' for more information.
-      #
-      # @param [String, Symbol] command the command to run.
-      # @param [String, Symbol] args the arguments to pass the command
-      # @yield [String] each line of the command's output
-      # @return [String] the command's output
-      def self.fs command, *args
-        command_string = "#{executable} fs -#{command} #{args.compact.map(&:to_str).join(' ')}"
-        command_string += " 2>&1" if command == :count # FIXME or else it just spams the screen when we do HDFS#refresh!
-        output = `#{command_string}`.chomp
-        if block_given?
-          output.split("\n").each do |line|
-            yield line
-          end
-        else
-          output
-        end
-      end
-
-      protected
-      # Returns the path to the Hadoop executable.
-      #
-      # @return [String]
-      def self.executable
-        @executable ||= begin
-                          string = `which hadoop`.chomp
-                          raise IMW::Error.new("Could not find hadoop command.  Is Hadoop installed?") if string.blank?
-                          string
-                        end
-      end
-    end
-
-    # Defines methods for reading data from HDFS files.
-    module HDFSFile
-
-      # Return the contents of this HDFS file as a string.
-      #
-      # Be VERY careful how you use this!
-      #
-      # @return [String]
-      def read
-        HDFS.fs(:cat, path)
-      end
-
-      # Iterate through each line of this HDFS resource.
-      #
-      # @yield [String] each line of the file
-      def each &block
-        HDFS.fs(:cat, path, &block)
-      end
-
-      # Return a handle on a StringIO object representing the
-      # content in this HDFS file.
-      #
-      # Be VERY careful how you use this!  It is a StringIO object
-      # so the whole HDFS file is read into a string before
-      # returning the handle.
-      #
-      # @return [StringIO]
-      def io
-        @io ||= StringIO.new(read)
-      end
-
-      # Map over the lines of this HDFS resource.
-      #
-      # @yield [String] each line of the file
-      # @return [Array] the result of the block on each line
-      def map &block
-        [].tap do |output|
-          HDFS.fs(:cat, path) do |line|
-            output << block.call(line)
-          end
-        end
-      end
-
-    end
-
-    # Defines methods for listing contents of HDFS directories.
-    module HDFSDirectory
-
-      # Return the paths of all files and directories directly below
-      # this directory on the HDFS.
-      #
-      # @return [Array<String>]
-      def contents
-        [].tap do |paths|
-          HDFS.fs(:ls, path) do |line|
-            next if line =~ /^Found.*items$/
-            paths << line.split.last
-          end
-        end
-      end
-
-      # Return the resources directly below this directory on the
-      # HDFS.
-      #
-      # @return [Array<IMW::Resource>]
-      def resources
-        contents.map { |path| IMW.open(path) }
-      end
-
-      # Return the resource at the base path of this resource joined
-      # to +path+.
-      #
-      #   IMW.open('hdfs:///path/to/dir').join('subdir')
-      #   #=> IMW::Resource at 'hdfs:///path/to/dir/subdir'
-      #
-      # @param [Array<String>] paths
-      # @return [IMW::Resource]
-      def join *paths
-        IMW.open(File.join(stripped_uri.to_s, *paths))
-      end
-    end
-  end
-end