imw 0.2.18 → 0.3.0

data/lib/imw/schemes/remote.rb

@@ -1,119 +0,0 @@
-module IMW
-  module Schemes
-
-    # Contains modules which define methods appropriate for remote
-    # resources, no matter the protocol.
-    module Remote
-
-      # Defines methods appropriate for accessing a remote resource,
-      # no matter the protocol.
-      module Base
-        
-        #
-        # TODO -- self.extended should extend by RemoteDirectory when appropriate
-        #
-        
-        def self.extended obj
-          obj.extend(RemoteFile)
-        end
-
-        # Is this resource on a remote host?
-        #
-        # @return [true,false]
-        def is_remote?
-          true
-        end
-
-        # The host of this resource.
-        #
-        # @return [String]
-        def host
-          @host ||= uri.host
-        end
-
-        # Return the query string part of this resource's URI.  Will
-        # likely be +nil+ for local resources.
-        #
-        # @return [String]
-        def query_string
-          @query_string ||= uri.query
-        end
-
-        # Return the path part of this resource's URI.  Will _not_
-        # include the +query_string+ or +fragment+.
-        #
-        # @return [String]
-        def path
-          @path ||= uri.path
-        end
-
-      end
-      
-      module RemoteFile
-
-        # Return the IO object for this remote file.
-        #
-        # The mode of this resource is ignored.
-        #
-        # @return [StringIO]
-        def io
-          require 'open-uri'
-          @io ||= open(uri.to_s)              # ignore mode
-        end
-
-        # Read the contents of this remote file.
-        #
-        # @return [String]
-        def read
-          io.read
-        end
-
-        # Return the lines of this remote file.
-        #
-        # If passed a block then yield each line to the block.
-        #
-        # @return [Array] the lines of this remote file
-        # @yield [String] each line of this remote file
-        def load &block
-          if block_given?
-            io.each do |line|
-              yield line
-            end
-          else
-            read.split("\n")
-          end
-        end
-
-        # Map over the lines in this remote file.
-        #
-        # @yield [String] each line of the file
-        def map &block
-          io.map(&block)
-        end
-      end
-
-
-      module RemoteDirectory
-
-        # Return the resource at the base path of this resource joined
-        # to +path+.
-        #
-        #   IMW.open('http://example.com/path/to/dir').join('subdir')
-        #   #=> IMW::Resource at 'http://example.com/path/to/dir/subdir'
-        #
-        # @param [Array<String>] paths
-        # @return [IMW::Resource]
-        def join *paths
-          IMW.open(File.join(stripped_uri.to_s, *paths))
-        end
-
-        #
-        # TODO -- bloody everything.  what's the best way to tell if
-        # the remote URL is a directory?
-        #
-
-        
-      end
-    end
-  end
-end

data/lib/imw/schemes/s3.rb

@@ -1,143 +0,0 @@
-module IMW
-  module Schemes
-
-    # Defines methods for reading and writing data to {Amazon
-    # S3}[http://aws.amazon.com/s3] buckets.
-    #
-    #   IMW.open('s3://my_bucket/path/to/some/file.csv')
-    #
-    # Learn more about {Amazon Web Services}[http://aws.amazon.com].
-    module S3
-
-      # For an S3 resource, the bucket is just the hostname.
-      #
-      # @return [String]
-      def bucket
-        host
-      end
-
-      # Is this resource an S3 resource?
-      #
-      # @return [true, false] 
-      def on_s3?
-        true
-      end
-      alias_method :is_s3?, :on_s3?
-
-      # Copy this resource to the +new_uri+.
-      #
-      # @param [String, IMW::Resource] new_uri
-      # @return [IMW::Resource] the new resource
-      def cp new_uri
-        #IMW::Tools::Transferer.new(:cp, self, new_uri).transfer!
-        IMW::Schemes::S3.get(self, new_uri)
-      end
-
-      # Does this resource exist on S3?
-      #
-      # @return [true, false]
-      def exist?
-        AWS::S3::S3Object.exists?(raw_path, bucket)
-      end
-      alias_method :exists?, :exist?
-
-      # Remove this resource from S3.
-      #
-      # @return [IMW::Resource] the deleted object
-      def rm
-        AWS::S3::S3Object.delete(raw_path, bucket)
-      end
-      alias_method :rm!, :rm
-
-      # Return the S3N URL for this S3 object
-      #
-      #   resource = IMW.open('s3://my_bucket/path/to/some/obj')
-      #   resource.s3n_url
-      #   => 's3n://my_bucket/path/to/some/obj'
-      #
-      # @return [String]
-      def s3n_url
-        uri.to_s.gsub(/^s3:/, 's3n:')
-      end
-
-      # Return the contents of this S3 object.
-      #
-      # @return [String]
-      def read
-        AWS::S3::S3Object.value(raw_path, bucket)
-      end
-
-      # Store +source+ into +destination+.
-      # 
-      # @param [String, IMW::Resource, #io] source
-      # @param [String, IMW::Resource, #path, #bucket] destination
-      # @return [IMW::Resource] the new S3 object
-      def self.put source, destination
-        source       = IMW.open(source)
-        destintation = IMW.open(destination)
-        raise IMW::ArgumentError.new("destination must be on S3 -- #{destination} given") unless destination.on_s3?
-        make_connection!
-        AWS::S3::S3Object.store(destination.raw_path, source.io, destination.bucket)
-        destination
-      end
-
-      # Download +source+ from S3 into +destination+.
-      #
-      # @param [String, IMW::Resource, #path, #bucket] source
-      # @param [String, IMW::Resource, #write] destination
-      # @return [IMW::Resource] the new resource
-      def self.get source, destination
-        source      = IMW.open(source)
-        destination = IMW.open!(destination)
-        raise IMW::ArgumentError.new("source must be on S3 -- #{source} given") unless source.on_s3?
-        make_connection!
-        AWS::S3::S3Object.stream(source.raw_path, source.bucket) do |chunk|
-          destination.write(chunk)
-        end
-        destination.close
-        destination.reopen
-      end
-
-      # Copy S3 resource +source+ to +destination+.
-      #
-      # @param [String, IMW::Resource, #path, #bucket] source
-      # @param [String, IMW::Resource, #path, #bucket] destination
-      # @return [IMW::Resource] the new resource
-      def self.copy source, destination
-        source      = IMW.open(source)
-        destination = IMW.open(destination)
-        raise IMW::PathError.new("Bucket names must be non-blank and match to 'copy'") unless source.bucket.present? && destination.bucket.present? && source.bucket == destination.bucket
-        make_connection!
-        AWS::S3::Object.copy(source.raw_path, destination.raw_path, destination.bucket)
-        destination
-      end
-
-      # Return the resource at the base path of this resource joined
-      # to +path+.
-      #
-      #   IMW.open('s3:://bucket/path/to/dir').join('subdir')
-      #   #=> IMW::Resource at 's3://bucket/path/to/dir/subdir'
-      #
-      # @param [Array<String>] paths
-      # @return [IMW::Resource]
-      def join *paths
-        IMW.open(File.join(stripped_uri.to_s, *paths))
-      end
-
-      protected
-      # Make an S3 connection.
-      #
-      # Uses settings defined in IMW::AWS_CREDENTIALS.
-      #
-      # @return [AWS
-      def self.make_connection!
-        return @connection if @connection
-        raise IMW::Error.new("Must define a constant IMW::AWS_CREDENTIALS with an :access_key_id and a :secret_access_key before using S3 resources") unless defined?(IMW::AWS_CREDENTIALS)
-        require 'aws/s3'
-        @connection = AWS::S3::Base.establish_connection!(IMW::AWS_CREDENTIALS)
-      end
-      
-    end
-  end
-end
-

data/lib/imw/schemes/sql.rb

@@ -1,129 +0,0 @@
-require 'dbi'
-
-module IMW
-  module Schemes
-
-    # Encapsulates a connection to a relational database.
-    #
-    # Calling
-    #
-    #   IMW.open('sql://host:port/database_name')
-    #
-    # shold create a connection to a database at the given +port+ on
-    # the given +host+ using the given +database_name+.
-    module SQL
-
-      # A base implementation of a connection to a relational
-      # database.
-      #
-      # The Base#extended method will examine the +scheme+ of an
-      # object extended with this module and choose a more specific
-      # database adaptor module to extend with as well.
-      module Base
-
-        # When an IMW::Resource is extended use URI's scheme to choose
-        # which other module inside IMW::Schemes::SQL to extend with.
-        def self.extended obj
-          case obj.scheme
-          when 'mysql'      then obj.extend(IMW::Schemes::SQL::MySQL)
-          when 'postgresql' then obj.extend(IMW::Schemes::SQL::PostgreSQL)
-          else raise IMW::ArgumentError.new("Unknown database type: #{obj.scheme}")
-          end
-        end
-        
-        # For an SQL connection the database will be the same as the
-        # path.
-        #
-        # @return [String]
-        def database
-          @database ||= path.tr('/','')
-        end
-
-        # Redefineeach method inappropriate for databases.
-        [:dirname, :basename, :extname, :extension, :name].each do |method|
-          define_method(method) do
-            nil
-          end
-        end
-
-        # Return a summary of this database.
-        #
-        # Purposefully does not call +super+.
-        #
-        # @return [Hash]
-        def external_summary
-          {
-            :uri      => uri.to_s,
-            :database => database
-          }
-        end
-
-        # The (cached) database connection for this resource.
-        #
-        # @return [DBI::DatabaseHandle]
-        def connection
-          @connection ||= DBI.connect("#{dbi_module}:#{database}:#{host}", user, password)
-        end
-
-        # Return the password associated with user's account on the
-        # given database.
-        #
-        # @return [String]
-        def password
-          @password ||= resource_options[:password]
-        end
-
-        # Return an array of the table names in the current database.
-        #
-        # @return [Array<String>]
-        def tables
-          [].tap do |table_names|
-            execute("SHOW TABLES") do |row|
-              table_names << row.first
-            end
-          end
-        end
-
-        # Execute the (joined) +query_string_parts+ using this
-        # resource's cached connection.
-        #
-        # If passed a block, yield each row of the result set to the
-        # block.
-        #
-        # @param [Array<String>] query_string_parts
-        # @yield [DBI::Row]
-        # @return [DBI::StatementHandle]
-        def execute *query_string_parts, &block
-          query = query_string_parts.join(' ')
-          IMW.announce_if_verbose "Querying #{self}: #{query}"
-          statement = connection.execute(query)
-          block_given? ? statement.fetch(&block) : statement
-        end
-      end
-
-      # Module for MySQL databases.
-      module MySQL
-
-        # Return the name of the DBI module used to connect to MySQL.
-        #
-        # @return [String]
-        def dbi_module
-          "DBI:Mysql"
-        end
-      end
-
-      # Module for PostgreSQL databases.
-      module PostgreSQL
-
-        # Return the name of the DBI module used to connect to PostgreSQL.
-        #
-        # @return [String]
-        def dbi_module
-          "DBI:Pg"
-        end
-      end
-      
-    end
-  end
-end
-  

data/lib/imw/tools.rb

@@ -1,12 +0,0 @@
-module IMW
-  module Tools
-    autoload :Aggregator,        'imw/tools/aggregator'
-    autoload :Archiver,          'imw/tools/archiver'
-    autoload :Transferer,        'imw/tools/transferer'
-    autoload :Summarizer,        'imw/tools/summarizer'
-    autoload :ExtensionAnalyzer, 'imw/tools/extension_analyzer'
-    autoload :Downloader,        'imw/tools/downloader'
-  end
-end
-
-

data/lib/imw/tools/aggregator.rb

@@ -1,148 +0,0 @@
-require 'imw/resource'
-
-module IMW
-  module Tools
-    
-    # Aggregates resources into a single local directory.
-    #
-    # The directory should already exist.
-    #
-    # Any local resources will be copied into the directory.
-    #
-    # Any remote resources will be downloaded into the directory.
-    # 
-    # If any of the resources are archives, they will first be
-    # extracted, with only their contents winding up in the final
-    # directory (the file hierarchy of the archive will be preserved).
-    #
-    # If any of the resources are compressed, they will first be
-    # uncompressed before being added to the directory.
-    #
-    # As an example:
-    # 
-    #   aggregator = IMW::Tools::Aggregator.new '/path/to/agg_dir'
-    #   aggregator.aggregate '/path/to/my/regular_file.tsv', '/path/to/an/archive.tar.bz2', '/path/to/my_compressed_file.gz', 'http://mywebsite.com/index.html'
-    #
-    # This will create a directory at <tt>/path/to/agg_dir</tt> which
-    # looks like
-    #
-    #   path_to_agg_dir
-    #   |-- regular_file.tsv
-    #   |-- archive
-    #   |   |-- internal_archive_file_1
-    #   |   |-- internal_archive_file_2
-    #   |   ...
-    #   |   `-- internal_archive_file_N
-    #   |-- my_compressed_file
-    #   `-- index.html
-    #
-    # Notice that
-    #
-    # - the local file was copied over
-    #
-    # - the remote file was downloaded and copied over
-    #
-    # - the tar archive was first exctracted
-    #
-    # - the compressed file was aggregated
-    #
-    # This process can take a while when the constituent files are
-    # large.
-    class Aggregator
-
-      attr_reader :dir
-
-      def initialize dir
-        self.dir = IMW.open(dir)
-      end
-
-      # Set the directory for this Aggregator.
-      #
-      # Will raise unless +new_dir+ is an existing, local directory.
-      #
-      # @param [String, IMW::Resource] new_dir
-      # @return [IMW::Resource]
-      def dir= new_dir
-        @dir = IMW.open(new_dir)
-        raise IMW::SchemError.new("Aggregator requires a local directory, not #{@dir}") unless @dir.is_local?
-        @dir.should_exist! "Aggregator requires the aggregation directory to already exist"
-        raise IMW::PathError.new("Aggregator requires a directory, not #{@dir}") unless @dir.is_directory?
-        @dir
-      end
-      
-      # Return a list of error messages for this Aggregator.
-      #
-      # @return [Array] the error messages
-      def errors
-        @errors ||= []      
-      end
-
-      # Was this archiver successful (did it not have any errors)?
-      #
-      # @return [true, false]
-      def success?
-        errors.empty?
-      end
-
-      # Aggregate the given inputs into this Aggregator's +dir+.
-      #
-      # @param [Array<IMW::Resource,String>] inputs
-      # @return [IMW::Tools::Aggregator]
-      def aggregate *paths_or_inputs
-        @errors = []
-        paths_or_inputs.flatten.compact.each do |path_or_input|
-          input = IMW.open(path_or_input)
-          if input.is_local?
-            aggregate_local_input(input)
-          else
-            download = download_remote_input(input)
-            if download.is_compressed? || download.is_archive?
-              aggregate_local_input(download)
-              download.rm!
-            end
-          end
-        end
-      end
-
-      protected
-
-      # Aggregate a local input.
-      #
-      # Will extract archives, decompress compressed files, and copy
-      # regular files and directories (but will not recurse into
-      # directories to find archives or compressed files).
-      #
-      # @param [IMW::Resource] input
-      def aggregate_local_input input
-        new_path      = File.join(dir.path, input.basename)
-        case
-        when input.is_archive?
-          IMW.announce_if_verbose("Aggregating and extracting #{input} to #{dir}...")
-          FileUtils.cd(dir.path) do
-            input.extract
-          end
-        when input.is_compressed?
-          IMW.announce_if_verbose("Decompressing #{input}...")
-          input.cp(new_path).decompress!
-        else
-          IMW.announce_if_verbose("Copying #{input}...")
-          input.cp(new_path)
-        end
-      end
-
-      # Download a remote input to this Aggregator's +dir+.
-      #
-      # @param [IMW::Resource] input
-      def download_remote_input input
-        IMW.announce_if_verbose("Downloading #{input}...")
-        input.cp(File.join(dir.path, input.effective_basename))
-      end
-      
-      def add_processing_error error # :nodoc:
-        IMW.logger.warn error      
-        errors << error
-      end
-      
-    end
-  end
-end