imw 0.2.7 → 0.2.8

data/lib/imw/schemes/http.rb

@@ -28,7 +28,6 @@ module IMW
         IMW::Tools::Transferer.new(:cp, self, new_uri).transfer!
       end
       
-
       # Return the basename of the URI or <tt>_index</tt> if it's
       # blank, as in the case of <tt>http://www.google.com</tt>.
       #
@@ -79,7 +78,7 @@ module IMW
       # @raise [RestClient::NotModified, RestClient::Unauthorized, RestClient::ResourceNotFound, RestClient::RequestFailed] error from RestClient on non-2xx response codes
       def post payload, headers={}, &block
         make_restclient_request do
-          RestClient.post(uri.to_s, payload, headers, &block)
+          RestClient.post(uri.to_s, payload, &block)
         end
       end
 

data/lib/imw/schemes/local.rb

@@ -65,7 +65,7 @@ module IMW
         def dir
           IMW.open(dirname)
         end
-
+        
       end
 
       # Defines methods for appropriate for a local file.
@@ -93,6 +93,14 @@ module IMW
           @io ||= open(path, mode)
         end
 
+        # Close this resource's file handle if it exists.
+        def close
+          # explicitly check the @io instance variable b/c self.io
+          # will open up a new handle by default
+          io.close if @io
+          super()
+        end
+
         # Read from this file.
         #
         # @param [Fixnum] length bytes to read
@@ -101,6 +109,13 @@ module IMW
           io.read(length)
         end
 
+        # Read a line from this file.
+        #
+        # @return [String]
+        def readline
+          io.readline
+        end
+
         # Write to this file
         #
         # @param [String, #to_s] text text to write
@@ -109,6 +124,14 @@ module IMW
           io.write text
         end
 
+        # Write the text with a trailing newline to this resource.
+        #
+        # @param [String, #to_s] text
+        def puts text
+          io.write text.to_s + "\n"
+        end
+        alias_method :<<, :puts
+
         # Return the lines in this file.
         #
         # If passed a block, yield each line of the file to the block.
@@ -132,15 +155,50 @@ module IMW
           io.map(&block)
         end
 
-        # Dump +data+ into this file.
+        # Emit +data+ into this file.
         #
-        # @param [String, Array, #each] data object to dump
-        # @option options [true, false] :persist (false) Don't close the file after writing
-        def dump data, options={}
+        # @param [String, Array, #each] data object to emit
+        def emit data, options={}
           data.each do |element|  # works if data is an Array or a String
             io.puts(element.to_s)
           end
-          io.close unless options[:persist]
+        end
+
+        # Return a snippet of text from this resource.
+        #
+        # Will read the first 1024 bytes and strip non-ASCII
+        # characters from them.  For more control, redefine this
+        # method in another module.
+        #
+        # @return [String]
+        def snippet
+          returning([]) do |snip|
+            io.read(1024).bytes.each do |byte|
+                                        # CR            LF          SPACE            ~
+              snip << byte.chr if byte == 13 || byte == 10 || byte >= 32 && byte <= 126
+            end
+          end.join
+        end
+
+        # Return the number of lines in this file.
+        #
+        # @return [Integer]
+        def num_lines
+          wc[0]
+        end
+
+        # Return the number of words in this file.
+        #
+        # @return [Integer]
+        def num_words
+          wc[1]
+        end
+
+        # Return the number of characters in this file.
+        #
+        # @return [Integer]
+        def num_chars
+          wc[2]
         end
 
         # Return a summary of properties of this local file.
@@ -154,20 +212,40 @@ module IMW
           data = {
             :basename  => basename,
             :size      => size,
-            :extension => extension
+            :extension => extension,
+            :num_lines => num_lines
           }
-          if respond_to?(:snippet)
-            data[:snippet] = snippet
-          end
+          data[:snippet] = snippet if respond_to?(:snippet)
+          data[:schema]  = schema  if respond_to?(:schema)
           data
         end
 
+        protected
+        
+        # Return a triple of line, word, and character counts for this
+        # resource.
+        #
+        # Relies on the Unix utility +wc+.
+        #
+        # @return [Array<Integer>]
+        def wc
+          @wc ||= begin
+                    `wc #{path}`.chomp.strip.split.map(&:to_i)
+                  rescue
+                    [0,0,0] # FIXME
+                  end
+        end
+
       end
 
       # Defines methods for manipulating the contents of a local
       # directory.
       module LocalDirectory
 
+        # Lets local directories contain a special metadata file which
+        # describes their contents.
+        include IMW::Metadata::ContainsMetadata
+
         # Is this resource a directory?
         #
         # @return [true, false]
@@ -207,11 +285,11 @@ module IMW
         # @param [String, IMW::Resource] obj
         # @return [true, false]
         def contains? obj
-          require 'find'
-          obj_path = obj.is_a?(String) ? obj : obj.path
-          Find.find(path) do |sub_path|
-            return true if sub_path.ends_with?(obj_path)
-          end
+          obj = IMW.open(obj)
+          return false unless obj.is_local?
+          return true  if obj.path == path
+          return false unless obj.path.starts_with?(path)
+          return true  if self[obj.path[path.length..-1]].size > 0
           false
         end
 
@@ -277,6 +355,31 @@ module IMW
           self
         end
 
+        # Return the resource at the base path of this resource joined
+        # to +path+.
+        #
+        #   IMW.open('/path/to/dir').join('subdir')
+        #   #=> IMW::Resource at '/path/to/dir/subdir'
+        #
+        # @param [Array<String>] paths
+        # @return [IMW::Resource]
+        def join *paths
+          IMW.open(File.join(stripped_uri.to_s, *paths))
+        end
+
+        # Recursively walk down this directory
+        def walk(options={}, &block)
+          require 'find'
+          Find.find(path) do |path|
+            if options[:only]
+              next if options[:only] == :files && !File.file?(path)
+              next if options[:only] == :directories && !File.directory?(path)
+              next if options[:only] == :symlinks && !File.symlink?(path)
+            end
+            yield path
+          end
+        end
+        
         # Return a hash summarizing this directory with a key
         # <tt>:contents</tt> containing an array of hashes summarizing
         # this directories contents.
@@ -293,10 +396,30 @@ module IMW
             :basename  => basename,
             :size      => size,
             :num_files => contents.length,
-            :contents  => resources.map { |resource| resource.summary }
+            :contents  => resources.map do |resource|
+              resource.guess_schema! if guess_schema? && resource.respond_to?(:guess_schema!)
+              resource_summary = resource.summary
+              resource_summary[:schema] = metadata[resource] if metadata && metadata.describe?(resource) # this should be handled by 'resources' method above
+              resource_summary
+            end
           }
         end
 
+        # Whether or not to have this directory's resources guess
+        # their schemas when none is provided.
+        #
+        # @return [true, false]
+        def guess_schema?
+          (!! @guess_schema)
+        end
+
+        # Force this directory's resources to guess at their schema.
+        #
+        # @return [true]
+        def guess_schema!
+          @guess_schema = true
+        end
+
       end
     end
   end

data/lib/imw/schemes/remote.rb

@@ -39,14 +39,6 @@ module IMW
           @query_string ||= uri.query
         end
 
-        # Return the fragment part of this resource's URI.  Will likely be
-        # +nil+ for local resources.
-        #
-        # @return [String]
-        def fragment
-          @fragment ||= uri.fragment
-        end
-
         # Return the path part of this resource's URI.  Will _not_
         # include the +query_string+ or +fragment+.
         #
@@ -103,8 +95,21 @@ module IMW
 
       module RemoteDirectory
 
+        # Return the resource at the base path of this resource joined
+        # to +path+.
+        #
+        #   IMW.open('http://example.com/path/to/dir').join('subdir')
+        #   #=> IMW::Resource at 'http://example.com/path/to/dir/subdir'
+        #
+        # @param [Array<String>] paths
+        # @return [IMW::Resource]
+        def join *paths
+          IMW.open(File.join(stripped_uri.to_s, *paths))
+        end
+
         #
-        # TODO -- bloody everything
+        # TODO -- bloody everything.  what's the best way to tell if
+        # the remote URL is a directory?
         #
 
         

data/lib/imw/schemes/s3.rb

@@ -116,6 +116,18 @@ module IMW
         destination
       end
 
+      # Return the resource at the base path of this resource joined
+      # to +path+.
+      #
+      #   IMW.open('s3:://bucket/path/to/dir').join('subdir')
+      #   #=> IMW::Resource at 's3://bucket/path/to/dir/subdir'
+      #
+      # @param [Array<String>] paths
+      # @return [IMW::Resource]
+      def join *paths
+        IMW.open(File.join(stripped_uri.to_s, *paths))
+      end
+
       protected
       # Make an S3 connection.
       #

data/lib/imw/schemes/sql.rb

@@ -0,0 +1,117 @@
+require 'dbi'
+
+module IMW
+  module Schemes
+
+    # Encapsulates a connection to a relational database.
+    #
+    # Calling
+    #
+    #   IMW.open('sql://host:port/database_name')
+    #
+    # shold create a connection to a database at the given +port+ on
+    # the given +host+ using the given +database_name+.
+    module SQL
+
+      # A base implementation of a connection to a relational
+      # database.
+      #
+      # The Base#extended method will examine the +scheme+ of an
+      # object extended with this module and choose a more specific
+      # database adaptor module to extend with as well.
+      module Base
+
+        # When an IMW::Resource is extended use URI's scheme to choose
+        # which other module inside IMW::Schemes::SQL to extend with.
+        def self.extended obj
+          case obj.scheme
+          when 'mysql'      then obj.extend(IMW::Schemes::SQL::MySQL)
+          when 'postgresql' then obj.extend(IMW::Schemes::SQL::PostgreSQL)
+          else raise IMW::ArgumentError.new("Unknown database type: #{obj.scheme}")
+          end
+        end
+        
+        # For an SQL connection the database will be the same as the
+        # path.
+        #
+        # @return [String]
+        def database
+          @database ||= path.tr('/','')
+        end
+
+        # Redefineeach method inappropriate for databases.
+        [:dirname, :basename, :extname, :extension, :name].each do |method|
+          define_method(method) do
+            nil
+          end
+        end
+
+        # The (cached) database connection for this resource.
+        #
+        # @return [DBI::DatabaseHandle]
+        def connection
+          @connection ||= DBI.connect("#{dbi_module}:#{database}:#{host}", user, password)
+        end
+
+        # Return the password associated with user's account on the
+        # given database.
+        #
+        # @return [String]
+        def password
+          @password ||= resource_options[:password]
+        end
+
+        # Return an array of the table names in the current database.
+        #
+        # @return [Array<String>]
+        def tables
+          returning([]) do |table_names|
+            execute("SHOW TABLES") do |row|
+              table_names << row.first
+            end
+          end
+        end
+
+        # Execute the (joined) +query_string_parts+ using this
+        # resource's cached connection.
+        #
+        # If passed a block, yield each row of the result set to the
+        # block.
+        #
+        # @param [Array<String>] query_string_parts
+        # @yield [DBI::Row]
+        # @return [DBI::StatementHandle]
+        def execute *query_string_parts, &block
+          query = query_string_parts.join(' ')
+          IMW.announce_if_verbose "Querying #{self}: #{query}"
+          statement = connection.execute(query)
+          block_given? ? statement.fetch(&block) : statement
+        end
+      end
+
+      # Module for MySQL databases.
+      module MySQL
+
+        # Return the name of the DBI module used to connect to MySQL.
+        #
+        # @return [String]
+        def dbi_module
+          "DBI:Mysql"
+        end
+      end
+
+      # Module for PostgreSQL databases.
+      module PostgreSQL
+
+        # Return the name of the DBI module used to connect to PostgreSQL.
+        #
+        # @return [String]
+        def dbi_module
+          "DBI:Pg"
+        end
+      end
+      
+    end
+  end
+end
+  

data/lib/imw/tools.rb

@@ -1,8 +1,10 @@
 module IMW
   module Tools
-    autoload :Archiver,   'imw/tools/archiver'
-    autoload :Transferer, 'imw/tools/transferer'
-    autoload :Summarizer, 'imw/tools/summarizer'
+    autoload :Archiver,          'imw/tools/archiver'
+    autoload :Transferer,        'imw/tools/transferer'
+    autoload :Summarizer,        'imw/tools/summarizer'
+    autoload :ExtensionAnalyzer, 'imw/tools/extension_analyzer'
+    autoload :Downloader,        'imw/tools/downloader'
   end
 end
 

data/lib/imw/tools/downloader.rb

@@ -0,0 +1,63 @@
+module IMW
+  module Tools
+
+    # A class to download a collection of resources to a shared
+    # directory.
+    class Downloader
+
+      def initialize dir, *inputs
+        self.dir    = dir
+        self.inputs = inputs unless inputs.blank?
+      end
+
+      def self.dir= new_dir
+        @dir = IMW.open(new_dir)
+        raise IMW::PathError.new("#{@dir} must be a local directory") unless @dir.is_local? && @dir.is_directory?
+        @dir
+      end
+      attr_reader :dir
+
+      def inputs= new_inputs
+        @inputs = new_inputs.flatten.compact.map { |raw_input| IMW.open(raw_input) }
+      end
+      attr_reader :inputs
+
+      def downloaded_path_for input
+        dir.join(input.respond_to?(:effective_basename) ? input.effective_basename : input.basename)
+      end
+
+      def download!
+        before_download
+        inputs.each do |input|
+          downloaded_path = downloaded_path_for(input)
+          IMW.log_if_verbose "Downloading #{input} to #{downloaded_path}"
+          input.cp(downloaded_path)
+        end
+        after_download
+      end
+
+      def downloaded?
+        downloaded_resources.all? { |resource| resource.exist? }
+      end
+
+      def downloaded_resources
+        inputs.map do |input|
+          IMW.open(downloaded_path_for(input))
+        end
+      end
+
+      def clean!
+        IMW.log_if_verbose("Deleting downloader directory #{dir}")
+        dir.rm_rf!
+      end
+
+      def before_download
+      end
+
+      def after_download
+      end
+
+    end
+  end
+end
+