imw 0.2.12 → 0.2.13

data/VERSION

@@ -1 +1 @@
-0.2.12
+0.2.13

data/lib/imw/schemes/ftp.rb

@@ -0,0 +1,142 @@
+module IMW
+  module Schemes
+
+    # Defines methods for reading and writing data from an FTP server.
+    #
+    #   IMW.open('ftp://user:pass@my_bucket/path/to/some/file.csv')
+    #
+    # Learn more about {Amazon Web Services}[http://aws.amazon.com].
+    module FTP
+
+      module Base
+
+      # Is this resource an FTP resource?
+      #
+      # @return [true, false] 
+      def on_ftp?
+        true
+      end
+      alias_method :is_ftp?, :on_ftp?
+
+      # Copy this resource to the +new_uri+.
+      #
+      # @param [String, IMW::Resource] new_uri
+      # @return [IMW::Resource] the new resource
+      def cp new_uri
+        local_obj = IMW.open(new_uri)
+        raise IMW::PathError.new("FTP resources (#{self}) can only be downloaded to a local path") unless local_obj.is_local?
+        local_obj.dir.should_exist!
+        FTP.open(host, user, password) do |ftp|
+          ftp.get(path, local_obj.path)
+        end
+        local_obj
+      end
+
+      # Does this resource exist on S3?
+      #
+      # @return [true, false]
+      def exist?
+        s3_object.exists?
+      end
+      alias_method :exists?, :exist?
+
+      # Remove this resource from S3.
+      #
+      # @return [IMW::Resource] the deleted object
+      def rm
+        s3_object.delete
+      end
+      alias_method :rm!, :rm
+
+      # Return the S3N URL for this S3 object
+      #
+      #   resource = IMW.open('s3://my_bucket/path/to/some/obj')
+      #   resource.s3n_url
+      #   => 's3n://my_bucket/path/to/some/obj'
+      #
+      # @return [String]
+      def s3n_url
+        uri.to_s.gsub(/^s3:/, 's3n:')
+      end
+
+      # Return the contents of this S3 object.
+      #
+      # @return [String]
+      def read
+        s3_object.value
+      end
+
+      # Store +source+ into +destination+.
+      # 
+      # @param [String, IMW::Resource, #io] source
+      # @param [String, IMW::Resource, #path, #bucket] destination
+      # @return [IMW::Resource] the new S3 object
+      def self.put source, destination
+        source       = IMW.open(source)
+        destintation = IMW.open(destination)
+        raise IMW::ArgumentError.new("destination must be on S3 -- #{destination} given") unless destination.on_s3?
+        make_connection!
+        AWS::S3::S3Object.store(destination.path, source.io, destination.bucket)
+        destination
+      end
+
+      # Download +source+ from S3 into +destination+.
+      #
+      # @param [String, IMW::Resource, #path, #bucket] source
+      # @param [String, IMW::Resource, #write] destination
+      # @return [IMW::Resource] the new resource
+      def self.get source, destination
+        source      = IMW.open(source)
+        destination = IMW.open!(destination)
+        raise IMW::ArgumentError.new("source must be on S3 -- #{source} given") unless source.on_s3?
+        make_connection!
+        AWS::S3::S3Object.stream(source.path, source.bucket) do |chunk|
+          destination.write(chunk)
+        end
+        destination.close
+        destination.reopen
+      end
+
+      # Copy S3 resource +source+ to +destination+.
+      #
+      # @param [String, IMW::Resource, #path, #bucket] source
+      # @param [String, IMW::Resource, #path, #bucket] destination
+      # @return [IMW::Resource] the new resource
+      def self.copy source, destination
+        source      = IMW.open(source)
+        destination = IMW.open(destination)
+        raise IMW::PathError.new("Bucket names must be non-blank and match to 'copy'") unless source.bucket.present? && destination.bucket.present? && source.bucket == destination.bucket
+        make_connection!
+        AWS::S3::Object.copy(source.path, destination.path, destination.bucket)
+        destination
+      end
+
+      # Return the resource at the base path of this resource joined
+      # to +path+.
+      #
+      #   IMW.open('s3:://bucket/path/to/dir').join('subdir')
+      #   #=> IMW::Resource at 's3://bucket/path/to/dir/subdir'
+      #
+      # @param [Array<String>] paths
+      # @return [IMW::Resource]
+      def join *paths
+        IMW.open(File.join(stripped_uri.to_s, *paths))
+      end
+
+      protected
+      # Make an S3 connection.
+      #
+      # Uses settings defined in IMW::AWS_CREDENTIALS.
+      #
+      # @return [AWS
+      def self.make_connection!
+        return @connection if @connection
+        raise IMW::Error.new("Must define a constant IMW::AWS_CREDENTIALS with an :access_key_id and a :secret_access_key before using S3 resources") unless defined?(IMW::AWS_CREDENTIALS)
+        require 'aws/s3'
+        @connection = AWS::S3::Base.establish_connection!(IMW::AWS_CREDENTIALS)
+      end
+      
+    end
+  end
+end
+

data/lib/imw/tools/extension_analyzer.rb

@@ -4,8 +4,10 @@ module IMW
     # Mixin with some heuristic methods for identifying common
     # extensions and likely data formats for a collection of files.
     #
-    # Requires the including class to define a method +resources+ which
-    # returns an array of IMW::Resource objects.
+    # Requires the including class to define a method +resources+
+    # which returns an array of IMW::Resource objects as well as a
+    # method +total_size+ which gives the total size of the resources
+    # (for weighting extensions by size).
     module ExtensionAnalyzer
 
       # Return the file counts of each extension.
@@ -24,7 +26,7 @@ module IMW
       # Return the most common extension by count of files.
       def most_common_extension_by_count
         return @most_common_extension_by_count if @most_common_extension_by_count
-        current_count, current_extension = 0, nil
+        current_count, current_extension = 0, ''
         extension_counts.each_pair do |extension, count|
           current_extension = extension if count > current_count
         end
@@ -63,11 +65,14 @@ module IMW
       # @return [String]
       def most_common_extension_by_size
         return @most_common_extension_by_size if @most_common_extension_by_size
-        current_size, current_extension = 0, nil
+        current_size, current_extension = 0, ''
         extension_sizes.each_pair do |extension, size|
-          current_extension = extension if size > current_size
+          if size > current_size
+            current_extension = extension
+            current_size      = size
+          end
         end
-        if current_extension.strip.blank? then current_extension = 'flat' end
+        current_extension = 'flat' if current_extension.strip.blank?
         @most_common_extension_by_size = current_extension
       end
 
@@ -90,8 +95,8 @@ module IMW
         return most_common_extension_by_size if most_common_extension_by_size == most_common_extension_by_count # no contest
         count_fraction = normalized_extension_counts[most_common_extension_by_count]
         size_fraction  = normalized_extension_sizes[most_common_extension_by_size]
-        return most_common_extension_by_count if count_fraction > 0.5 and size_fraction < 0.5 # FIXME arbitrary
-        return most_common_extension_by_size  if count_fraction < 0.5 and size_fraction > 0.5
+        return most_common_extension_by_count if count_fraction >= 0.5 and size_fraction < 0.5 # FIXME arbitrary
+        return most_common_extension_by_size  if count_fraction < 0.5 and size_fraction >= 0.5
         most_common_extension_by_size # default to size
       end
 

data/lib/imw/utils/has_uri.rb

@@ -79,6 +79,13 @@ module IMW
         @user ||= uri.user
       end
 
+      # Returns the password associated with access to this URI.
+      #
+      # @return [String]
+      def password
+        @password ||= uri.password
+      end
+      
       # Return the fragment part of this resource's URI.
       #
       # Will likely be +nil+ for local resources.

data/spec/imw/tools/extension_analyzer_spec.rb

@@ -0,0 +1,153 @@
+require File.dirname(__FILE__) + "/../../spec_helper"
+
+describe IMW::Tools::ExtensionAnalyzer do
+
+  before do
+    class Analyzer
+      attr_accessor :dir, :resources
+      include IMW::Tools::ExtensionAnalyzer
+      def initialize dir
+        self.dir = File.expand_path(dir)
+        @resources = IMW.open(self.dir).all_resources
+      end
+      def total_size
+        @total_size ||= resources.map(&:size).inject(0) { |e, sum| sum += e }
+      end
+    end
+  end
+
+  describe 'working with an empty directory' do
+    before do
+      @analyzer = Analyzer.new(IMWTest::TMP_DIR)
+    end
+
+    %w[most_common_extension_by_count most_common_extension_by_size most_common_extension].each do |method|
+      it "should return 'flat' when asked for its '#{method}'" do
+        @analyzer.send(method).should == 'flat'
+      end
+    end
+    
+    %w[extension_counts normalized_extension_counts extension_sizes normalized_extension_sizes].each do |method|
+      it "should return an empty hash when asked for its '#{method}'" do
+        @analyzer.send(method).should == {}
+      end
+    end
+  end
+
+  describe 'working with files that lack extensions' do
+    
+    before do
+      @dir = File.join(IMWTest::TMP_DIR, 'ext_dir')
+      FileUtils.mkdir_p(@dir)
+
+      @f1  = "foobar1"
+      @f2  = "foobar2"
+      @f3  = "foobar1"
+      @files = [@f1, @f2, @f3]
+      
+      @files.each do |basename|
+        IMWTest::Random.file File.join(@dir, basename)
+      end
+      
+      @analyzer = Analyzer.new(IMWTest::TMP_DIR)
+    end
+
+    %w[most_common_extension_by_count most_common_extension_by_size most_common_extension].each do |method|
+      it "should return 'flat' when asked for its '#{method}'" do
+        @analyzer.send(method).should == 'flat'
+      end
+    end
+  end
+  
+  describe 'working with a directory of files' do
+    before do
+      @dir = File.join(IMWTest::TMP_DIR, 'ext_dir')
+      FileUtils.mkdir_p(@dir)
+
+      @csv1  = "foobar1.csv"
+      @csv2  = "foobar2.csv"
+      @xml  = "foobar1.xml"
+      @txt  = "foobar1.txt"
+      @files = [@csv1, @csv2, @xml, @txt]
+      
+      @files.each do |basename|
+        IMWTest::Random.file File.join(@dir, basename)
+      end
+
+      def bloat basename
+        File.open(File.join(@dir, basename), 'a') do |f|
+          1000.times do
+            f.write( 'hello ' * 100)
+          end
+        end
+      end
+
+      @analyzer = Analyzer.new @dir
+    end
+
+    describe "working with extension counts" do
+      it "should be able to return counts by extension" do
+        @analyzer.extension_counts.should == {'xml' => 1, 'txt' => 1, 'csv' => 2 }
+      end
+
+      it "should be able to return the most common extension by count" do
+        @analyzer.most_common_extension_by_count.should == 'csv'
+      end
+
+      it "should be able to calculate extension weighted by number of files" do
+        @analyzer.normalized_extension_counts.should == { 'csv' => 0.5, 'xml' => 0.25, 'txt' => 0.25 }
+      end
+    end
+
+    describe "working with extension sizes" do
+      it "should be able to calculate extension sizes" do
+        csv_size = File.size(File.join(@dir, @csv1)) + File.size(File.join(@dir, @csv2))
+        xml_size = File.size(File.join(@dir, @xml))
+        txt_size = File.size(File.join(@dir, @txt))
+        @analyzer.extension_sizes.should == { 'csv' => csv_size, 'xml' => xml_size, 'txt' => txt_size }
+      end
+
+      it "should be able to return the most common extension by size" do
+        bloat @txt
+        @analyzer.most_common_extension_by_size.should == 'txt'
+      end
+
+      it "should be able to calculate extension sizes" do
+        csv_size = File.size(File.join(@dir, @csv1)) + File.size(File.join(@dir, @csv2))
+        xml_size = File.size(File.join(@dir, @xml))
+        txt_size = File.size(File.join(@dir, @txt))
+        total_size = csv_size + xml_size + txt_size
+        @analyzer.normalized_extension_sizes.should == { 'csv' => csv_size.to_f / total_size.to_f, 'xml' => xml_size.to_f / total_size.to_f, 'txt' => txt_size.to_f / total_size.to_f }
+      end
+    end
+
+    describe "determining the most common extension" do
+
+      it "should obviously return an extension if it is the most common by count as well as the most common by size" do
+        bloat @csv1
+        @analyzer.most_common_extension.should == 'csv'
+      end
+
+      it "should return the most common extension by count if the count fraction is half or greater and the size fraction is less than half" do
+        bloat @txt
+        bloat @xml
+        @analyzer.most_common_extension.should == 'csv'
+      end
+
+      it "should return the most common extension by size if the size fraction is half or greater and the count fraction is less than half" do
+        # need to add an xml file
+        @new_xml = File.join(@dir, 'xml2.xml')
+        IMWTest::Random.file(@new_xml)
+        bloat @txt
+        @analyzer = Analyzer.new @dir
+        @analyzer.most_common_extension.should == 'txt'
+      end
+
+      it "should return the most common extension by size if no other conditions are met" do
+        bloat @txt
+        @analyzer.most_common_extension.should == 'txt'
+      end
+      
+    end
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: imw
 version: !ruby/object:Gem::Version 
-  hash: 15
+  hash: 13
   prerelease: false
   segments: 
   - 0
   - 2
-  - 12
-  version: 0.2.12
+  - 13
+  version: 0.2.13
 platform: ruby
 authors: 
 - Dhruv Bansal
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-17 00:00:00 -06:00
+date: 2010-11-22 00:00:00 -06:00
 default_executable: 
 dependencies: []
 
@@ -81,6 +81,7 @@ files:
 - lib/imw/resource.rb
 - lib/imw/runner.rb
 - lib/imw/schemes.rb
+- lib/imw/schemes/ftp.rb
 - lib/imw/schemes/hdfs.rb
 - lib/imw/schemes/http.rb
 - lib/imw/schemes/local.rb
@@ -170,6 +171,7 @@ files:
 - spec/imw/schemes/sql_spec.rb
 - spec/imw/tools/aggregator_spec.rb
 - spec/imw/tools/archiver_spec.rb
+- spec/imw/tools/extension_analyzer_spec.rb
 - spec/imw/tools/summarizer_spec.rb
 - spec/imw/tools/transferer_spec.rb
 - spec/imw/utils/dynamically_extendable_spec.rb
@@ -225,6 +227,7 @@ test_files:
 - spec/imw/archives/tarbz2_spec.rb
 - spec/imw/archives/rar_spec.rb
 - spec/imw/tools/archiver_spec.rb
+- spec/imw/tools/extension_analyzer_spec.rb
 - spec/imw/tools/summarizer_spec.rb
 - spec/imw/tools/transferer_spec.rb
 - spec/imw/tools/aggregator_spec.rb