imw 0.2.9 → 0.2.10

data/VERSION

@@ -1 +1 @@
-0.2.9
+0.2.10

data/examples/metadata.yml

@@ -0,0 +1,10 @@
+---
+
+metadata:
+  /path/to/my/data.tsv:
+    - name: foobar
+      datatype: foobar
+      doc: foobar
+
+
+

data/lib/imw/tools/aggregator.rb

@@ -0,0 +1,148 @@
+require 'imw/resource'
+
+module IMW
+  module Tools
+    
+    # Aggregates resources into a single local directory.
+    #
+    # The directory should already exist.
+    #
+    # Any local resources will be copied into the directory.
+    #
+    # Any remote resources will be downloaded into the directory.
+    # 
+    # If any of the resources are (local and) archives, they will first be
+    # extracted, with only their contents winding up in the final
+    # directory (the file hierarchy of the archive will be preserved).
+    #
+    # If any of the resources are (local and) compressed, they will
+    # first be uncompressed before being added to the directory.
+    #
+    # As an example:
+    # 
+    #   aggregator = IMW::Tools::Aggregator.new '/path/to/agg_dir'
+    #   aggregator.aggregate '/path/to/my/regular_file.tsv', '/path/to/an/archive.tar.bz2', '/path/to/my_compressed_file.gz', 'http://mywebsite.com/index.html'
+    #
+    # This will create a directory at <tt>/path/to/agg_dir</tt> which
+    # looks like
+    #
+    #   path_to_agg_dir
+    #   |-- regular_file.tsv
+    #   |-- archive
+    #   |   |-- internal_archive_file_1
+    #   |   |-- internal_archive_file_2
+    #   |   ...
+    #   |   `-- internal_archive_file_N
+    #   |-- my_compressed_file
+    #   `-- index.html
+    #
+    # Notice that
+    #
+    # - the local file was copied over
+    #
+    # - the remote file was downloaded and copied over
+    #
+    # - the tar archive was first exctracted
+    #
+    # - the compressed file was aggregated
+    #
+    # This process can take a while when the constituent files are
+    # large.
+    class Aggregator
+
+      attr_reader :dir
+
+      def initialize dir
+        self.dir = IMW.open(dir)
+      end
+
+      # Set the directory for this Aggregator.
+      #
+      # Will raise unless +new_dir+ is an existing, local directory.
+      #
+      # @param [String, IMW::Resource] new_dir
+      # @return [IMW::Resource]
+      def dir= new_dir
+        @dir = IMW.open(new_dir)
+        raise IMW::SchemError.new("Aggregator requires a local directory, not #{@dir}") unless @dir.is_local?
+        @dir.should_exist! "Aggregator requires the aggregation directory to already exist"
+        raise IMW::PathError.new("Aggregator requires a directory, not #{@dir}") unless @dir.is_directory?
+        @dir
+      end
+      
+      # Return a list of error messages for this Aggregator.
+      #
+      # @return [Array] the error messages
+      def errors
+        @errors ||= []      
+      end
+
+      # Was this archiver successful (did it not have any errors)?
+      #
+      # @return [true, false]
+      def success?
+        errors.empty?
+      end
+
+      # Aggregate the given inputs into this Aggregator's +dir+.
+      #
+      # @param [Array<IMW::Resource,String>] inputs
+      # @return [IMW::Tools::Aggregator]
+      def aggregate *paths_or_inputs
+        @errors = []
+        paths_or_inputs.each do |path_or_input|
+          input = IMW.open(path_or_input)
+          if input.is_local?
+            aggregate_local_input(input)
+          else
+            download = download_remote_input(input)
+            if download.is_compressed? || download.is_archive?
+              aggregate_local_input(download)
+              download.rm!
+            end
+          end
+        end
+      end
+
+      protected
+
+      # Aggregate a local input.
+      #
+      # Will extract archives, decompress compressed files, and copy
+      # regular files and directories (but will not recurse into
+      # directories to find archives or compressed files).
+      #
+      # @param [IMW::Resource] input
+      def aggregate_local_input input
+        new_path      = File.join(dir.path, input.basename)
+        case
+        when input.is_archive?
+          IMW.announce_if_verbose("Aggregating and extracting #{input} to #{dir}...")
+          FileUtils.cd(dir.path) do
+            input.extract
+          end
+        when input.is_compressed?
+          IMW.announce_if_verbose("Decompressing #{input}...")
+          input.cp(new_path).decompress!
+        else
+          IMW.announce_if_verbose("Copying #{input}...")
+          input.cp(new_path)
+        end
+      end
+
+      # Download a remote input to this Aggregator's +dir+.
+      #
+      # @param [IMW::Resource] input
+      def download_remote_input input
+        IMW.announce_if_verbose("Downloading #{input}...")
+        input.cp(File.join(dir.path, input.effective_basename))
+      end
+      
+      def add_processing_error error # :nodoc:
+        IMW.logger.warn error      
+        errors << error
+      end
+      
+    end
+  end
+end

data/lib/imw/tools.rb

@@ -1,5 +1,6 @@
 module IMW
   module Tools
+    autoload :Aggregator,        'imw/tools/aggregator'
     autoload :Archiver,          'imw/tools/archiver'
     autoload :Transferer,        'imw/tools/transferer'
     autoload :Summarizer,        'imw/tools/summarizer'

data/lib/imw/utils/error.rb

@@ -22,7 +22,13 @@ module IMW
   # Error communicating with a remote entity.
   NetworkError = Class.new(Error)
 
-  # Error communicating with a remote entity.
+  # Raised when a resource is of the wrong scheme.
+  SchemeError = Class.new(Error)
+
+  # Raised when a resource is of the wrong (or malformed) format.
+  FormatError = Class.new(Error)
+
+  # Bad argument.
   ArgumentError = Class.new(Error)
 
   # Error in defining or matching a schema.

data/spec/imw/tools/aggregator_spec.rb

@@ -0,0 +1,71 @@
+require File.dirname(__FILE__) + "/../../spec_helper"
+
+describe IMW::Tools::Aggregator do
+  before do
+    @dir = 'agg_dir'
+    FileUtils.mkdir_p(@dir)
+
+    # remote files
+    @homepage = "http://www.google.com"
+    @website  = "http://www.google.com/support/"
+    @remote_files = [@homepage, @website]
+
+    # regular files
+    @csv  = "foobar-csv.csv"
+    @xml  = "foobar-xml.xml"
+    @txt  = "foobar-txt.txt"
+    @blah = "foobar"
+    @files = [@csv, @xml, @txt, @blah]
+
+    # compressed files
+    @bz2  = "foobar-bz2.bz2"
+    @gz   = "foobar-gz.gz"
+    @compressed_files = [@bz2, @gz]
+
+    # archives
+    @zip      = "foobar-zip.zip"
+    @tarbz2   = "foobar-tarbz2.tar.bz2"
+    @targz    = "foobar-targz.tar.gz"
+    @tar      = "foobar-tar.tar"
+    @rar      = "foobar-rar.rar"
+    @archives = [@zip, @tarbz2, @targz, @rar, @tar]
+
+    @local_files = @files + @compressed_files + @archives
+
+    @all_files = @remote_files + @local_files
+    
+    @local_files.each do |path|
+      IMWTest::Random.file path
+    end
+
+    @aggregator = IMW::Tools::Aggregator.new @dir
+  end
+    
+  it "should copy regular files to its directory" do
+    @aggregator.aggregate *@files
+    @aggregator.dir.path.should contain(*@files)
+    @files.each { |path| IMW.open(path).exist?.should be_true }
+  end
+
+  it "should copy remote files to its archive directory" do
+    @aggregator.aggregate *@remote_files
+    @aggregator.dir.path.should contain('_index', 'support') # _index from Http#effective_basename on http://www.google.com
+  end
+
+  it "should uncompress compressed files to its directory" do
+    @aggregator.aggregate *@compressed_files
+    @aggregator.dir.path.should     contain('foobar-bz2', 'foobar-gz')
+    @aggregator.dir.path.should_not contain(*@compressed_files)
+  end
+    
+  it "should copy the content of archive files to its archive directory (but not the actual archives)" do
+    @aggregator.aggregate *@archives
+    @archives.each do |archive|
+      @aggregator.dir.path.should_not contain(archive)
+      @aggregator.dir.path.should     contain(*IMW.open(archive).contents)
+    end
+  end
+  
+end
+
+

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: imw
 version: !ruby/object:Gem::Version 
-  hash: 5
+  hash: 3
   prerelease: false
   segments: 
   - 0
   - 2
-  - 9
-  version: 0.2.9
+  - 10
+  version: 0.2.10
 platform: ruby
 authors: 
 - Dhruv Bansal
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-26 00:00:00 -05:00
+date: 2010-10-29 00:00:00 -05:00
 default_executable: imw
 dependencies: []
 
@@ -40,6 +40,7 @@ files:
 - bin/imw
 - etc/imwrc.rb
 - examples/dataset.rb
+- examples/metadata.yml
 - lib/imw.rb
 - lib/imw/archives.rb
 - lib/imw/archives/rar.rb
@@ -85,6 +86,7 @@ files:
 - lib/imw/schemes/s3.rb
 - lib/imw/schemes/sql.rb
 - lib/imw/tools.rb
+- lib/imw/tools/aggregator.rb
 - lib/imw/tools/archiver.rb
 - lib/imw/tools/downloader.rb
 - lib/imw/tools/extension_analyzer.rb
@@ -164,6 +166,7 @@ files:
 - spec/imw/schemes/remote_spec.rb
 - spec/imw/schemes/s3_spec.rb
 - spec/imw/schemes/sql_spec.rb
+- spec/imw/tools/aggregator_spec.rb
 - spec/imw/tools/archiver_spec.rb
 - spec/imw/tools/summarizer_spec.rb
 - spec/imw/tools/transferer_spec.rb
@@ -222,6 +225,7 @@ test_files:
 - spec/imw/tools/archiver_spec.rb
 - spec/imw/tools/summarizer_spec.rb
 - spec/imw/tools/transferer_spec.rb
+- spec/imw/tools/aggregator_spec.rb
 - spec/imw/compressed_files/compressible_spec.rb
 - spec/imw/compressed_files/bz2_spec.rb
 - spec/imw/compressed_files/gz_spec.rb