imw 0.2.17 → 0.2.18

data/Gemfile

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 source :gemcutter
-gem 'activesupport', '2.3.5', :require => 'active_support'
+gem 'activesupport', :require => 'active_support'
 gem 'addressable', :require => 'addressable/uri'
 gem 'uuidtools'
 gem 'rake'

data/Gemfile.lock

@@ -2,8 +2,8 @@ GEM
   remote: http://rubygems.org/
   specs:
     Ascii85 (1.0.0)
-    activesupport (2.3.5)
-    addressable (2.2.2)
+    activesupport (3.0.3)
+    addressable (2.2.4)
     aws-s3 (0.6.2)
       builder
       mime-types
@@ -12,27 +12,27 @@ GEM
     dbi (0.4.5)
       deprecated (= 2.0.1)
     deprecated (2.0.1)
-    fastercsv (1.5.3)
+    fastercsv (1.5.4)
     hpricot (0.8.3)
-    json (1.4.6)
+    json (1.5.1)
     mime-types (1.16)
-    pdf-reader (0.9.0)
+    pdf-reader (0.9.1)
       Ascii85 (>= 0.9)
     rake (0.8.7)
     rest-client (1.6.1)
       mime-types (>= 1.16)
     rspec (1.3.0)
     ruby-ole (1.2.11.1)
-    spreadsheet (0.6.4.1)
-      ruby-ole
-    uuidtools (2.1.1)
-    xml-simple (1.0.12)
+    spreadsheet (0.6.5.1)
+      ruby-ole (>= 1.0)
+    uuidtools (2.1.2)
+    xml-simple (1.0.14)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  activesupport (= 2.3.5)
+  activesupport
   addressable
   aws-s3
   dbi

data/VERSION

@@ -1 +1 @@
-0.2.17
+0.2.18

data/lib/imw/boot.rb

@@ -1,5 +1,3 @@
-require 'imw/utils/extensions/hash'
-
 module IMW
 
   # IMW looks for configuration settings in the following places, in

data/lib/imw/formats/delimited.rb

@@ -105,12 +105,22 @@ module IMW
       # @return [Array<Array>]
       def snippet
         require 'fastercsv'
-        returning([]) do |rows|
-          row_num = 1
-          each do |row|
-            break if row_num > 10
-            rows << row.size.times.map { |index| row[index] }
-            row_num += 1
+        [].tap do |rows|
+          rows_sampled = 0
+          begin
+            each do |row|
+              begin
+                break if rows_sampled > 100
+                row_size = row.size.to_f
+                if (row.reject(&:blank?).size.to_f / row_size) >= 0.5
+                  rows << row.size.times.map { |index| row[index] }
+                  rows_sampled += 1
+                end
+              rescue => e
+                next
+              end
+            end
+          rescue => e
           end
         end
       end
@@ -125,7 +135,7 @@ module IMW
       #
       # @return [Hash]
       def resource_options_compatible_with_faster_csv
-        @compatible_options ||= returning({}) do |compatible_options|
+        @compatible_options ||= {}.tap do |compatible_options|
           FASTER_CSV_OPTION_NAMES.each do |option_name|
             compatible_options[option_name] = resource_options[option_name] if resource_options.has_key?(option_name.to_sym)
           end

data/lib/imw/formats/excel.rb

@@ -75,15 +75,22 @@ module IMW
       # 
       def snippet
         require 'spreadsheet'
-        returning([]) do |snip|
-          row_num = 1
+        [].tap do |snip|
+          rows_sampled = 0
           Spreadsheet.open(path).worksheets.each do |worksheet|
             worksheet.each do |row|
-              break if row_num > 10
-              snip << row.to_a
-              row_num += 1
+              begin
+                break if rows_sampled > 100
+                row_size = row.size.to_f
+                if (row.reject(&:blank?).size.to_f / row_size) > 0.5
+                  snip << row.to_a
+                  rows_sampled += 1
+                end
+              rescue => e
+                next
+              end
             end
-            break if row_num > 10
+            break if rows_sampled > 10
           end
         end
       end

data/lib/imw/metadata/contains_metadata.rb

@@ -27,7 +27,7 @@
       #
       # @return [String, nil]
       def default_metadata_uri
-        contents.detect { |path| path =~ /(icss|metadata).*\.(ya?ml|json)$/i } || File.join(path, "#{basename}.icss.yaml")
+        contents.detect { |path| path =~ /metadata.*\.(ya?ml|json)$/i } || File.join(path, "#{basename}.metadata.yaml")
       end
 
       # Return the metadata for this resource if it exists.

data/lib/imw/metadata/has_metadata.rb

@@ -24,19 +24,24 @@ module IMW
       # @return [Hash]
       def schema
         return @schema if @schema
-        @schema             = IMW::Metadata::Schema.new
+        @schema = IMW::Metadata::Schema.new
         @schema[:type]      = "record"
         @schema[:namespace] = "schema.imw.resource"
-        @schema[:name]      = (basename || '')
-        @schema[:doc]       = description
-        @schema[:fields]    = fields
-        
-        @schema[:non_avro ] = {}
-        @schema[:non_avro][:snippet]      = snippet      if respond_to?(:snippet)
-        @schema[:non_avro][:record_count] = record_count if respond_to?(:record_count)
-        @schema
+        @schema[:name]      = (basename || '')        
+        begin
+          @schema[:doc]       = description
+          @schema[:fields]    = fields
+          
+          @schema[:non_avro ] = {}
+          @schema[:non_avro][:snippet]      = snippet      if respond_to?(:snippet) rescue nil
+          @schema[:non_avro][:record_count] = record_count if respond_to?(:record_count)
+          @schema
+        rescue => e
+          $stdout.puts "Error in producing schema for #{self}: #{e.class} -- #{e.message}"
+          return @schema
+        end
       end
-
+      
       # Return the metadata object that contains metadata for this
       # resource.
       #

data/lib/imw/metadata/has_summary.rb

@@ -20,10 +20,16 @@ module IMW
       # @return [Hash]
       def summary
         return @summary if @summary
-        @summary            = external_summary
-        @summary[:schema]   = schema                   if respond_to?(:schema)
-        @summary[:contents] = resources.map(&:summary) if respond_to?(:resources)
-        @summary
+        @summary = {}
+        begin
+          @summary.merge!(external_summary)
+          @summary[:schema]   = schema                   if respond_to?(:schema)
+          @summary[:contents] = resources.map(&:summary) if respond_to?(:resources)
+          @summary
+        rescue => e
+          # IMW.warn "Error in producing summary for #{self}: #{e.class} -- #{e.message}"
+          return @summary
+        end
       end
 
       # Return information (usually scheme-dependent) on how this

data/lib/imw/parsers/regexp_parser.rb

@@ -56,7 +56,7 @@ module IMW
 
       def parse_line line
         match_data = regexp.match(line.chomp)
-        returning({}) do |hsh|
+        {}.tap do |hsh|
           if match_data
             match_data.captures.each_with_index do |capture, index|
               hsh[fields[index]] = capture

data/lib/imw/schemes/hdfs.rb

@@ -202,7 +202,7 @@ module IMW
       # @yield [String] each line of the file
       # @return [Array] the result of the block on each line
       def map &block
-        returning([]) do |output|
+        [].tap do |output|
           HDFS.fs(:cat, path) do |line|
             output << block.call(line)
           end
@@ -219,7 +219,7 @@ module IMW
       #
       # @return [Array<String>]
       def contents
-        returning([]) do |paths|
+        [].tap do |paths|
           HDFS.fs(:ls, path) do |line|
             next if line =~ /^Found.*items$/
             paths << line.split.last

data/lib/imw/schemes/local.rb

@@ -173,7 +173,7 @@ module IMW
         #
         # @return [String]
         def snippet
-          returning([]) do |snip|
+          [].tap do |snip|
             (io.read(1024) || '').bytes.each do |byte|
               # CR            LF          SPACE            ~
               snip << byte.chr if byte == 13 || byte == 10 || byte >= 32 && byte <= 126

data/lib/imw/schemes/sql.rb

@@ -77,7 +77,7 @@ module IMW
         #
         # @return [Array<String>]
         def tables
-          returning([]) do |table_names|
+          [].tap do |table_names|
             execute("SHOW TABLES") do |row|
               table_names << row.first
             end

data/lib/imw/tools/extension_analyzer.rb

@@ -14,7 +14,7 @@ module IMW
       #
       # @return [Hash]
       def extension_counts
-        @extension_counts ||= returning({}) do |counts|
+        @extension_counts ||= {}.tap do |counts|
           resources.each do |resource|
             next if resource.is_directory?
             counts[resource.extension] = 0 unless counts.has_key?(resource.extension)
@@ -39,7 +39,7 @@ module IMW
       #
       # @return [Hash]
       def normalized_extension_counts
-        @normalized_extension_counts ||= returning({}) do |weighted|
+        @normalized_extension_counts ||= {}.tap do |weighted|
           num_files = resources.reject(&:is_directory?).length.to_f
           extension_counts.each_pair do |extension, count|
             weighted[extension] = count.to_f / num_files
@@ -51,7 +51,7 @@ module IMW
       #
       # @return [Hash]
       def extension_sizes
-        @extension_sizes ||= returning({}) do |sizes|
+        @extension_sizes ||= {}.tap do |sizes|
           resources.each do |resource|
             next if resource.is_directory?
             sizes[resource.extension] = 0 unless sizes.has_key?(resource.extension)            
@@ -80,7 +80,7 @@ module IMW
       #
       # @return [Hash]
       def normalized_extension_sizes
-        @normalized_extension_sizes ||= returning({}) do |weighted|
+        @normalized_extension_sizes ||= {}.tap do |weighted|
           extension_sizes.each_pair do |extension, size|
             weighted[extension] = size.to_f / total_size.to_f
           end

data/lib/imw/utils/has_uri.rb

@@ -100,7 +100,7 @@ module IMW
       #
       # @return [URI::Generic]
       def stripped_uri
-        uri_args = returning({}) do |args|
+        uri_args = {}.tap do |args|
           %w[scheme userinfo host port path].each do |method|
             args[method.to_sym] = respond_to?(method) ? send(method) : uri.send(method)
           end

data/lib/imw/utils.rb

@@ -1,12 +1,12 @@
 require 'rubygems'
 require 'fileutils'
 require 'active_support/core_ext/object/blank'
-require 'active_support/core_ext/object/misc'
+require 'active_support/core_ext/hash/reverse_merge'
+require 'active_support/core_ext/string/starts_ends_with'
 require 'imw/utils/error'
 require 'imw/utils/log'
 require 'imw/utils/paths'
 require 'imw/utils/misc'
-require 'imw/utils/extensions'
 
 module IMW
 
@@ -15,5 +15,60 @@ module IMW
     autoload :DynamicallyExtendable, 'imw/utils/dynamically_extendable'
     autoload :HasURI,                'imw/utils/has_uri'
   end
+
+  # A replacement for the standard system call which raises an
+  # IMW::SystemCallError if the command fails which prints better
+  # debugging info.
+  #
+  # This function relies upon Kernel.system and obeys the same rules:
+  #
+  # - if +commands+ has only only a single element then no shell
+  #   characters or spaces are escaped -- you have to do it yourself
+  #   or you get to use shell characters, depending on your
+  #   perspective.
+  #
+  # - if +commands+ is a list of elements then the second and further
+  #   elements in the list have their shell characters and spaces
+  #   escaped
+  #
+  # But it also has its own rules:
+  #
+  # - When one of the +commands+ is an empty or blank string,
+  #   Kernel.system honors it and escapes it properly and sends it
+  #   along for evaluation.  This can be a problem for some programs
+  #   and so IMW.system excludes blank (as in <tt>blank?</tt>)
+  #   elements of +commands+.
+  #
+  # - +commands+ will be flattened (see the gotcha below)
+  #
+  # Calling out to the shell like this is often brittle.  Imagine
+  # defining
+  #
+  #   prog  = 'some_prog'
+  #   flags = '-v -f'
+  #   args  = 'file.txt'
+  #
+  # and later calling
+  # 
+  #   IMW.system prog, flags, args
+  #
+  # The space in the second argument ('-v -f') will be escaped and
+  # will therefore not be properly parsed by +some_prog+.  Instead try
+  #
+  #   prog  = 'some_prog'
+  #   flags = ['-v', '-f']
+  #   args = ['file.txt']
+  #   
+  #   IMW.system prog, flags, *args
+  #
+  # which will work fine since +flags+ will automatically be flattend.
+  def self.system *commands
+    stripped_commands = commands.flatten.map { |command| command.to_s unless command.blank? }.compact
+    IMW.announce_if_verbose(stripped_commands.join(" "))
+    exit_code = Kernel.system(*stripped_commands)
+    raise IMW::SystemCallError.new($?.dup, commands.join(' ')) unless $?.success?
+    exit_code
+  end
+  
 end
 

data/spec/imw/parsers/line_parser_spec.rb

@@ -56,7 +56,7 @@ describe IMW::Parsers::LineParser do
       end
 
       it "should pass each hash to a block when given one" do
-        results = returning([]) do |array|
+        results = [].tap do |array|
           @parser.parse!(@file) do |hsh|
             hsh.delete(:id)
             array << hsh
@@ -78,7 +78,7 @@ describe IMW::Parsers::LineParser do
       
       it "should pass each object to a block when given one and defined with a class" do
         @parser.klass = OpenStruct
-        results = returning([]) do |array|
+        results = [].tap do |array|
           @parser.parse!(@file) do |obj|
             obj.genus = nil
             array << obj

data/spec/imw/parsers/regexp_parser_spec.rb

@@ -38,5 +38,5 @@ EOF
     end
   end
 end
-  
+
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: imw
 version: !ruby/object:Gem::Version 
-  hash: 53
+  hash: 51
   prerelease: false
   segments: 
   - 0
   - 2
-  - 17
-  version: 0.2.17
+  - 18
+  version: 0.2.18
 platform: ruby
 authors: 
 - Dhruv Bansal
@@ -16,22 +16,76 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-17 00:00:00 -06:00
+date: 2011-02-16 00:00:00 -06:00
 default_executable: 
-dependencies: []
-
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  name: activesupport
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id001
+  type: :runtime
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  name: addressable
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id002
+  type: :runtime
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  name: uuidtools
+  version_requirements: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id003
+  type: :runtime
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  name: rake
+  version_requirements: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id004
+  type: :runtime
 description: The Infinite Monkeywrench (IMW) is a Ruby frameworks to simplify the tasks of acquiring, extracting, transforming, loading, and packaging data. It minimizes programmer time by encapsulating common data workflows and patterns and creating interfaces to many other useful Ruby libraries.
 email: coders@infochimps.org
 executables: 
-- tsv_to_json.rb
 - imw
+- tsv_to_json.rb
 extensions: []
 
 extra_rdoc_files: 
 - LICENSE
 - README.rdoc
 files: 
-- .gitignore
 - Gemfile
 - Gemfile.lock
 - LICENSE
@@ -99,13 +153,7 @@ files:
 - lib/imw/utils.rb
 - lib/imw/utils/dynamically_extendable.rb
 - lib/imw/utils/error.rb
-- lib/imw/utils/extensions.rb
-- lib/imw/utils/extensions/array.rb
-- lib/imw/utils/extensions/hash.rb
 - lib/imw/utils/extensions/hpricot.rb
-- lib/imw/utils/extensions/string.rb
-- lib/imw/utils/extensions/struct.rb
-- lib/imw/utils/extensions/symbol.rb
 - lib/imw/utils/has_uri.rb
 - lib/imw/utils/log.rb
 - lib/imw/utils/misc.rb
@@ -195,8 +243,8 @@ homepage: http://github.com/infochimps/imw
 licenses: []
 
 post_install_message: 
-rdoc_options: 
-- --charset=UTF-8
+rdoc_options: []
+
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -225,52 +273,52 @@ signing_key:
 specification_version: 3
 summary: The Infinite Monkeywrench (IMW) makes acquiring, extracting, transforming, loading, and packaging data easy.
 test_files: 
-- spec/imw/archives/targz_spec.rb
+- examples/dataset.rb
+- spec/imw/archives/rar_spec.rb
 - spec/imw/archives/tar_spec.rb
-- spec/imw/archives/zip_spec.rb
 - spec/imw/archives/tarbz2_spec.rb
-- spec/imw/archives/rar_spec.rb
-- spec/imw/tools/archiver_spec.rb
-- spec/imw/tools/extension_analyzer_spec.rb
-- spec/imw/tools/summarizer_spec.rb
-- spec/imw/tools/transferer_spec.rb
-- spec/imw/tools/aggregator_spec.rb
-- spec/imw/compressed_files/compressible_spec.rb
+- spec/imw/archives/targz_spec.rb
+- spec/imw/archives/zip_spec.rb
+- spec/imw/archives_spec.rb
 - spec/imw/compressed_files/bz2_spec.rb
+- spec/imw/compressed_files/compressible_spec.rb
 - spec/imw/compressed_files/gz_spec.rb
+- spec/imw/compressed_files_spec.rb
 - spec/imw/dataset/paths_spec.rb
 - spec/imw/dataset/workflow_spec.rb
-- spec/imw/compressed_files_spec.rb
-- spec/imw/parsers/line_parser_spec.rb
-- spec/imw/parsers/regexp_parser_spec.rb
-- spec/imw/resource_spec.rb
-- spec/imw/metadata_spec.rb
+- spec/imw/formats/delimited_spec.rb
 - spec/imw/formats/excel_spec.rb
 - spec/imw/formats/json_spec.rb
-- spec/imw/formats/yaml_spec.rb
-- spec/imw/formats/delimited_spec.rb
 - spec/imw/formats/sgml_spec.rb
-- spec/imw/archives_spec.rb
-- spec/imw/metadata/schema_spec.rb
+- spec/imw/formats/yaml_spec.rb
 - spec/imw/metadata/contains_metadata_spec.rb
+- spec/imw/metadata/field_spec.rb
 - spec/imw/metadata/has_metadata_spec.rb
 - spec/imw/metadata/has_summary_spec.rb
-- spec/imw/metadata/field_spec.rb
+- spec/imw/metadata/schema_spec.rb
+- spec/imw/metadata_spec.rb
+- spec/imw/parsers/line_parser_spec.rb
+- spec/imw/parsers/regexp_parser_spec.rb
+- spec/imw/resource_spec.rb
+- spec/imw/schemes/hdfs_spec.rb
 - spec/imw/schemes/http_spec.rb
 - spec/imw/schemes/local_spec.rb
-- spec/imw/schemes/sql_spec.rb
 - spec/imw/schemes/remote_spec.rb
-- spec/imw/schemes/hdfs_spec.rb
 - spec/imw/schemes/s3_spec.rb
+- spec/imw/schemes/sql_spec.rb
+- spec/imw/tools/aggregator_spec.rb
+- spec/imw/tools/archiver_spec.rb
+- spec/imw/tools/extension_analyzer_spec.rb
+- spec/imw/tools/summarizer_spec.rb
+- spec/imw/tools/transferer_spec.rb
+- spec/imw/utils/dynamically_extendable_spec.rb
 - spec/imw/utils/has_uri_spec.rb
 - spec/imw/utils/paths_spec.rb
-- spec/imw/utils/dynamically_extendable_spec.rb
 - spec/imw/utils/shared_paths_spec.rb
 - spec/imw_spec.rb
 - spec/spec_helper.rb
-- spec/support/without_regard_to_order_matcher.rb
 - spec/support/custom_matchers.rb
-- spec/support/random.rb
-- spec/support/paths_matcher.rb
 - spec/support/file_contents_matcher.rb
-- examples/dataset.rb
+- spec/support/paths_matcher.rb
+- spec/support/random.rb
+- spec/support/without_regard_to_order_matcher.rb

data/.gitignore

@@ -1,18 +0,0 @@
-\#*
-.#*
-*~
-.bzr
-doc/*
-.hg
-Icon?
-*.org
-.project
-.svn
-TAGS
-tmp/*
-*.tmproj
-pkg/*
-*gemspec
-tags
-.yardoc/*
-*/.yardoc/*

data/lib/imw/utils/extensions/array.rb

@@ -1,114 +0,0 @@
-# class Array #:nodoc:
-#   include ActiveSupport::CoreExtensions::Array::ExtractOptions
-# end
-
-class Array
-
-  # Return all but the last  element
-  # This will be [] for both an empty array and a length-1 array
-  def most() self[0..-2] end
-
-  # Return all but the first element.
-  # This will be nil for an empty array and [] for a length-1 array
-  def rest() self[1..-1] end
-
-  # 'Un'-zip()s an array.  Returns an array of arrays: the first array has the
-  # first element of each member, the second array has the second element of
-  # each member, and so on.  Returns as many arrays as the first element in self
-  # and inserts a nil where the member array wasn't long enough.
-  #
-  # foo, bar = foo.zip(bar).unzip should leave foo and bar with the same values
-  # if foo and bar have the same length.
-  #
-  # Will fail on a not-array-of-arrays.
-  def unzip()
-    # An array of empty arrays, one for each vertical slot
-    vslices = self[0].map{ Array.new }
-    self.each do |hslice|
-      # push the elements of each array onto its slice.
-      vslices.zip(hslice).map{|vslice,h_el| vslice << h_el }
-    end
-    vslices
-  end
-
-  # Return a random element of this array.
-  def random_element
-    self[rand(self.length) - 1]
-  end
-
-  # convert an assoc (list of [key, val, [...]]'s) to a hash
-  def to_openstruct
-    mapped = {}
-    each{ |key,value| mapped[key] = value.to_openstruct }
-    OpenStruct.new(mapped)
-  end
-
-  # Return the elements of this array in a pretty-printed string,
-  # inserting +final_string+ between the last two items.
-  #
-  #   >> [:one, :two, :three].quote_items_with "or"
-  #   `one', `two', or `three'
-  #
-  def quote_items_with final_string = nil
-    string_items = self.map { |item| "`" + item.to_s + "'" }
-    case string_items.length
-    when 0
-      ""
-    when 1
-      string_items.first
-    when 2
-      if final_string then
-        string_items.join(" #{final_string} ")
-      else
-        string_items.join(', ')
-      end
-    else
-      string = string_items[0,string_items.length - 1].join ', '
-      if final_string then
-        string += ', ' + final_string + ' ' + string_items.last
-      else
-        string += ', ' + string_items.last
-      end
-      string
-    end
-  end
-
-  def in_groups_of(number, fill_with = nil, &block)
-    require 'enumerator'
-    collection = dup
-    collection << fill_with until collection.size.modulo(number).zero?
-    collection.each_slice(number, &block)
-  end
-
-  # Returns a single hash containing the merge of all hashes in this
-  # array.  This is useful when dealing with badly written YAML files.
-  # Only merges hashes at depth zero, i.e. - this isn't recursive.
-  def merge_hashes
-    merged_hash = {}
-    self.each do |element|
-      merged_hash.merge!(element) if element.is_a?(Hash)
-    end
-    merged_hash
-  end
-
-  # Recurses through the elements of this Array collecting all String
-  # or Symbol "terminal" nodes.
-  def terminals &block
-    terminals = []
-    each do |element|
-      if element.respond_to? :terminals then
-        terminals += element.terminals
-      else
-        terminals << element
-      end
-    end
-    terminals.map! {|terminal| yield terminal } if block
-    terminals
-  end
-
-  # Emit the data in this array to the resource at the given +uri+.
-  def emit uri
-    IMW.open!(uri).emit(self)
-  end
-
-end

data/lib/imw/utils/extensions/hash.rb

@@ -1,206 +0,0 @@
-class Hash
-  # Return the elements of this hash in a pretty-printed string,
-  # inserting +final_string+ between the last two items.
-  #
-  #   >> {:one => 1, :two => 2, :three => 3}.quote_keys_with "or"
-  #   `one', `two', or `three'
-  #
-  def quote_keys_with final_string = nil
-    self.keys.quote_items_with final_string
-  end
-
-  # Stolen from ActiveSupport::CoreExtensions::Hash::ReverseMerge.
-  def reverse_merge(other_hash)
-    other_hash.merge(self)
-  end
-
-  # Stolen from ActiveSupport::CoreExtensions::Hash::ReverseMerge.
-  def reverse_merge!(other_hash)
-    replace(reverse_merge(other_hash))
-  end
-
-  # Create a hash from an array of keys and corresponding values.
-  def self.zip(keys, values, default=nil, &block)
-    hash = block_given? ? Hash.new(&block) : Hash.new(default)
-    keys.zip(values) { |k,v| hash[k]=v }
-    hash
-  end
-
-  # Turns a collection of pairs into a hash.  The first of each pair
-  # make the keys and the second the values. Elements with length
-  # longer than two will lose those values.
-  #
-  # If there are multiple values of
-  #
-  def from_pairs()
-    hsh = { }
-    self.each{ |k,v| hsh[k] = v }
-    hsh
-  end
-
-  # Merges self with another hash, recursively.
-  #
-  # first  = {
-  #   :balls=> "monkey",
-  #   :data=> {
-  #     :name=> {:first=> "Sam", :middle=>"I", :last=>"am"}}}
-  # second = {
-  #   :data=> {
-  #     :name=> {:middle=>["you", "me", "everyone we know"], :last => "are"}},
-  #   1    => [1,2,5] }
-  #
-  # p first.deep_merge(second)
-  # # => {:data=>{:name=>{:last=>"are", :middle=>["you", "me", "everyone we know"], :first=>"Sam"}}, 1=>[1, 2, 5], :balls=>"monkey"}
-  # from http://snippets.dzone.com/posts/show/4706
-  # From: http://pastie.textmate.org/pastes/30372, Elliott Hird
-  def deep_merge(second)
-    target = dup
-    second.keys.each do |key|
-      if second[key].is_a?(Hash) && self[key].is_a?(Hash)
-        target[key] = target[key].deep_merge(second[key])
-      else
-        target[key] = second[key]
-      end
-    end
-    target
-  end
-
-  # Merges self in-place with another hash, recursively.
-  #
-  # first  = {
-  #   :balls=> "monkey",
-  #   :data=> {
-  #     :name=> {:first=> "Sam", :middle=>"I", :last=>"am"}}}
-  # second = {
-  #   :data=> {
-  #     :name=> {:middle=>["you", "me", "everyone we know"], :last => "are"}},
-  #   1    => [1,2,5] }
-  #
-  # p first.deep_merge(second)
-  # # => {:data=>{:name=>{:last=>"are", :middle=>["you", "me", "everyone we know"], :first=>"Sam"}}, 1=>[1, 2, 5], :balls=>"monkey"}
-  #
-  # From: http://www.gemtacular.com/gemdocs/cerberus-0.2.2/doc/classes/Hash.html
-  # File lib/cerberus/utils.rb, line 42
-  def deep_merge!(second)
-    second.keys.each do |key|
-      if second[key].is_a?(Hash) && self[key].is_a?(Hash)
-        self[key].deep_merge!(second[key])
-      else
-        self[key] = second[key]
-      end
-    end
-    self
-  end
-
-  # Merge another array with this one, accumulating values that appear in both
-  # into arrays.
-  #
-  # Note: array values will be flatten'ed. Sorry.
-  #
-  # first  = {
-  #   :balls=> "monkey",
-  #   :data=> {
-  #     :name=> {:first=> "Sam", :middle=>"I", :last=>"am"}}}
-  # second = {
-  #   :data=> {
-  #     :name=> {:middle=>["you", "me", "everyone we know"], :last => "are"}},
-  #   1    => [1,2,5] }
-  #
-  # p first.deep_merge(second)
-  # # => {:data=>{:name=>{:last=>"are",         :middle=>["you", "me", "everyone we know"],      :first=>"Sam"}}, 1=>[1, 2, 5], :balls=>"monkey"}
-  # p first.keep_merge(second)
-  # # => {:data=>{:name=>{:last=>["am", "are"], :middle=>["I", "you", "me", "everyone we know"], :first=>"Sam"}}, 1=>[1, 2, 5], :balls=>"monkey"}
-  def keep_merge(second)
-    target = dup
-    second.each do |key, val2|
-      if second[key].is_a?(Hash) && self[key].is_a?(Hash)
-        target[key] = target[key].keep_merge(val2)
-      else
-        target[key] = target.include?(key) ? [target[key], val2].flatten.uniq : val2
-      end
-    end
-    target
-  end
-
-  # This is polymorphic to Array#assoc -- that is, it allows you treat a Hash
-  # and an array of pairs equivalently using assoc(). We remind you that Array#assoc
-  #
-  #   "Searches through an array whose elements are also arrays comparing obj
-  #    with the first element of each contained array using obj.== . Returns the
-  #    first contained array that matches (that is, the first associated array)
-  #    or nil if no match is found. See also Array#rassoc."
-  #
-  # Note that this returns an /array/ of [key, val] pairs.
-  def assoc(key)
-    self.include?(key)   ? [key, self[key]] : nil
-  end
-  def rassoc(key)
-    self.has_value?(key) ? [key, self[key]] : nil
-  end
-
-  # Allows loading ostruct directly from YAML
-  def to_openstruct
-    map{ |el| el.to_openstruct }
-  end
-
-
-  # Slice a hash to include only the given keys. This is useful for
-  # limiting an options hash to valid keys before passing to a method:
-  #
-  #   def search(criteria = {})
-  #     assert_valid_keys(:mass, :velocity, :time)
-  #   end
-  #
-  #   search(options.slice(:mass, :velocity, :time))
-  # Returns a new hash with only the given keys.
-  def slice(*keys)
-    require 'set'
-    allowed = Set.new(respond_to?(:convert_key) ? keys.map { |key| convert_key(key) } : keys)
-    reject { |key,| !allowed.include?(key) }
-  end
-
-  # Replaces the hash with only the given keys.
-  def slice!(*keys)
-    replace(slice(*keys))
-  end
-
-  # remove all key-value pairs where the value is nil
-  def compact
-    reject{|k,v| v.nil? }
-  end
-
-  # Replaces the hash with its compacted self
-  def compact!
-    replace(compact)
-  end
-
-  # Works like <tt>Enumerable::find</tt> but loops over the keys of
-  # this Hash instead of of arrays of [key,value] and, in the absence
-  # of a matching key, doesn't call the default argument, merely
-  # returns it instead.
-  def dispatch(default=nil, &block)
-    match = self.keys.find(nil,&block)
-    match ? self[match] : default
-  end
-
-
-  # Recurses through the pairs of this Hash collecting all String or
-  # Symbol "terminal" nodes.
-  def terminals &block
-    terminals = []
-    each_value do |value|
-      if value.respond_to? :terminals then
-        terminals += value.terminals
-      else
-        terminals << value
-      end
-    end
-    terminals.map! {|terminal| yield terminal } if block
-    terminals
-  end
-
-  # Emit the data from this Hash into the given +uri+.
-  def emit uri
-    IMW.open!(uri).emit(self)
-  end
-end

data/lib/imw/utils/extensions/string.rb

@@ -1,39 +0,0 @@
-class String
-
-  # Does the string end with the specified +suffix+ (stolen from
-  # <tt>ActiveSupport::CoreExtensions::String::StartsEndsWith</tt>)?
-  def ends_with?(suffix)
-    suffix = suffix.to_s
-    self[-suffix.length, suffix.length] == suffix
-  end
-
-  # Does the string start with the specified +prefix+ (stolen from
-  # <tt>ActiveSupport::CoreExtensions::String::StartsEndsWith</tt>)?
-  def starts_with?(prefix)
-    prefix = prefix.to_s
-    self[0, prefix.length] == prefix
-  end
-
-  # # Downcases a string and replaces spaces with underscores.  This
-  # # works slightly differently than
-  # # <tt>ActiveSupport::CoreExtensions::String::Inflections.underscore</tt>
-  # # which is intended to be used for camel-cased Ruby constants.
-  # #
-  # #   "A long and unwieldy phrase".underscore #=> "a_long_and_unwieldy_phrase"
-  # def underscore
-  #   self.to_s.tr("-", "_").tr(" ","_").downcase
-  # end
-
-  # Returns the handle corresponding to this string as a symbol:
-  #
-  #   "A possible title of a dataset".handle #=> :a_possible_title_of_a_dataset
-  def to_handle
-    self.downcase.underscore.to_sym
-  end
-
-  # Emit this string into the given +uri+.
-  def emit uri
-    IMW.open!(uri).emit(self)
-  end
-
-end

data/lib/imw/utils/extensions/struct.rb

@@ -1,42 +0,0 @@
-Struct.class_eval do
-  def slice *attrs
-    hsh = {}
-    attrs.each{|attr| hsh[attr] = self.send(attr) }
-    hsh
-  end
-
-  def to_hash
-    slice(*self.class.members)
-  end
-  def self.from_hash(hsh)
-    self.new *hsh.values_at(*self.members.map(&:to_sym))
-  end
-
-
-  #
-  # values_at like a hash
-  #
-  def values_of *attrs
-    slice(*attrs).values_at(*attrs)
-  end
-  def each_pair *args, &block
-    self.to_hash.each_pair(*args, &block)
-  end
-
-  def merge *args
-    self.dup.merge! *args
-  end
-  def merge! hashlike, &block
-    raise "can't handle block arg yet" if block
-    hashlike.each_pair{|k,v| self[k] = v }
-    self
-  end
-  alias_method :update, :merge!
-  def indifferent_merge  *args, &block
-    self.dup.indifferent_merge! *args
-  end
-  def indifferent_merge! hashlike, &block
-    merge! hashlike.reject{|k,v| ! self.members.include?(k.to_s) }
-  end
-
-end

data/lib/imw/utils/extensions/symbol.rb

@@ -1,15 +0,0 @@
-class Symbol
-
-  # Turn the symbol into a simple proc (stolen from
-  # <tt>ActiveSupport::CoreExtensions::Symbol</tt>).
-  def to_proc
-    Proc.new { |*args| args.shift.__send__(self, *args) }
-  end
-
-  # Returns the symbol itself (for compatibility with
-  # <tt>String.uniqnae</tt> and so on.
-  def handle
-    self
-  end
-
-end

data/lib/imw/utils/extensions.rb

@@ -1,63 +0,0 @@
-require 'imw/utils/extensions/string'
-require 'imw/utils/extensions/array'
-require 'imw/utils/extensions/hash'
-require 'imw/utils/extensions/struct'
-require 'imw/utils/extensions/symbol'
-
-module IMW
-  # A replacement for the standard system call which raises an
-  # IMW::SystemCallError if the command fails which prints better
-  # debugging info.
-  #
-  # This function relies upon Kernel.system and obeys the same rules:
-  #
-  # - if +commands+ has only only a single element then no shell
-  #   characters or spaces are escaped -- you have to do it yourself
-  #   or you get to use shell characters, depending on your
-  #   perspective.
-  #
-  # - if +commands+ is a list of elements then the second and further
-  #   elements in the list have their shell characters and spaces
-  #   escaped
-  #
-  # But it also has its own rules:
-  #
-  # - When one of the +commands+ is an empty or blank string,
-  #   Kernel.system honors it and escapes it properly and sends it
-  #   along for evaluation.  This can be a problem for some programs
-  #   and so IMW.system excludes blank (as in <tt>blank?</tt>)
-  #   elements of +commands+.
-  #
-  # - +commands+ will be flattened (see the gotcha below)
-  #
-  # Calling out to the shell like this is often brittle.  Imagine
-  # defining
-  #
-  #   prog  = 'some_prog'
-  #   flags = '-v -f'
-  #   args  = 'file.txt'
-  #
-  # and later calling
-  # 
-  #   IMW.system prog, flags, args
-  #
-  # The space in the second argument ('-v -f') will be escaped and
-  # will therefore not be properly parsed by +some_prog+.  Instead try
-  #
-  #   prog  = 'some_prog'
-  #   flags = ['-v', '-f']
-  #   args = ['file.txt']
-  #   
-  #   IMW.system prog, flags, *args
-  #
-  # which will work fine since +flags+ will automatically be flattend.
-  def self.system *commands
-    stripped_commands = commands.flatten.map { |command| command.to_s unless command.blank? }.compact
-    IMW.announce_if_verbose(stripped_commands.join(" "))
-    exit_code = Kernel.system(*stripped_commands)
-    raise IMW::SystemCallError.new($?.dup, commands.join(' ')) unless $?.success?
-    exit_code
-  end
-end
-
-