masticate 0.4.0 → 0.4.1

data/lib/masticate/datify.rb

@@ -1,5 +1,4 @@
 # convert date columns to numerics
-require "csv"
 
 class Masticate::Datify < Masticate::Base
   def configure(opts)
@@ -20,8 +19,7 @@ class Masticate::Datify < Masticate::Base
         @index = row.index(@field) or raise "Unable to find column '#{@field}'"
       end
     elsif row
-      ts = DateTime.strptime(row[@index], @format).to_time
-      row[@index] = ts.to_i rescue nil
+      row[@index] = DateTime.strptime(row[@index], @format).to_time.to_i rescue nil
     end
     row
   end

data/lib/masticate/mender.rb

@@ -10,6 +10,7 @@ class Masticate::Mender < Masticate::Base
     @inlined = opts[:inlined]
     @snip = opts[:snip]
     @dejunk = opts[:dejunk]
+    @buried = opts[:buried]
 
     @expected_field_count = nil
     @holding = ''
@@ -47,6 +48,15 @@ class Masticate::Mender < Masticate::Base
       else
         raise "Do not understand snip instruction [#{@snip.inspect}]"
       end
+
+      if @buried
+        if @buried.is_a?(Fixnum) || @buried =~ /^\d+/
+          @buried_index = @buried.to_i
+        else
+          @buried_index = row.index(@buried) or raise "Unable to find column '#{@buried}'"
+        end
+      end
+
       @expected_field_count = @headers.count
       row = @headers
     elsif row
@@ -65,6 +75,12 @@ class Masticate::Mender < Masticate::Base
         @holding = ''
       end
 
+      if @buried && (row.count > @expected_field_count)
+        # buried delimiter
+        # take the N+1th field and merge it onto the Nth field, moving up the remaining fields
+        row[@buried_index] += row.delete_at(@buried_index + 1)
+      end
+
       if @dejunk && row && row.select {|s| s && !s.strip.empty?}.count <= 2
         # junky row, suppress output
         nil

data/lib/masticate/version.rb

@@ -1,3 +1,3 @@
 module Masticate
-  VERSION = "0.4.0"
+  VERSION = "0.4.1"
 end

data/masticate.gemspec

@@ -16,8 +16,8 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = Masticate::VERSION
 
-  gem.add_development_dependency "rake", "~> 0.9.2"
-  gem.add_development_dependency "rspec", "~> 2.9.0"
-  gem.add_development_dependency "guard-rspec", "~> 0.7.0"
-  gem.add_development_dependency "ruby_gntp", "~> 0.3.4"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "guard-rspec"
+  gem.add_development_dependency "ruby_gntp"
 end

data/spec/data/buried_delims_in_input.psv

@@ -0,0 +1,4 @@
+LastName|FirstName|Comment|ID|Date
+Washington|George|text without embedded pipes|111|3/13/2012
+Adams|John|text with|embedded pipe|222|4/14/2012
+Jefferson|Thomas|text with no embedded pipe|333|5/15/2012

data/spec/data/delims_untangled.csv

@@ -0,0 +1,4 @@
+LastName,FirstName,Comment,ID,Date
+Washington,George,text without embedded pipes,111,3/13/2012
+Adams,John,text withembedded pipe,222,4/14/2012
+Jefferson,Thomas,text with no embedded pipe,333,5/15/2012

data/spec/lib/mender_spec.rb

@@ -51,4 +51,16 @@ describe "mending" do
     results[:output_count].should == 4
     output.should == correct_output
   end
+
+  it "should consolidate fields with embedded delimiters" do
+    filename = File.dirname(__FILE__) + "/../data/buried_delims_in_input.psv"
+    tmp = Tempfile.new('mending')
+    results = Masticate.mend(filename, :col_sep => '|', :buried => 'Comment', :output => tmp)
+    output = File.read(tmp)
+    correct_output = File.read(File.dirname(__FILE__) + "/../data/delims_untangled.csv")
+
+    results[:input_count].should == 4
+    results[:output_count].should == 4
+    output.should == correct_output
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: masticate
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,72 +9,72 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-21 00:00:00.000000000 Z
+date: 2012-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.9.2
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.9.2
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 2.9.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 2.9.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: guard-rspec
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.7.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.7.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: ruby_gntp
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.3.4
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.3.4
+        version: '0'
 description: Data file crunching
 email:
 - jmay@pobox.com
@@ -111,6 +111,7 @@ files:
 - spec/data/badnums.csv
 - spec/data/badnums_fixed.csv
 - spec/data/broken_psv.txt
+- spec/data/buried_delims_in_input.psv
 - spec/data/concat_result.txt
 - spec/data/cookery_input.psv
 - spec/data/cookery_result.csv
@@ -118,6 +119,7 @@ files:
 - spec/data/cooking_result.csv
 - spec/data/datify_input.csv
 - spec/data/datify_result.csv
+- spec/data/delims_untangled.csv
 - spec/data/downcase_results.csv
 - spec/data/events.csv
 - spec/data/events_reduced.csv
@@ -166,7 +168,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -519721259904741395
+      hash: 2222163119454515723
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -175,7 +177,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -519721259904741395
+      hash: 2222163119454515723
 requirements: []
 rubyforge_project: masticate
 rubygems_version: 1.8.24
@@ -186,6 +188,7 @@ test_files:
 - spec/data/badnums.csv
 - spec/data/badnums_fixed.csv
 - spec/data/broken_psv.txt
+- spec/data/buried_delims_in_input.psv
 - spec/data/concat_result.txt
 - spec/data/cookery_input.psv
 - spec/data/cookery_result.csv
@@ -193,6 +196,7 @@ test_files:
 - spec/data/cooking_result.csv
 - spec/data/datify_input.csv
 - spec/data/datify_result.csv
+- spec/data/delims_untangled.csv
 - spec/data/downcase_results.csv
 - spec/data/events.csv
 - spec/data/events_reduced.csv