masticate 0.1.1 → 0.1.3

data/bin/masticate

@@ -13,11 +13,15 @@ OptionParser.new do |opts|
     options[:format] = v
   end
 
-  opts.on("--delim DELIMITER", "Specify field delimiter (character or TAB)") do |v|
+  opts.on("--delim DELIMITER", "Specify field delimiter (character or TAB; default is ',')") do |v|
     options[:col_sep] = v
     options[:col_sep] = "\t" if options[:col_sep] == "TAB"
   end
 
+  opts.on("--quote QUOTE-CHAR", "Specify character used for quoting fields (optional; default is no quoting)") do |char|
+    options[:quote_char] = char
+  end
+
   opts.on("--fields LIST", Array, "Specify fields to select") do |list|
     options[:fields] = list
   end
@@ -41,6 +45,10 @@ OptionParser.new do |opts|
   opts.on("--inlined", "(for *mend* only) Source file has headers inlined on each line") do |v|
     options[:inlined] = v
   end
+
+  opts.on("--dejunk", "(for *mend* only) Expunge junk lines from source") do |v|
+    options[:dejunk] = v
+  end
 end.parse!
 
 filename = ARGV.shift # use stdin if no filename provided
@@ -51,6 +59,9 @@ def logmessage(command, options, results)
   Lines in input: #{results[:input_count]}
   Lines in output: #{results[:output_count]}
 EOT
+  if results[:field_counts]
+    $stderr.puts "  Field counts: #{results[:field_counts].inspect}"
+  end
 end
 
 case command

data/lib/masticate/mender.rb

@@ -18,6 +18,7 @@ class Masticate::Mender < Masticate::Base
     expected_field_count = nil
     headers = nil
     @output_count = 0
+    fieldcounts = Hash.new(0)
     with_input do |input|
       while (line = get) do
         unless line =~ /^\s*$/
@@ -27,13 +28,15 @@ class Masticate::Mender < Masticate::Base
             if !expected_field_count
               headers = row[0..ncells]
               expected_field_count = headers.count
+              fieldcounts[headers.count] += 1
               emit(headers.to_csv(:col_sep => @col_sep))
             else
               if row[0..ncells] != headers
                 raise "Header mismatch on line #{@input_count}\n  Expected: #{headers.join(',')}\n     Found: #{row[0..ncells].join(',')}"
               end
             end
-            row = row[ncells+1..-1]
+            row = row[ncells+1, expected_field_count]
+            fieldcounts[row.count] += 1
             emit(row.to_csv(:col_sep => @col_sep))
           elsif !expected_field_count
             # trust the first row
@@ -49,6 +52,7 @@ class Masticate::Mender < Masticate::Base
               raise "Do not understand snip instruction [#{opts[:snip].inspect}]"
             end
             expected_field_count = headers.count
+            fieldcounts[headers.count] += 1
             emit(headers.to_csv(:col_sep => @col_sep))
           else
             running_count = fieldcount(line)
@@ -61,6 +65,7 @@ class Masticate::Mender < Masticate::Base
             end
 
             unless opts[:dejunk] && junky?(line)
+              fieldcounts[fieldcount(line)] += 1
               emit(line)
             end
           end
@@ -72,6 +77,7 @@ class Masticate::Mender < Masticate::Base
     {
       :input_count => @input_count,
       :output_count => @output_count,
+      :field_counts => fieldcounts,
       :headers => headers
     }
   end
@@ -81,7 +87,7 @@ class Masticate::Mender < Masticate::Base
   end
 
   def explode(line)
-    CSV.parse_line(line, :col_sep => col_sep, :quote_char => @quote_char)
+    CSV.parse_line(line, :col_sep => @col_sep, :quote_char => @quote_char)
   end
 
   # a line is "junky" if it has 2 or fewer fields with any content

data/lib/masticate/version.rb

@@ -1,3 +1,3 @@
 module Masticate
-  VERSION = "0.1.1"
+  VERSION = "0.1.3"
 end

data/spec/data/inlined_headers.csv

@@ -4,7 +4,7 @@ LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NA
 LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,ADAMS,JOHN QUINCY,A,209,8637,02/24/2003,12/02/2007,TM,Imaging Svcs - MRI,F,11/03/1966,3/10/2012
 LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,HAMILTON,ANDREW,,278,10065,01/09/2007,11/16/2007,TM,Information Technology,M,09/16/1968,3/10/2012
 LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,MADISON,JAMES,F,672,10720,01/05/2009,02/16/2010,TM,Rehab Svcs - Outpatients,F,04/15/1985,3/10/2012
-LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,FRANKLIN,BENJAMIN,R,674,8340,05/01/2002,09/01/2003,TM,Rehab Svcs - PRN,F,03/15/1973,3/10/2012
+LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,FRANKLIN,BENJAMIN,R,674,8340,05/01/2002,09/01/2003,TM,"Rehab Svcs, xyz",F,03/15/1973,3/10/2012
 LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,LINCOLN,ABRAHAM,M,634,11340,05/02/2011,,PN,Sibley Ambulatory Surgery Ctr,F,07/11/1960,3/10/2012
 LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,MONROE,JAMES,L,614,10757,02/16/2009,,RF,Labor & Delivery,F,11/06/1983,3/10/2012
 LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,REVERE,PAUL,B,424,8568,11/18/2002,06/27/2006,TM,Laundry & Linen,M,12/31/1976,3/10/2012

data/spec/data/inlined_headers.csv.output

@@ -1,11 +1,11 @@
 LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE
-WASHINGTON,GEORGE,D,824,9556,09/10/2005,07/01/2006,TM,Surgical House Staff,M,09/23/1975,3/10/2012
-JEFFERSON,TOM,,621,8052,07/23/2001,01/28/2011,TM,Telemetry,F,12/24/1976,3/10/2012
-ADAMS,JOHN,,655,8834,09/22/2003,,WA,6 East,F,08/07/1978,3/10/2012
-ADAMS,JOHN QUINCY,A,209,8637,02/24/2003,12/02/2007,TM,Imaging Svcs - MRI,F,11/03/1966,3/10/2012
-HAMILTON,ANDREW,,278,10065,01/09/2007,11/16/2007,TM,Information Technology,M,09/16/1968,3/10/2012
-MADISON,JAMES,F,672,10720,01/05/2009,02/16/2010,TM,Rehab Svcs - Outpatients,F,04/15/1985,3/10/2012
-FRANKLIN,BENJAMIN,R,674,8340,05/01/2002,09/01/2003,TM,Rehab Svcs - PRN,F,03/15/1973,3/10/2012
-LINCOLN,ABRAHAM,M,634,11340,05/02/2011,,PN,Sibley Ambulatory Surgery Ctr,F,07/11/1960,3/10/2012
-MONROE,JAMES,L,614,10757,02/16/2009,,RF,Labor & Delivery,F,11/06/1983,3/10/2012
-REVERE,PAUL,B,424,8568,11/18/2002,06/27/2006,TM,Laundry & Linen,M,12/31/1976,3/10/2012
+WASHINGTON,GEORGE,D,824,9556,09/10/2005,07/01/2006,TM,Surgical House Staff,M,09/23/1975
+JEFFERSON,TOM,,621,8052,07/23/2001,01/28/2011,TM,Telemetry,F,12/24/1976
+ADAMS,JOHN,,655,8834,09/22/2003,,WA,6 East,F,08/07/1978
+ADAMS,JOHN QUINCY,A,209,8637,02/24/2003,12/02/2007,TM,Imaging Svcs - MRI,F,11/03/1966
+HAMILTON,ANDREW,,278,10065,01/09/2007,11/16/2007,TM,Information Technology,M,09/16/1968
+MADISON,JAMES,F,672,10720,01/05/2009,02/16/2010,TM,Rehab Svcs - Outpatients,F,04/15/1985
+FRANKLIN,BENJAMIN,R,674,8340,05/01/2002,09/01/2003,TM,"Rehab Svcs, xyz",F,03/15/1973
+LINCOLN,ABRAHAM,M,634,11340,05/02/2011,,PN,Sibley Ambulatory Surgery Ctr,F,07/11/1960
+MONROE,JAMES,L,614,10757,02/16/2009,,RF,Labor & Delivery,F,11/06/1983
+REVERE,PAUL,B,424,8568,11/18/2002,06/27/2006,TM,Laundry & Linen,M,12/31/1976

data/spec/data/junk_header.csv

@@ -1,5 +1,5 @@
 3/7/2012,hospid,usrorder,dteorder,usrsend,dtesend,usrdone,dtedone,department
 15267,407,201201060140,407,201201060140,0,201201060309,L
-15267,381,201201060222,381,201201060222,0,201201060647,X
+15267,381,201201060222,381,201201060222,"abc,def",201201060647,X
 15267,407,201201060311,407,201201060311,0,201201060339,L
 15267,407,201201060514,108,201201060515,108,201201060515,SEC

data/spec/lib/mender_spec.rb

@@ -30,12 +30,13 @@ describe "mending" do
   it "should unfold inlined headers" do
     filename = File.dirname(__FILE__) + "/../data/inlined_headers.csv"
     tmp = Tempfile.new('mending')
-    results = Masticate.mend(filename, :inlined => true, :output => tmp)
+    results = Masticate.mend(filename, :col_sep => ',', :quote_char => '"', :inlined => true, :output => tmp)
     output = File.read(tmp)
     correct_output = File.read(File.dirname(__FILE__) + "/../data/inlined_headers.csv.output")
 
     results[:input_count].should == 11
     results[:output_count].should == 11
+    results[:field_counts].should == {11 => 11}
     output.should == correct_output
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: masticate
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2152293880 !ruby/object:Gem::Requirement
+  requirement: &2152447240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 2.9.0
   type: :development
   prerelease: false
-  version_requirements: *2152293880
+  version_requirements: *2152447240
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &2152293360 !ruby/object:Gem::Requirement
+  requirement: &2152446740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 0.7.0
   type: :development
   prerelease: false
-  version_requirements: *2152293360
+  version_requirements: *2152446740
 - !ruby/object:Gem::Dependency
   name: ruby_gntp
-  requirement: &2152292900 !ruby/object:Gem::Requirement
+  requirement: &2152446280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,7 +43,7 @@ dependencies:
         version: 0.3.4
   type: :development
   prerelease: false
-  version_requirements: *2152292900
+  version_requirements: *2152446280
 description: Data file crunching
 email:
 - jmay@pobox.com