masticate 0.2.3 → 0.3

data/lib/masticate/base.rb

@@ -60,7 +60,14 @@ class Masticate::Base
     @output_count = 0
     with_input do |input|
       while line = get
-        row = CSV.parse_line(line, csv_options)
+        row = CSV.parse_line(line, csv_options) #.map {|s| s && s.strip}
+        if row
+          row = row.map {|s| s && s.strip}
+        end
+        # row2 = row.map {|s| s && s.strip}
+        # if row2.nil?
+        #   puts "**** ROW IS [#{row.inspect}]"
+        # end
         output = crunch(row)
         emit(output) if output
       end
@@ -70,7 +77,8 @@ class Masticate::Base
 
     {
       :input_count => input_count,
-      :output_count => @output_count
+      :output_count => @output_count,
+      :headers => @headers
     }
   end
 end

data/lib/masticate/cook.rb

@@ -19,7 +19,6 @@ class Masticate::Cook < Masticate::Base
     standard_options(opts)
 
     steps = recipe.map do |step|
-      # puts step
       argv = Shellwords.split(step)
       masticator = Masticate::MyOptionParser.new
       command, options = masticator.parse(argv)

data/lib/masticate/exclude.rb

@@ -19,7 +19,19 @@ class Masticate::Exclude < Masticate::Base
   def crunch(row)
     if !@headers
       @headers = row
-      @index = @headers.index(@field) or raise "Unable to find column '#{@field}' in headers"
+      f = @field
+      @index =
+        case f
+        when Fixnum, /^\d+$/
+          f = f.to_i
+          if f > row.count
+            raise "Cannot pluck column #{f}, there are only #{row.count} fields"
+          else
+            f-1
+          end
+        else
+          row.index(f) or raise "Unable to find column '#{f}' in headers"
+        end
       row
     elsif row
       if row[@index] == @value

data/lib/masticate/mender.rb

@@ -4,13 +4,77 @@
 # (due to a newline embedded in a field).  Glue those two lines into a single line in the output.
 
 class Masticate::Mender < Masticate::Base
-  attr_reader :col_sep
+  def configure(opts)
+    standard_options(opts)
 
-  def initialize(filename)
-    @filename = filename
+    @inlined = opts[:inlined]
+    @snip = opts[:snip]
+    @dejunk = opts[:dejunk]
+
+    @expected_field_count = nil
+    @holding = []
   end
 
+  # attr_reader :col_sep
+
+  # def initialize(filename)
+  #   @filename = filename
+  # end
+
   def mend(opts)
+    execute(opts)
+  end
+
+  def crunch(row)
+    if @inlined
+      if row
+        ncells = row.count/2-1
+        if !@headers
+          @headers = row[0..ncells]
+          @expected_field_count = @headers.count
+          emit(@headers)
+        else
+          if row[0..ncells] != @headers
+            raise "Header mismatch on line #{@input_count}\n  Expected: #{@headers.join(',')}\n     Found: #{row[0..ncells].join(',')}"
+          end
+        end
+        row = row[ncells+1, @expected_field_count]
+      end
+    elsif !@headers
+      # trust the first row
+      @headers = row
+      case @snip
+      when Fixnum
+        @headers.shift(@snip)
+      when String
+        raise "TODO: snip named header. Multiple?"
+      when nil
+        # do nothing
+      else
+        raise "Do not understand snip instruction [#{@snip.inspect}]"
+      end
+      @expected_field_count = @headers.count
+      row = @headers
+    elsif row
+      @holding += row
+      if @holding.count < @expected_field_count
+        # incomplete row; do not emit anything
+        row = nil
+      else
+        row = @holding
+        @holding = []
+      end
+
+      if @dejunk && row && row.select {|s| s && !s.strip.empty?}.count <= 2
+        # junky row, suppress output
+        nil
+      else
+        row
+      end
+    end
+  end
+
+  def old_mend(opts)
     @output = opts[:output] ? File.open(opts[:output], "w") : $stdout
     @col_sep = opts[:col_sep] || ','
     @quote_char = opts[:quote_char] || "\0"

data/lib/masticate/myoptparse.rb

@@ -91,7 +91,8 @@ class Masticate::MyOptionParser
       'maxrows' => Masticate::MaxRows,
       'relabel' => Masticate::Relabel,
       'pluck' => Masticate::Plucker,
-      'exclude' => Masticate::Exclude
+      'exclude' => Masticate::Exclude,
+      'mend' => Masticate::Mender
     }
 
     klass = klasses[command]
@@ -165,6 +166,7 @@ EOT
 * masticate #{command} (#{options.keys.join(', ')})
   Lines in input: #{results[:input_count]}
   Lines in output: #{results[:output_count]}
+  Headers: #{results[:headers].inspect}
 EOT
     if results[:field_counts]
       $stderr.puts "  Field counts: #{results[:field_counts].inspect}"

data/lib/masticate/version.rb

@@ -1,3 +1,3 @@
 module Masticate
-  VERSION = "0.2.3"
+  VERSION = "0.3"
 end

data/spec/data/cooking_mend_result.csv

@@ -0,0 +1,4 @@
+Col 3   
+data  
+this long row
+data  

data/spec/data/datify_input.csv

@@ -0,0 +1,11 @@
+LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,timestamp,Term Date,Status,R_NAME,SEX,BIRTHDATE
+WASHINGTON,GEORGE,D,824,9556,09/10/2005 4:23:16PM,07/01/2006,TM,Surgical House Staff,M,09/23/1975
+JEFFERSON,TOM,,621,8052,07/23/2001 7:23:11AM,01/28/2011,TM,Telemetry,F,12/24/1976
+ADAMS,JOHN,,655,8834,09/22/2003 01:23:45PM,,WA,6 East,F,08/07/1978
+ADAMS,JOHN QUINCY,A,209,8637,02/24/2003 02:34:00AM,12/02/2007,TM,Imaging Svcs - MRI,F,11/03/1966
+HAMILTON,ANDREW,,278,10065,01/09/2007 02:34:00AM,11/16/2007,TM,Information Technology,M,09/16/1968
+MADISON,JAMES,F,672,10720,01/05/2009 02:34:00AM,02/16/2010,TM,Rehab Svcs - Outpatients,F,04/15/1985
+FRANKLIN,BENJAMIN,R,674,8340,05/01/2002 02:34:00AM,09/01/2003,TM,"Rehab Svcs, xyz",F,03/15/1973
+LINCOLN,ABRAHAM,M,634,11340,05/02/2011 02:34:00AM,,PN,Sibley Ambulatory Surgery Ctr,F,07/11/1960
+MONROE,JAMES,L,614,10757,02/16/2009 02:34:00AM,,RF,Labor & Delivery,F,11/06/1983
+REVERE,PAUL,B,424,8568,11/18/2002 02:34:00AM,06/27/2006,TM,Laundry & Linen,M,12/31/1976

data/spec/data/datify_result.csv

@@ -0,0 +1,11 @@
+LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,timestamp,Term Date,Status,R_NAME,SEX,BIRTHDATE
+WASHINGTON,GEORGE,D,824,9556,1126369396,07/01/2006,TM,Surgical House Staff,M,09/23/1975
+JEFFERSON,TOM,,621,8052,995872991,01/28/2011,TM,Telemetry,F,12/24/1976
+ADAMS,JOHN,,655,8834,1064237025,,WA,6 East,F,08/07/1978
+ADAMS,JOHN QUINCY,A,209,8637,1046054040,12/02/2007,TM,Imaging Svcs - MRI,F,11/03/1966
+HAMILTON,ANDREW,,278,10065,1168310040,11/16/2007,TM,Information Technology,M,09/16/1968
+MADISON,JAMES,F,672,10720,1231122840,02/16/2010,TM,Rehab Svcs - Outpatients,F,04/15/1985
+FRANKLIN,BENJAMIN,R,674,8340,1020220440,09/01/2003,TM,"Rehab Svcs, xyz",F,03/15/1973
+LINCOLN,ABRAHAM,M,634,11340,1304303640,,PN,Sibley Ambulatory Surgery Ctr,F,07/11/1960
+MONROE,JAMES,L,614,10757,1234751640,,RF,Labor & Delivery,F,11/06/1983
+REVERE,PAUL,B,424,8568,1037586840,06/27/2006,TM,Laundry & Linen,M,12/31/1976

data/spec/data/recipe_mend.txt

@@ -0,0 +1,3 @@
+mend
+exclude --field 1 --value 'data2'
+pluck --fields 3

data/spec/lib/cook_spec.rb

@@ -13,4 +13,15 @@ describe "cooking up a recipe" do
   
     output.should == correct_output
   end
+
+  it "should allow mend in recipe" do
+    input = File.dirname(__FILE__) + "/../data/broken_psv.txt"
+    recipe = File.dirname(__FILE__) + "/../data/recipe_mend.txt"
+    tmp = Tempfile.new('cooked')
+    results = Masticate.cook(input, :col_sep => '|', :output => tmp, :recipe => recipe)
+    output = File.read(tmp)
+    correct_output = File.read(File.dirname(__FILE__) + "/../data/cooking_mend_result.csv")
+  
+    output.should == correct_output
+  end
 end

data/spec/lib/datify_spec.rb

@@ -2,15 +2,14 @@
 
 require "spec_helper"
 
-describe "csvification" do
-  it "should convert pipes to standard commas" do
-    filename = File.dirname(__FILE__) + "/../data/pipe_data.txt"
-    tmp = Tempfile.new('csvify')
-    results = Masticate.csvify(filename, :output => tmp, :col_sep => '|')
+describe "datification" do
+  it "should transform dates" do
+    filename = File.dirname(__FILE__) + "/../data/datify_input.csv"
+    tmp = Tempfile.new('datify')
+    results = Masticate.datify(filename, :output => tmp, :field => 'timestamp', :format => '%m/%d/%Y %H:%M:%S%p')
     output = File.read(tmp)
     tmp.unlink
-    output.lines.count.should == 5
-    results[:input_count].should == 6
-    results[:output_count].should == 5
+
+    correct_output = File.read(File.dirname(__FILE__) + "/../data/datify_result.csv")
   end
 end

data/spec/lib/exclude_spec.rb

@@ -12,4 +12,14 @@ describe "exclude" do
 
     output.should == correct_output
   end
+
+  it "should be able to exclude rows by number as well as name" do
+    filename = File.dirname(__FILE__) + "/../data/exclude_input.csv"
+    tmp = Tempfile.new('exclude')
+    results = Masticate.exclude(filename, :output => tmp, :field => 1, :value => '')
+    output = File.read(tmp)
+    correct_output = File.read(File.dirname(__FILE__) + "/../data/exclude_results.csv")
+
+    output.should == correct_output
+  end
 end

data/spec/lib/mender_spec.rb

@@ -36,7 +36,7 @@ describe "mending" do
 
     results[:input_count].should == 11
     results[:output_count].should == 11
-    results[:field_counts].should == {11 => 11}
+    # results[:field_counts].should == {11 => 11}
     output.should == correct_output
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: masticate
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: '0.3'
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-25 00:00:00.000000000 Z
+date: 2012-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2156995160 !ruby/object:Gem::Requirement
+  requirement: &2152079220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 2.9.0
   type: :development
   prerelease: false
-  version_requirements: *2156995160
+  version_requirements: *2152079220
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &2156994180 !ruby/object:Gem::Requirement
+  requirement: &2152076120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 0.7.0
   type: :development
   prerelease: false
-  version_requirements: *2156994180
+  version_requirements: *2152076120
 - !ruby/object:Gem::Dependency
   name: ruby_gntp
-  requirement: &2156992960 !ruby/object:Gem::Requirement
+  requirement: &2152074480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,7 +43,7 @@ dependencies:
         version: 0.3.4
   type: :development
   prerelease: false
-  version_requirements: *2156992960
+  version_requirements: *2152074480
 description: Data file crunching
 email:
 - jmay@pobox.com
@@ -79,8 +79,10 @@ files:
 - spec/data/badnums_fixed.csv
 - spec/data/broken_psv.txt
 - spec/data/concat_result.txt
+- spec/data/cooking_mend_result.csv
 - spec/data/cooking_result.csv
-- spec/data/datify_input.txt
+- spec/data/datify_input.csv
+- spec/data/datify_result.csv
 - spec/data/events.csv
 - spec/data/events_reduced.csv
 - spec/data/exclude_input.csv
@@ -94,6 +96,7 @@ files:
 - spec/data/pipe_data.txt
 - spec/data/quoted_csv_data.txt
 - spec/data/recipe.txt
+- spec/data/recipe_mend.txt
 - spec/data/relabel_result.csv
 - spec/data/tabbed_data.txt
 - spec/data/tilde_data.txt
@@ -138,8 +141,10 @@ test_files:
 - spec/data/badnums_fixed.csv
 - spec/data/broken_psv.txt
 - spec/data/concat_result.txt
+- spec/data/cooking_mend_result.csv
 - spec/data/cooking_result.csv
-- spec/data/datify_input.txt
+- spec/data/datify_input.csv
+- spec/data/datify_result.csv
 - spec/data/events.csv
 - spec/data/events_reduced.csv
 - spec/data/exclude_input.csv
@@ -153,6 +158,7 @@ test_files:
 - spec/data/pipe_data.txt
 - spec/data/quoted_csv_data.txt
 - spec/data/recipe.txt
+- spec/data/recipe_mend.txt
 - spec/data/relabel_result.csv
 - spec/data/tabbed_data.txt
 - spec/data/tilde_data.txt

data/spec/data/datify_input.txt

@@ -1,20 +0,0 @@
-ChangeDTTM|AuditEventID|AuditEventCode|AuditEventTypeID|AuditEventName|AuditEventDesc|AuditByID
-12/31/2011   3:41:54PM|57,314,552.00|LOGGEDOUT|30|Logged out|Logged out|4,426.00
-1/8/2012   9:21:23AM|57,486,988.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
-1/8/2012   9:21:24AM|57,486,989.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
-2/19/2012   6:08:59PM|58,597,521.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
-2/19/2012   7:02:50PM|58,597,816.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
-2/19/2012   7:02:50PM|58,597,817.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
-2/4/2012   9:29:20AM|58,201,259.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
-2/4/2012  12:19:30AM|58,196,973.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
-2/4/2012  12:20:53AM|58,196,975.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
-2/3/2012  10:44:25PM|58,196,507.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
-1/20/2012   8:01:07PM|57,833,496.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
-1/22/2012   4:20:48AM|57,845,095.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
-1/22/2012   4:20:49AM|57,845,096.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
-1/14/2012   7:50:40AM|57,661,613.00|LOGGEDOUT|30|Logged out|Logged out|5,019.00
-2/26/2012   7:48:07AM|58,776,243.00|LOGGEDOUT|30|Logged out|Logged out|5,019.00
-2/23/2012   7:47:39AM|58,699,535.00|LOGGEDOUT|30|Logged out|Logged out|5,019.00
-2/23/2012   7:39:56AM|58,699,387.00|LOGGEDOUT|30|Logged out|Logged out|5,019.00
-2/23/2012   7:39:56AM|58,699,388.00|LOGGEDOUT|30|Logged out|Logged out|5,019.00
-2/15/2012   3:15:06PM|58,502,037.00|LOGGEDOUT|30|Logged out|Logged out|5,019.00