masticate 0.3.1 → 0.3.2

data/lib/masticate/base.rb

@@ -70,11 +70,11 @@ class Masticate::Base
         if row
           row = row.map {|s| s && s.strip}
         end
-        # row2 = row.map {|s| s && s.strip}
-        # if row2.nil?
-        #   puts "**** ROW IS [#{row.inspect}]"
-        # end
-        output = crunch(row)
+        if self.class == Masticate::Mender
+          output = crunch(row, line, csv_options)
+        else
+          output = crunch(row)
+        end
         emit(output) if output
       end
     end

data/lib/masticate/cook.rb

@@ -33,7 +33,13 @@ class Masticate::Cook < Masticate::Base
         row = CSV.parse_line(line, csv_options)
 
         steps.each do |step|
-          row = step.crunch(row) if row
+          if row
+            if step.class == Masticate::Mender
+              row = step.crunch(row, line, csv_options)
+            else
+              row = step.crunch(row)
+            end
+          end
         end
 
         emit(row) if row
@@ -48,7 +54,6 @@ class Masticate::Cook < Masticate::Base
       end
     end
     more_rows.each {|row| emit(row)}
-    # step.crunch(nil) {|row| emit(row)}
 
     @output.close if opts[:output]
 

data/lib/masticate/mender.rb

@@ -12,20 +12,14 @@ class Masticate::Mender < Masticate::Base
     @dejunk = opts[:dejunk]
 
     @expected_field_count = nil
-    @holding = []
+    @holding = ''
   end
 
-  # attr_reader :col_sep
-
-  # def initialize(filename)
-  #   @filename = filename
-  # end
-
   def mend(opts)
     execute(opts)
   end
 
-  def crunch(row)
+  def crunch(row, line = '', csv_options = {})
     if @inlined
       if row
         ncells = row.count/2-1
@@ -56,13 +50,19 @@ class Masticate::Mender < Masticate::Base
       @expected_field_count = @headers.count
       row = @headers
     elsif row
-      @holding += row
-      if @holding.count < @expected_field_count
+      @holding << ' ' unless @holding.empty?
+      @holding << line
+
+      row = CSV.parse_line(@holding, csv_options) #.map {|s| s && s.strip}
+      if row
+        row = row.map {|s| s && s.strip}
+      end
+
+      if row.count < @expected_field_count
         # incomplete row; do not emit anything
         row = nil
       else
-        row = @holding
-        @holding = []
+        @holding = ''
       end
 
       if @dejunk && row && row.select {|s| s && !s.strip.empty?}.count <= 2
@@ -74,78 +74,6 @@ class Masticate::Mender < Masticate::Base
     end
   end
 
-  def old_mend(opts)
-    @output = opts[:output] ? File.open(opts[:output], "w") : $stdout
-    @col_sep = opts[:col_sep] || ','
-    @quote_char = opts[:quote_char] || "\0"
-
-    expected_field_count = nil
-    headers = nil
-    @output_count = 0
-    fieldcounts = Hash.new(0)
-    with_input do |input|
-      while (line = get) do
-        unless line =~ /^\s*$/
-          if opts[:inlined]
-            row = explode(line)
-            ncells = row.count/2-1
-            if !expected_field_count
-              headers = row[0..ncells]
-              expected_field_count = headers.count
-              fieldcounts[headers.count] += 1
-              emit(headers.to_csv(:col_sep => @col_sep))
-            else
-              if row[0..ncells] != headers
-                raise "Header mismatch on line #{@input_count}\n  Expected: #{headers.join(',')}\n     Found: #{row[0..ncells].join(',')}"
-              end
-            end
-            row = row[ncells+1, expected_field_count]
-            fieldcounts[row.count] += 1
-            emit(row.to_csv(:col_sep => @col_sep))
-          elsif !expected_field_count
-            # trust the first row
-            headers = explode(line).map(&:strip)
-            case opts[:snip]
-            when Fixnum
-              headers.shift(opts[:snip])
-            when String
-              raise "TODO: snip named header. Multiple?"
-            when nil
-              # do nothing
-            else
-              raise "Do not understand snip instruction [#{opts[:snip].inspect}]"
-            end
-            expected_field_count = headers.count
-            fieldcounts[headers.count] += 1
-            emit(headers.to_csv(:col_sep => @col_sep))
-          else
-            running_count = fieldcount(line)
-            while !input.eof? && running_count < expected_field_count do
-              nextbit = get
-              if nextbit
-                line = line + ' ' + nextbit
-                running_count = fieldcount(line)
-              end
-            end
-
-            unless opts[:dejunk] && junky?(line)
-              fieldcounts[fieldcount(line)] += 1
-              emit(line)
-            end
-          end
-        end
-      end
-    end
-
-    @output.close if opts[:output]
-    {
-      :input_count => @input_count,
-      :output_count => @output_count,
-      :field_counts => fieldcounts,
-      :headers => headers
-    }
-  end
-
   def fieldcount(line)
     explode(line).count
   end

data/lib/masticate/version.rb

@@ -1,3 +1,3 @@
 module Masticate
-  VERSION = "0.3.1"
+  VERSION = "0.3.2"
 end

data/spec/data/cooking_mend_result.csv

@@ -1,4 +1,4 @@
 Col 3   
-data  
-this long row
-data  
+data
+this long row is split across lines
+data

data/spec/data/newlines_in_input.psv

@@ -0,0 +1,8 @@
+LastName|FirstName|MiddleName|TitleName|AuditID|AuditByID|ChangeDTTM|AuditCode|AuditDescription|ObjectType|ReasonCodeID|ObjectID|AuthorizingName|DocumentID|DocumentAudTypeID|DocumentAudOrgID|DocumentAudComments|DocumentAudFileCount|DocumentAudIsInactiveFlag|PatientID|InternalExamID|PatientLast|PatientMiddle|PatientFirst
+Charles|Dingle|W|RIS Administrator|12345|1234|4/20/2012   2:54:26PM|MERGE|Merge Folders|DOCUMENT|0|999||888|38|2|Dropped 8 lb weight on foot.  c/o pain, swelling over 3rd, 4th metatarsals.  Possible non-displaced fractures of proximal 3rd and 4th metatarsals seen on one view.|0|N|888|777|WHIPPLE|D|ANNE
+Gomez|Bungle|||6|8|4/24/2012  10:31:53AM|MERGE|Merge Folders|DOCUMENT|0|101||505|38|2|broke little toe
+
+fracture prox phalanx, angulated (I reduced it but did not order a post reduction film.)
+
+sumner|0|N|499|172|SMITH|E|STEFANIE
+Gomez|Bangle|||59|8|3/28/2012   3:26:39PM|MERGE|Merge Folders|DOCUMENT|0|11||22|38|2|neg.|0|N|380|173|OCHS|JONES|CARL

data/spec/data/newlines_merged.csv

@@ -0,0 +1,4 @@
+LastName,FirstName,MiddleName,TitleName,AuditID,AuditByID,ChangeDTTM,AuditCode,AuditDescription,ObjectType,ReasonCodeID,ObjectID,AuthorizingName,DocumentID,DocumentAudTypeID,DocumentAudOrgID,DocumentAudComments,DocumentAudFileCount,DocumentAudIsInactiveFlag,PatientID,InternalExamID,PatientLast,PatientMiddle,PatientFirst
+Charles,Dingle,W,RIS Administrator,12345,1234,4/20/2012   2:54:26PM,MERGE,Merge Folders,DOCUMENT,0,999,,888,38,2,"Dropped 8 lb weight on foot.  c/o pain, swelling over 3rd, 4th metatarsals.  Possible non-displaced fractures of proximal 3rd and 4th metatarsals seen on one view.",0,N,888,777,WHIPPLE,D,ANNE
+Gomez,Bungle,,,6,8,4/24/2012  10:31:53AM,MERGE,Merge Folders,DOCUMENT,0,101,,505,38,2,"broke little toe fracture prox phalanx, angulated (I reduced it but did not order a post reduction film.) sumner",0,N,499,172,SMITH,E,STEFANIE
+Gomez,Bangle,,,59,8,3/28/2012   3:26:39PM,MERGE,Merge Folders,DOCUMENT,0,11,,22,38,2,neg.,0,N,380,173,OCHS,JONES,CARL

data/spec/lib/mender_spec.rb

@@ -39,4 +39,16 @@ describe "mending" do
     # results[:field_counts].should == {11 => 11}
     output.should == correct_output
   end
+
+  it "should consolidate fields with embedded newlines" do
+    filename = File.dirname(__FILE__) + "/../data/newlines_in_input.psv"
+    tmp = Tempfile.new('mending')
+    results = Masticate.mend(filename, :col_sep => '|', :output => tmp)
+    output = File.read(tmp)
+    correct_output = File.read(File.dirname(__FILE__) + "/../data/newlines_merged.csv")
+
+    results[:input_count].should == 6
+    results[:output_count].should == 4
+    output.should == correct_output
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: masticate
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2153051960 !ruby/object:Gem::Requirement
+  requirement: &2157784500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *2153051960
+  version_requirements: *2157784500
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2153051460 !ruby/object:Gem::Requirement
+  requirement: &2157784000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 2.9.0
   type: :development
   prerelease: false
-  version_requirements: *2153051460
+  version_requirements: *2157784000
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &2153051000 !ruby/object:Gem::Requirement
+  requirement: &2157783540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 0.7.0
   type: :development
   prerelease: false
-  version_requirements: *2153051000
+  version_requirements: *2157783540
 - !ruby/object:Gem::Dependency
   name: ruby_gntp
-  requirement: &2153050540 !ruby/object:Gem::Requirement
+  requirement: &2157783080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,7 +54,7 @@ dependencies:
         version: 0.3.4
   type: :development
   prerelease: false
-  version_requirements: *2153050540
+  version_requirements: *2157783080
 description: Data file crunching
 email:
 - jmay@pobox.com
@@ -107,6 +107,8 @@ files:
 - spec/data/junk_trailer.txt
 - spec/data/namedcols.csv
 - spec/data/namedcols.csv.output
+- spec/data/newlines_in_input.psv
+- spec/data/newlines_merged.csv
 - spec/data/pipe_data.txt
 - spec/data/quoted_csv_data.txt
 - spec/data/recipe.txt
@@ -172,6 +174,8 @@ test_files:
 - spec/data/junk_trailer.txt
 - spec/data/namedcols.csv
 - spec/data/namedcols.csv.output
+- spec/data/newlines_in_input.psv
+- spec/data/newlines_merged.csv
 - spec/data/pipe_data.txt
 - spec/data/quoted_csv_data.txt
 - spec/data/recipe.txt