masticate 0.1.0 → 0.1.1

data/Guardfile

@@ -3,7 +3,7 @@
 
 guard 'rspec', :version => 2 do
   watch(%r{^spec/.+_spec\.rb$})
-  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1].gsub(/masticate./, '')}_spec.rb" }
   watch('spec/spec_helper.rb')  { "spec" }
 
   watch(%r{^spec/support/(.+)\.rb$})                  { "spec" }

data/bin/masticate

@@ -37,61 +37,55 @@ OptionParser.new do |opts|
   opts.on("--to STRING", "Result string for gsub conversion") do |s|
     options[:to] = s
   end
+
+  opts.on("--inlined", "(for *mend* only) Source file has headers inlined on each line") do |v|
+    options[:inlined] = v
+  end
 end.parse!
 
 filename = ARGV.shift # use stdin if no filename provided
 
+def logmessage(command, options, results)
+  $stderr.puts <<-EOT
+* masticate #{command} (#{options.keys.join(', ')})
+  Lines in input: #{results[:input_count]}
+  Lines in output: #{results[:output_count]}
+EOT
+end
+
 case command
 when 'sniff'
   results = Masticate.sniff(filename)
   col_sep = results[:col_sep]
   col_sep = "TAB" if col_sep == "\t"
+  quote_char = results[:quote_char] || "NONE"
   $stderr.puts <<-EOT
 Processing complete.
   Input delimiter: #{col_sep}
+  Quote char: #{quote_char}
   Field counts: #{results[:field_counts].inspect}
   Headers: #{results[:headers].join(',')}
 EOT
 
 when 'mend'
   results = Masticate.mend(filename, options)
-  $stderr.puts <<-EOT
-Processing complete.
-  Lines in input: #{results[:input_count]}
-  Lines in output: #{results[:output_count]}
-EOT
+  logmessage(command, options, results)
 
 when 'csvify'
   results = Masticate.csvify(filename, options)
-  $stderr.puts <<-EOT
-Processing complete.
-  Lines in input: #{results[:input_count]}
-  Lines in output: #{results[:output_count]}
-EOT
+  logmessage(command, options, results)
 
 when 'pluck'
   results = Masticate.pluck(filename, options)
-  $stderr.puts <<-EOT
-Processing complete.
-  Lines in input: #{results[:input_count]}
-  Lines in output: #{results[:output_count]}
-EOT
+  logmessage(command, options, results)
 
 when 'datify'
   results = Masticate.datify(filename, options)
-  $stderr.puts <<-EOT
-Processing complete.
-  Lines in input: #{results[:input_count]}
-  Lines in output: #{results[:output_count]}
-EOT
+  logmessage(command, options, results)
 
 when 'gsub'
   results = Masticate.gsub(filename, options)
-#   $stderr.puts <<-EOT
-# Processing complete.
-#   Lines in input: #{results[:input_count]}
-#   Lines in output: #{results[:output_count]}
-# EOT
+  logmessage(command, options, results)
 
 else
   raise "unknown command #{command}"

data/lib/masticate/base.rb

@@ -2,6 +2,7 @@ class Masticate::Base
   attr_reader :filename
   attr_reader :input, :output
   attr_reader :input_count, :output_count
+  attr_reader :csv_options
 
   def initialize(filename)
     @filename = filename
@@ -30,4 +31,13 @@ class Masticate::Base
       # silently ignore this condition, it's not fatal and doesn't need a warning
     end
   end
+
+  def standard_options(opts)
+    @output = opts[:output] ? File.open(opts[:output], "w") : $stdout
+    @csv_options = {}
+    @csv_options[:col_sep] = opts[:col_sep] if opts[:col_sep]
+    if opts[:col_sep]
+      @csv_options[:quote_char] = opts[:quote_char] || "\0"
+    end
+  end
 end

data/lib/masticate/csvify.rb

@@ -7,10 +7,7 @@ class Masticate::Csvify < Masticate::Base
   end
 
   def csvify(opts)
-    @output = opts[:output] ? File.open(opts[:output], "w") : $stdout
-    csv_options = {}
-    csv_options[:col_sep] = opts[:col_sep] if opts[:col_sep]
-    csv_options[:quote_char] = opts[:quote_char] || opts[:col_sep] if opts[:quote_char] || opts[:col_sep]
+    standard_options(opts)
 
     @output_count = 0
     with_input do |input|

data/lib/masticate/datify.rb

@@ -3,10 +3,7 @@ require "csv"
 
 class Masticate::Datify < Masticate::Base
   def datify(opts)
-    @output = opts[:output] ? File.open(opts[:output], "w") : $stdout
-    csv_options = {}
-    csv_options[:col_sep] = opts[:col_sep] if opts[:col_sep]
-    csv_options[:quote_char] = opts[:quote_char] || opts[:col_sep] if opts[:quote_char] || opts[:col_sep]
+    standard_options(opts)
 
     field = opts[:field] or raise "missing field to datify"
     format = opts[:format] or raise "strptime format required for parsing timestamps"
@@ -29,7 +26,7 @@ class Masticate::Datify < Masticate::Base
     @output.close if opts[:output]
 
     {
-      :input_count => input_count,
+      :input_count => @input_count,
       :output_count => @output_count
     }
   end

data/lib/masticate/gsubber.rb

@@ -3,10 +3,7 @@ require "csv"
 
 class Masticate::Gsubber < Masticate::Base
   def gsub(opts)
-    @output = opts[:output] ? File.open(opts[:output], "w") : $stdout
-    csv_options = {}
-    csv_options[:col_sep] = opts[:col_sep] if opts[:col_sep]
-    csv_options[:quote_char] = opts[:quote_char] || opts[:col_sep] if opts[:quote_char] || opts[:col_sep]
+    standard_options(opts)
 
     field = opts[:field] or raise "missing field to gsub"
     from = Regexp.new(opts[:from]) or raise "Invalid regex '#{opts[:from]}' for conversion"
@@ -19,7 +16,7 @@ class Masticate::Gsubber < Masticate::Base
         row = CSV.parse_line(line, csv_options)
         if !headers
           headers = row
-          index = headers.index(field) or raise "Unable to find column '#{field}'"
+          index = headers.index(field) or raise "Unable to find column '#{field}' in headers"
           emit(line)
         else
           oldval = row[index]

data/lib/masticate/max_rows.rb

@@ -0,0 +1,51 @@
+# extract subset of columns from CSV
+require "csv"
+
+class Masticate::MaxRows < Masticate::Base
+  def maxrows(opts)
+    @output = opts[:output] ? File.open(opts[:output], "w") : $stdout
+    csv_options = {}
+    csv_options[:col_sep] = opts[:col_sep] if opts[:col_sep]
+    csv_options[:quote_char] = opts[:quote_char] || "\0"
+
+    groupby = opts[:by] or raise "missing field to group by"
+    maxon = opts[:max] or raise "missing field to max on"
+
+    @output_count = 0
+    headers = nil
+    accum = {}
+    with_input do |input|
+      while line = get
+        row = CSV.parse_line(line, csv_options)
+        if !headers
+          headers = row
+          index_by = headers.index(groupby) or raise "Unable to find column '#{groupby}'"
+          index_max = headers.index(maxon) or raise "Unable to find column '#{maxon}'"
+          emit(line)
+        else
+          key = row[index_by]
+          if !accum[key]
+            accum[key] = row
+          else
+            oldscore = accum[key][index_max]
+            newscore = row[index_max]
+            if newscore > oldscore
+              accum[key] = row
+            end
+          end
+        end
+      end
+    end
+
+    accum.each do |k,row|
+      emit(row.to_csv)
+    end
+
+    @output.close if opts[:output]
+
+    {
+      :input_count => input_count,
+      :output_count => @output_count
+    }
+  end
+end

data/lib/masticate/mender.rb

@@ -13,6 +13,7 @@ class Masticate::Mender < Masticate::Base
   def mend(opts)
     @output = opts[:output] ? File.open(opts[:output], "w") : $stdout
     @col_sep = opts[:col_sep] || ','
+    @quote_char = opts[:quote_char] || "\0"
 
     expected_field_count = nil
     headers = nil
@@ -20,12 +21,28 @@ class Masticate::Mender < Masticate::Base
     with_input do |input|
       while (line = get) do
         unless line =~ /^\s*$/
-          if !expected_field_count
+          if opts[:inlined]
+            row = explode(line)
+            ncells = row.count/2-1
+            if !expected_field_count
+              headers = row[0..ncells]
+              expected_field_count = headers.count
+              emit(headers.to_csv(:col_sep => @col_sep))
+            else
+              if row[0..ncells] != headers
+                raise "Header mismatch on line #{@input_count}\n  Expected: #{headers.join(',')}\n     Found: #{row[0..ncells].join(',')}"
+              end
+            end
+            row = row[ncells+1..-1]
+            emit(row.to_csv(:col_sep => @col_sep))
+          elsif !expected_field_count
             # trust the first row
-            headers = explode(line)
+            headers = explode(line).map(&:strip)
             case opts[:snip]
             when Fixnum
               headers.shift(opts[:snip])
+            when String
+              raise "TODO: snip named header. Multiple?"
             when nil
               # do nothing
             else
@@ -43,7 +60,7 @@ class Masticate::Mender < Masticate::Base
               end
             end
 
-            if line.count(col_sep) > 2
+            unless opts[:dejunk] && junky?(line)
               emit(line)
             end
           end
@@ -60,18 +77,15 @@ class Masticate::Mender < Masticate::Base
   end
 
   def fieldcount(line)
-    if col_sep == ','
-      CSV.parse_line(line).count
-    else
-      line.count(col_sep)+1
-    end
+    explode(line).count
   end
 
   def explode(line)
-    if col_sep == ','
-      CSV.parse_line(line).map(&:strip)
-    else
-      line.split(col_sep).map(&:strip)
-    end
+    CSV.parse_line(line, :col_sep => col_sep, :quote_char => @quote_char)
+  end
+
+  # a line is "junky" if it has 2 or fewer fields with any content
+  def junky?(line)
+    explode(line).select {|s| s && !s.strip.empty?}.count <= 2
   end
 end

data/lib/masticate/plucker.rb

@@ -3,10 +3,7 @@ require "csv"
 
 class Masticate::Plucker < Masticate::Base
   def pluck(opts)
-    @output = opts[:output] ? File.open(opts[:output], "w") : $stdout
-    csv_options = {}
-    csv_options[:col_sep] = opts[:col_sep] if opts[:col_sep]
-    csv_options[:quote_char] = opts[:quote_char] || opts[:col_sep] if opts[:quote_char] || opts[:col_sep]
+    standard_options(opts)
 
     fields = opts[:fields] or raise "missing fields to pluck"
 

data/lib/masticate/sniffer.rb

@@ -67,13 +67,8 @@ class Masticate::Sniffer < Masticate::Base
   end
 
   def stats
-    counts = with_input do |input|
-      if col_sep == ',' && quote_char
-        input.lines.each_with_object(Hash.new(0)) {|line, counts| counts[CSV.parse_line(line, :quote_char => quote_char).count] += 1}
-      else
-        input.lines.each_with_object(Hash.new(0)) {|line, counts| counts[line.split(col_sep).count] += 1}
-      end
+    with_input do |input|
+      input.lines.each_with_object(Hash.new(0)) {|line, counts| counts[CSV.parse_line(line, :col_sep => col_sep, :quote_char => quote_char || "\0").count] += 1}
     end
-    counts
   end
 end

data/lib/masticate/version.rb

@@ -1,3 +1,3 @@
 module Masticate
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/masticate.rb

@@ -8,6 +8,7 @@ require_relative "masticate/csvify"
 require_relative "masticate/plucker"
 require_relative "masticate/datify"
 require_relative "masticate/gsubber"
+require_relative "masticate/max_rows"
 
 module Masticate
   def self.sniff(filename)
@@ -33,4 +34,8 @@ module Masticate
   def self.gsub(filename, opts)
     Gsubber.new(filename).gsub(opts)
   end
+
+  def self.maxrows(filename, opts)
+    MaxRows.new(filename).maxrows(opts)
+  end
 end

data/spec/data/events.csv

@@ -0,0 +1,8 @@
+timestamp,uid,description
+1326448188,123,stuff
+1327481049,234,stuff
+1327405172,123,stuff
+1327655234,345,stuff
+1327486334,456,stuff
+1329447578,345,stuff
+1326297465,123,stuff

data/spec/data/events_reduced.csv

@@ -0,0 +1,5 @@
+timestamp,uid,description
+1327405172,123,stuff
+1327481049,234,stuff
+1329447578,345,stuff
+1327486334,456,stuff

data/spec/data/inlined_headers.csv

@@ -0,0 +1,10 @@
+LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,WASHINGTON,GEORGE,D,824,9556,09/10/2005,07/01/2006,TM,Surgical House Staff,M,09/23/1975,3/10/2012
+LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,JEFFERSON,TOM,,621,8052,07/23/2001,01/28/2011,TM,Telemetry,F,12/24/1976,3/10/2012
+LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,ADAMS,JOHN,,655,8834,09/22/2003,,WA,6 East,F,08/07/1978,3/10/2012
+LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,ADAMS,JOHN QUINCY,A,209,8637,02/24/2003,12/02/2007,TM,Imaging Svcs - MRI,F,11/03/1966,3/10/2012
+LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,HAMILTON,ANDREW,,278,10065,01/09/2007,11/16/2007,TM,Information Technology,M,09/16/1968,3/10/2012
+LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,MADISON,JAMES,F,672,10720,01/05/2009,02/16/2010,TM,Rehab Svcs - Outpatients,F,04/15/1985,3/10/2012
+LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,FRANKLIN,BENJAMIN,R,674,8340,05/01/2002,09/01/2003,TM,Rehab Svcs - PRN,F,03/15/1973,3/10/2012
+LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,LINCOLN,ABRAHAM,M,634,11340,05/02/2011,,PN,Sibley Ambulatory Surgery Ctr,F,07/11/1960,3/10/2012
+LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,MONROE,JAMES,L,614,10757,02/16/2009,,RF,Labor & Delivery,F,11/06/1983,3/10/2012
+LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,REVERE,PAUL,B,424,8568,11/18/2002,06/27/2006,TM,Laundry & Linen,M,12/31/1976,3/10/2012

data/spec/data/inlined_headers.csv.output

@@ -0,0 +1,11 @@
+LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE
+WASHINGTON,GEORGE,D,824,9556,09/10/2005,07/01/2006,TM,Surgical House Staff,M,09/23/1975,3/10/2012
+JEFFERSON,TOM,,621,8052,07/23/2001,01/28/2011,TM,Telemetry,F,12/24/1976,3/10/2012
+ADAMS,JOHN,,655,8834,09/22/2003,,WA,6 East,F,08/07/1978,3/10/2012
+ADAMS,JOHN QUINCY,A,209,8637,02/24/2003,12/02/2007,TM,Imaging Svcs - MRI,F,11/03/1966,3/10/2012
+HAMILTON,ANDREW,,278,10065,01/09/2007,11/16/2007,TM,Information Technology,M,09/16/1968,3/10/2012
+MADISON,JAMES,F,672,10720,01/05/2009,02/16/2010,TM,Rehab Svcs - Outpatients,F,04/15/1985,3/10/2012
+FRANKLIN,BENJAMIN,R,674,8340,05/01/2002,09/01/2003,TM,Rehab Svcs - PRN,F,03/15/1973,3/10/2012
+LINCOLN,ABRAHAM,M,634,11340,05/02/2011,,PN,Sibley Ambulatory Surgery Ctr,F,07/11/1960,3/10/2012
+MONROE,JAMES,L,614,10757,02/16/2009,,RF,Labor & Delivery,F,11/06/1983,3/10/2012
+REVERE,PAUL,B,424,8568,11/18/2002,06/27/2006,TM,Laundry & Linen,M,12/31/1976,3/10/2012

data/spec/data/junk_trailer.txt

@@ -1,6 +1,7 @@
 COL1|COL 2|Col 3   |col-4|   col5  |col6
 one|  data |data  |d a t a|data|data
 two|  data |data  |d a t a|data|data
+||||1|
 three|  data |data  |d a t a|data,data|data
 four|  data |data "more data"  |d a t a|data|data
 trailer|123

data/spec/lib/maxrow_spec.rb

@@ -0,0 +1,21 @@
+# spec for picking most-recent or highest-scoring rows
+#
+# something like:
+#   select * from rows group by col_a having col_b = max(col_b)
+#
+# usage: masticate maxrows --by col_a --max col_b
+
+require "spec_helper"
+require "tempfile"
+
+describe "maxrows" do
+  it "should find " do
+    filename = File.dirname(__FILE__) + "/../data/events.csv"
+    tmp = Tempfile.new('maxrows')
+    results = Masticate.maxrows(filename, :output => tmp, :by => 'uid', :max => 'timestamp')
+    output = File.read(tmp)
+    correct_output = File.read(File.dirname(__FILE__) + "/../data/events_reduced.csv")
+
+    output.should == correct_output
+  end
+end

data/spec/lib/{mend_spec.rb → mender_spec.rb}

@@ -1,6 +1,7 @@
 # spec for file-sniffing functions
 
 require "spec_helper"
+require "tempfile"
 
 describe "mending" do
   it "should merge lines when delimiter counts don't match'" do
@@ -12,9 +13,8 @@ describe "mending" do
 
   it "should strip trailer records" do
     filename = File.dirname(__FILE__) + "/../data/junk_trailer.txt"
-    metadata = Masticate.sniff(filename)
-    results = Masticate.mend(filename, metadata.merge(:output => "/dev/null"))
-    results[:input_count].should == 9
+    results = Masticate.mend(filename, :col_sep => '|', :output => "/dev/null", :dejunk => true)
+    results[:input_count].should == 10
     results[:output_count].should == 5
     results[:headers].should == ['COL1', 'COL 2', 'Col 3', 'col-4', 'col5', 'col6']
   end
@@ -26,4 +26,16 @@ describe "mending" do
     results[:output_count].should == 5
     results[:headers].should == %w(hospid usrorder dteorder usrsend dtesend usrdone dtedone department)
   end
+
+  it "should unfold inlined headers" do
+    filename = File.dirname(__FILE__) + "/../data/inlined_headers.csv"
+    tmp = Tempfile.new('mending')
+    results = Masticate.mend(filename, :inlined => true, :output => tmp)
+    output = File.read(tmp)
+    correct_output = File.read(File.dirname(__FILE__) + "/../data/inlined_headers.csv.output")
+
+    results[:input_count].should == 11
+    results[:output_count].should == 11
+    output.should == correct_output
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: masticate
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-05 00:00:00.000000000 Z
+date: 2012-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2153339260 !ruby/object:Gem::Requirement
+  requirement: &2152293880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 2.9.0
   type: :development
   prerelease: false
-  version_requirements: *2153339260
+  version_requirements: *2152293880
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &2153338760 !ruby/object:Gem::Requirement
+  requirement: &2152293360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 0.7.0
   type: :development
   prerelease: false
-  version_requirements: *2153338760
+  version_requirements: *2152293360
 - !ruby/object:Gem::Dependency
   name: ruby_gntp
-  requirement: &2153338100 !ruby/object:Gem::Requirement
+  requirement: &2152292900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,7 +43,7 @@ dependencies:
         version: 0.3.4
   type: :development
   prerelease: false
-  version_requirements: *2153338100
+  version_requirements: *2152292900
 description: Data file crunching
 email:
 - jmay@pobox.com
@@ -64,6 +64,7 @@ files:
 - lib/masticate/csvify.rb
 - lib/masticate/datify.rb
 - lib/masticate/gsubber.rb
+- lib/masticate/max_rows.rb
 - lib/masticate/mender.rb
 - lib/masticate/plucker.rb
 - lib/masticate/sniffer.rb
@@ -72,6 +73,10 @@ files:
 - spec/data/badnums.csv
 - spec/data/badnums_fixed.csv
 - spec/data/broken_psv.txt
+- spec/data/events.csv
+- spec/data/events_reduced.csv
+- spec/data/inlined_headers.csv
+- spec/data/inlined_headers.csv.output
 - spec/data/junk_header.csv
 - spec/data/junk_trailer.txt
 - spec/data/namedcols.csv
@@ -81,7 +86,8 @@ files:
 - spec/data/tabbed_data.txt
 - spec/lib/csvify_spec.rb
 - spec/lib/gsub_spec.rb
-- spec/lib/mend_spec.rb
+- spec/lib/maxrow_spec.rb
+- spec/lib/mender_spec.rb
 - spec/lib/plucker_spec.rb
 - spec/lib/sniffer_spec.rb
 - spec/spec_helper.rb
@@ -113,6 +119,10 @@ test_files:
 - spec/data/badnums.csv
 - spec/data/badnums_fixed.csv
 - spec/data/broken_psv.txt
+- spec/data/events.csv
+- spec/data/events_reduced.csv
+- spec/data/inlined_headers.csv
+- spec/data/inlined_headers.csv.output
 - spec/data/junk_header.csv
 - spec/data/junk_trailer.txt
 - spec/data/namedcols.csv
@@ -122,7 +132,8 @@ test_files:
 - spec/data/tabbed_data.txt
 - spec/lib/csvify_spec.rb
 - spec/lib/gsub_spec.rb
-- spec/lib/mend_spec.rb
+- spec/lib/maxrow_spec.rb
+- spec/lib/mender_spec.rb
 - spec/lib/plucker_spec.rb
 - spec/lib/sniffer_spec.rb
 - spec/spec_helper.rb