masticate 0.0.4 → 0.1.0

data/bin/masticate

@@ -1,10 +1,47 @@
 #!/usr/bin/env ruby
 
 require_relative "../lib/masticate"
+require "optparse"
 
-command, filename = ARGV
+command = ARGV.shift
 
-case ARGV.shift
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: example.rb [options]"
+
+  opts.on("--format FORMAT", "Specify format") do |v|
+    options[:format] = v
+  end
+
+  opts.on("--delim DELIMITER", "Specify field delimiter (character or TAB)") do |v|
+    options[:col_sep] = v
+    options[:col_sep] = "\t" if options[:col_sep] == "TAB"
+  end
+
+  opts.on("--fields LIST", Array, "Specify fields to select") do |list|
+    options[:fields] = list
+  end
+
+  opts.on("--field FIELD", "Specify field to convert") do |f|
+    options[:field] = f
+  end
+
+  opts.on("--snip DIRECTIVE", "Specify header fields to snip: first N, or by name") do |f|
+    options[:snip] = f.to_i
+  end
+
+  opts.on("--from REGEXP", "Regular expression for gsub conversion") do |s|
+    options[:from] = s
+  end
+
+  opts.on("--to STRING", "Result string for gsub conversion") do |s|
+    options[:to] = s
+  end
+end.parse!
+
+filename = ARGV.shift # use stdin if no filename provided
+
+case command
 when 'sniff'
   results = Masticate.sniff(filename)
   col_sep = results[:col_sep]
@@ -13,30 +50,49 @@ when 'sniff'
 Processing complete.
   Input delimiter: #{col_sep}
   Field counts: #{results[:field_counts].inspect}
+  Headers: #{results[:headers].join(',')}
 EOT
 
 when 'mend'
-  metadata = Masticate.sniff(filename)
-  col_sep = metadata[:col_sep]
-  col_sep = "TAB" if col_sep == "\t"
-  results = Masticate.mend(filename, metadata)
+  results = Masticate.mend(filename, options)
   $stderr.puts <<-EOT
 Processing complete.
-  Input delimiter: #{col_sep}
-  Lines in input: #{results[:input_records]}
-  Lines in output: #{results[:output_records]}
+  Lines in input: #{results[:input_count]}
+  Lines in output: #{results[:output_count]}
 EOT
 
 when 'csvify'
-  metadata = Masticate.sniff(filename)
-  results = Masticate.csvify(filename, metadata)
+  results = Masticate.csvify(filename, options)
   $stderr.puts <<-EOT
 Processing complete.
-  Input delimiter: #{metadata[:col_sep]}
   Lines in input: #{results[:input_count]}
   Lines in output: #{results[:output_count]}
 EOT
 
+when 'pluck'
+  results = Masticate.pluck(filename, options)
+  $stderr.puts <<-EOT
+Processing complete.
+  Lines in input: #{results[:input_count]}
+  Lines in output: #{results[:output_count]}
+EOT
+
+when 'datify'
+  results = Masticate.datify(filename, options)
+  $stderr.puts <<-EOT
+Processing complete.
+  Lines in input: #{results[:input_count]}
+  Lines in output: #{results[:output_count]}
+EOT
+
+when 'gsub'
+  results = Masticate.gsub(filename, options)
+#   $stderr.puts <<-EOT
+# Processing complete.
+#   Lines in input: #{results[:input_count]}
+#   Lines in output: #{results[:output_count]}
+# EOT
+
 else
   raise "unknown command #{command}"
 end

data/lib/masticate/base.rb

@@ -0,0 +1,33 @@
+class Masticate::Base
+  attr_reader :filename
+  attr_reader :input, :output
+  attr_reader :input_count, :output_count
+
+  def initialize(filename)
+    @filename = filename
+  end
+
+  def with_input
+    @input = @filename ? open(@filename) : $stdin
+    @input_count = 0
+    result = yield @input
+    @input.close if @filename
+    result
+  end
+
+  def get
+    line = @input.gets
+    @input_count += 1
+    line && line.chomp
+  end
+
+  def emit(line)
+    @output_count += 1
+    begin
+      @output.puts line
+    rescue Errno::EPIPE
+      # output was closed, e.g. ran piped into `head`
+      # silently ignore this condition, it's not fatal and doesn't need a warning
+    end
+  end
+end

data/lib/masticate/csvify.rb

@@ -1,11 +1,9 @@
 # convert input to clean standard CSV
 require "csv"
 
-class Masticate::Csvify
-  attr_reader :input
-
+class Masticate::Csvify < Masticate::Base
   def initialize(filename)
-    @input = File.open(filename)
+    @filename = filename
   end
 
   def csvify(opts)
@@ -14,26 +12,18 @@ class Masticate::Csvify
     csv_options[:col_sep] = opts[:col_sep] if opts[:col_sep]
     csv_options[:quote_char] = opts[:quote_char] || opts[:col_sep] if opts[:quote_char] || opts[:col_sep]
 
-    input_count = @output_count = 0
-    CSV.foreach(input, csv_options) do |row|
-      input_count += 1
-      emit(row.to_csv)
+    @output_count = 0
+    with_input do |input|
+      while line = get
+        row = CSV.parse_line(line, csv_options)
+        emit(row.to_csv) if row
+      end
     end
     @output.close if opts[:output]
-    @input.close
+
     {
       :input_count => input_count,
       :output_count => @output_count
     }
   end
-
-  def emit(line)
-    @output_count += 1
-    begin
-      @output.puts line
-    rescue Errno::EPIPE
-      # output was closed, e.g. ran piped into `head`
-      # silently ignore this condition, it's not fatal and doesn't need a warning
-    end
-  end
 end

data/lib/masticate/datify.rb

@@ -0,0 +1,36 @@
+# convert date columns to numerics
+require "csv"
+
+class Masticate::Datify < Masticate::Base
+  def datify(opts)
+    @output = opts[:output] ? File.open(opts[:output], "w") : $stdout
+    csv_options = {}
+    csv_options[:col_sep] = opts[:col_sep] if opts[:col_sep]
+    csv_options[:quote_char] = opts[:quote_char] || opts[:col_sep] if opts[:quote_char] || opts[:col_sep]
+
+    field = opts[:field] or raise "missing field to datify"
+    format = opts[:format] or raise "strptime format required for parsing timestamps"
+
+    @output_count = 0
+    headers = nil
+    with_input do |input|
+      while line = get
+        row = CSV.parse_line(line, csv_options)
+        if !headers
+          headers = row
+          index = headers.index(field) or raise "Unable to find column '#{field}'"
+          emit(headers.to_csv)
+        else
+          row[index] = DateTime.strptime(row[index], format).to_time.to_i rescue nil
+          emit(row.to_csv)
+        end
+      end
+    end
+    @output.close if opts[:output]
+
+    {
+      :input_count => input_count,
+      :output_count => @output_count
+    }
+  end
+end

data/lib/masticate/gsubber.rb

@@ -0,0 +1,39 @@
+# extract subset of columns from CSV
+require "csv"
+
+class Masticate::Gsubber < Masticate::Base
+  def gsub(opts)
+    @output = opts[:output] ? File.open(opts[:output], "w") : $stdout
+    csv_options = {}
+    csv_options[:col_sep] = opts[:col_sep] if opts[:col_sep]
+    csv_options[:quote_char] = opts[:quote_char] || opts[:col_sep] if opts[:quote_char] || opts[:col_sep]
+
+    field = opts[:field] or raise "missing field to gsub"
+    from = Regexp.new(opts[:from]) or raise "Invalid regex '#{opts[:from]}' for conversion"
+    to = opts[:to] or raise "missing 'to' string for gsub"
+
+    @output_count = 0
+    headers = nil
+    with_input do |input|
+      while line = get
+        row = CSV.parse_line(line, csv_options)
+        if !headers
+          headers = row
+          index = headers.index(field) or raise "Unable to find column '#{field}'"
+          emit(line)
+        else
+          oldval = row[index]
+          newval = oldval.gsub(from, to)
+          row[index] = newval
+          emit(row.to_csv)
+        end
+      end
+    end
+    @output.close if opts[:output]
+
+    {
+      :input_count => input_count,
+      :output_count => @output_count
+    }
+  end
+end

data/lib/masticate/mender.rb

@@ -3,61 +3,75 @@
 # A row that contains fewer delimiters than expected has been split across two lines
 # (due to a newline embedded in a field).  Glue those two lines into a single line in the output.
 
-class Masticate::Mender
-  attr_reader :input
+class Masticate::Mender < Masticate::Base
+  attr_reader :col_sep
 
   def initialize(filename)
-    @input = open(filename)
+    @filename = filename
   end
 
   def mend(opts)
     @output = opts[:output] ? File.open(opts[:output], "w") : $stdout
-    col_sep = opts[:col_sep]
-
-    expected_delim_count = nil
-    @input_count = @output_count = 0
-    while (line = get) do
-      unless line =~ /^\s*$/
-        if !expected_delim_count
-          # trust the first row
-          expected_delim_count = line.count(col_sep)
-        else
-          running_count = line.count(col_sep)
-          while !input.eof? && running_count < expected_delim_count do
-            nextbit = get
-            if nextbit
-              line = line + ' ' + nextbit
-              running_count = line.count(col_sep)
+    @col_sep = opts[:col_sep] || ','
+
+    expected_field_count = nil
+    headers = nil
+    @output_count = 0
+    with_input do |input|
+      while (line = get) do
+        unless line =~ /^\s*$/
+          if !expected_field_count
+            # trust the first row
+            headers = explode(line)
+            case opts[:snip]
+            when Fixnum
+              headers.shift(opts[:snip])
+            when nil
+              # do nothing
+            else
+              raise "Do not understand snip instruction [#{opts[:snip].inspect}]"
+            end
+            expected_field_count = headers.count
+            emit(headers.to_csv(:col_sep => @col_sep))
+          else
+            running_count = fieldcount(line)
+            while !input.eof? && running_count < expected_field_count do
+              nextbit = get
+              if nextbit
+                line = line + ' ' + nextbit
+                running_count = fieldcount(line)
+              end
+            end
+
+            if line.count(col_sep) > 2
+              emit(line)
             end
           end
         end
-        if line.count(col_sep) > 2
-          emit(line)
-        end
       end
     end
 
-    @input.close
     @output.close if opts[:output]
     {
-      :input_records => @input_count,
-      :output_records => @output_count
+      :input_count => @input_count,
+      :output_count => @output_count,
+      :headers => headers
     }
   end
 
-  def get
-    line = input.gets
-    @input_count += 1
-    line && line.chomp
+  def fieldcount(line)
+    if col_sep == ','
+      CSV.parse_line(line).count
+    else
+      line.count(col_sep)+1
+    end
   end
 
-  def emit(line)
-    @output_count += 1
-    begin
-      @output.puts line
-    rescue Errno::EPIPE
-      # output was closed, e.g. ran piped into `head`
-      # silently ignore this condition, it's not fatal and doesn't need a warning
+  def explode(line)
+    if col_sep == ','
+      CSV.parse_line(line).map(&:strip)
+    else
+      line.split(col_sep).map(&:strip)
     end
   end
 end

data/lib/masticate/plucker.rb

@@ -0,0 +1,34 @@
+# extract subset of columns from CSV
+require "csv"
+
+class Masticate::Plucker < Masticate::Base
+  def pluck(opts)
+    @output = opts[:output] ? File.open(opts[:output], "w") : $stdout
+    csv_options = {}
+    csv_options[:col_sep] = opts[:col_sep] if opts[:col_sep]
+    csv_options[:quote_char] = opts[:quote_char] || opts[:col_sep] if opts[:quote_char] || opts[:col_sep]
+
+    fields = opts[:fields] or raise "missing fields to pluck"
+
+    @output_count = 0
+    headers = nil
+    with_input do |input|
+      while line = get
+        row = CSV.parse_line(line, csv_options)
+        if !headers
+          headers = row
+          indexes = fields.map {|f| headers.index(f) or raise "Unable to find column '#{f}'"}
+          emit(fields.to_csv)
+        else
+          emit(indexes.map {|i| row[i]}.to_csv) if row
+        end
+      end
+    end
+    @output.close if opts[:output]
+
+    {
+      :input_count => input_count,
+      :output_count => @output_count
+    }
+  end
+end

data/lib/masticate/sniffer.rb

@@ -1,5 +1,8 @@
-class Masticate::Sniffer
-  attr_reader :col_sep, :stats
+require "set"
+
+class Masticate::Sniffer < Masticate::Base
+  attr_reader :col_sep, :quote_char, :stats
+  attr_reader :delimstats
 
   CandidateDelimiters = [',', '|', "\t"]
 
@@ -9,32 +12,68 @@ class Masticate::Sniffer
 
   def sniff
     @col_sep = find_col_sep
+    @quote_char = delimstats[@col_sep][:quote_char]
     @stats = stats
     {
       :col_sep => @col_sep,
+      :quote_char => @quote_char,
       :field_counts => @stats,
-      :line1 => @line1
+      :headers => @line1.split(@col_sep).map(&:strip)
     }
   end
 
   def find_col_sep
-    input = open(@filename)
-    @line1 = input.lines.first
-    delimcounts = CandidateDelimiters.each_with_object({}) do |delim,h|
-      h[delim] = consider_delim(@line1, delim)
+    @delimstats = {}
+    with_input do |input|
+      input.lines.take(10).each do |line|
+        @line1 = line unless @line1
+
+        CandidateDelimiters.each do |delim|
+          delimstats[delim] ||= { :counts => Set.new, :quote_char => nil}
+          h = delimstats[delim]
+          fieldcount, quote_char = consider_delim(line, delim)
+          h[:counts] << fieldcount
+          h[:quote_char] ||= quote_char
+        end
+      end
     end
-    input.close
-    delimcounts.sort_by{|h,v| -v}.first.first
+    delimstats.sort_by{|delim,stats| stats[:counts].max || 0}.last.first
   end
 
   def consider_delim(line, delim)
-    line.count(delim)
+    @quote_char = nil
+    n = count_fields(line, delim)
+    [n, @quote_char]
+  end
+
+  def count_fields(line, delim)
+    if delim == ','
+      straight_count = line.count(delim) + 1
+      count_with_quoting = begin
+        CSV.parse_line(line).count
+      rescue CSV::MalformedCSVError
+        # this is not valid CSV, e.g. has incorrectly embedded quotes
+        0
+      end
+      if count_with_quoting < straight_count
+        @quote_char = '"'
+        count_with_quoting
+      else
+        straight_count
+      end
+    else
+      line.count(delim) + 1
+    end
   end
 
   def stats
-    input = open(@filename)
-    counts = input.lines.each_with_object(Hash.new(0)) {|line, counts| counts[line.split(col_sep).count] += 1}
-    input.close
+    counts = with_input do |input|
+      if col_sep == ',' && quote_char
+        input.lines.each_with_object(Hash.new(0)) {|line, counts| counts[CSV.parse_line(line, :quote_char => quote_char).count] += 1}
+      else
+        input.lines.each_with_object(Hash.new(0)) {|line, counts| counts[line.split(col_sep).count] += 1}
+      end
+    end
     counts
   end
 end

data/lib/masticate/version.rb

@@ -1,3 +1,3 @@
 module Masticate
-  VERSION = "0.0.4"
+  VERSION = "0.1.0"
 end

data/lib/masticate.rb

@@ -1,9 +1,13 @@
 require "open-uri"
 
 require_relative "masticate/version"
+require_relative "masticate/base"
 require_relative "masticate/sniffer"
 require_relative "masticate/mender"
 require_relative "masticate/csvify"
+require_relative "masticate/plucker"
+require_relative "masticate/datify"
+require_relative "masticate/gsubber"
 
 module Masticate
   def self.sniff(filename)
@@ -17,4 +21,16 @@ module Masticate
   def self.csvify(filename, opts)
     Csvify.new(filename).csvify(opts)
   end
+
+  def self.pluck(filename, opts)
+    Plucker.new(filename).pluck(opts)
+  end
+
+  def self.datify(filename, opts)
+    Datify.new(filename).datify(opts)
+  end
+
+  def self.gsub(filename, opts)
+    Gsubber.new(filename).gsub(opts)
+  end
 end

data/spec/data/badnums.csv

@@ -0,0 +1,8 @@
+ChangeDTTM,AuditCode,AuditByID
+1326448188,MERGE,"518,437.00"
+1327481049,MERGEMATCH,0.00
+1327405172,MERGEMATCH,0.00
+1327655234,MERGEMATCH,0.00
+1327486334,MERGE,"429,073.00"
+1326447578,MERGEMATCH,0.00
+1326297465,MERGE,"123,456.00"

data/spec/data/badnums_fixed.csv

@@ -0,0 +1,8 @@
+ChangeDTTM,AuditCode,AuditByID
+1326448188,MERGE,"518,437.00"
+1327481049,MERGEMATCH,0.00
+1327405172,MERGEMATCH,0.00
+1327655234,MERGEMATCH,0.00
+1327486334,MERGE,"429,073.00"
+1326447578,MERGEMATCH,0.00
+1326297465,MERGE,"123,456.00"

data/spec/data/junk_header.csv

@@ -0,0 +1,5 @@
+3/7/2012,hospid,usrorder,dteorder,usrsend,dtesend,usrdone,dtedone,department
+15267,407,201201060140,407,201201060140,0,201201060309,L
+15267,381,201201060222,381,201201060222,0,201201060647,X
+15267,407,201201060311,407,201201060311,0,201201060339,L
+15267,407,201201060514,108,201201060515,108,201201060515,SEC

data/spec/data/namedcols.csv

@@ -0,0 +1,4 @@
+one,two,three,four,five
+data1,data2,data3,data4,data5
+111,22,333,44,555
+91,92,93,94,95

data/spec/data/namedcols.csv.output

@@ -0,0 +1,4 @@
+three,five
+data3,data5
+333,555
+93,95

data/spec/data/quoted_csv_data.txt

@@ -0,0 +1,100 @@
+site,ibex,unit,face,doctor,hospid,usrorder,dteorder,usrsend,dtesend,usrdone,dtedone,department,order_number
+1,20120106003230,2044272,L,407,15267,407,201201060140,407,201201060140,0,201201060309,L,"594,756"
+1,20120106003230,2044277,X,407,15267,381,201201060222,381,201201060222,0,201201060647,X,"594,761"
+1,20120106003230,2044309,L,407,15267,407,201201060311,407,201201060311,0,201201060339,L,"594,766"
+1,20120106003230,,Q,407,15267,407,201201060514,108,201201060515,108,201201060515,SEC,"594,787"
+1,20120106024355,,Q,407,15267,407,201201060309,90,201201060316,90,201201060316,IV,"594,764"
+1,20120106024355,2044306,L,407,15267,407,201201060309,407,201201060309,0,201201060345,L,"594,763"
+1,20120106024355,2044308,X,407,15267,407,201201060310,407,201201060310,0,201201060556,X,"594,765"
+1,20120106024355,2044307,L,407,15267,407,201201060309,407,201201060309,0,201201060333,L,"594,762"
+1,20120106024355,,Q,407,15267,407,201201060520,108,201201060522,108,201201060522,SEC,"594,789"
+1,20120106024355,2044579,L,407,15267,68,201201060826,68,201201060826,0,201201071149,L,"594,823"
+1,20120106032719,2044345,L,407,15267,407,201201060348,407,201201060348,0,201201060442,L,"594,775"
+1,20120106032719,2044344,L,407,15267,407,201201060348,407,201201060348,0,201201060442,L,"594,777"
+1,20120106032719,2044343,L,407,15267,407,201201060348,407,201201060348,0,201201060428,L,"594,773"
+1,20120106032719,,Q,407,15267,407,201201060348,426,201201060408,426,201201060408,IV,"594,774"
+1,20120106032719,,Q,407,15267,407,201201060348,426,201201060634,426,201201060634,URINE,"594,776"
+1,20120106032719,2044386,L,407,15267,407,201201060445,407,201201060445,0,201201060519,L,"594,785"
+1,20120106032719,2044401,X,407,15267,407,201201060521,407,201201060521,0,201201060646,X,"594,790"
+1,20120106033235,,Q,407,15267,407,201201060347,74,201201060353,74,201201060353,IV,"594,769"
+1,20120106033235,2044349,L,407,15267,407,201201060347,74,201201060353,0,201201060443,L,"594,771"
+1,20120106033235,2044350,L,407,15267,407,201201060347,74,201201060353,0,201201060434,URINE,"594,770"
+1,20120106033235,2044347,L,407,15267,407,201201060347,74,201201060353,0,201201060428,L,"594,768"
+1,20120106033235,2044348,L,407,15267,407,201201060347,74,201201060353,0,201201060443,L,"594,772"
+1,20120106033235,2044372,X,407,15267,407,201201060429,407,201201060429,0,201201060649,X,"594,780"
+1,20120106035346,,Q,407,15267,407,201201060446,426,201201060448,426,201201060448,N,"594,786"
+1,20120106041426,2044383,L,407,15267,407,201201060445,407,201201060445,0,201201060657,L,"594,784"
+1,20120106041426,2044384,L,407,15267,407,201201060445,407,201201060445,0,201201060657,L,"594,782"
+1,20120106041426,2044382,L,407,15267,407,201201060445,407,201201060445,0,201201060522,L,"594,781"
+1,20120106041426,,Q,407,15267,407,201201060445,381,201201060452,381,201201060452,IV,"594,783"
+1,20120106043025,2044400,X,407,15267,407,201201060515,407,201201060515,0,201201060554,X,"594,788"
+1,20120106045326,2044411,R,407,15267,407,201201060535,407,201201060535,0,201201060630,RS,"594,791"
+1,20120106045326,,Q,407,15267,407,201201060535,108,201201060540,108,201201060540,SEC,"594,794"
+1,20120106045326,2044412,R,407,15267,407,201201060535,407,201201060535,0,201201060629,RS,"594,795"
+1,20120106045326,2044413,X,407,15267,407,201201060536,407,201201060536,0,201201060649,X,"594,796"
+1,20120106045326,,Q,407,15267,407,201201060535,108,201201060541,108,201201060541,SEC,"594,792"
+1,20120106045326,2044410,R,407,15267,407,201201060535,407,201201060535,0,201201060628,RS,"594,793"
+1,20120106052714,2044421,L,407,15267,407,201201060544,407,201201060544,0,201201060605,L,"594,797"
+1,20120106052714,,Q,407,15267,407,201201060544,90,201201060545,90,201201060545,IV,"594,799"
+1,20120106052714,,Q,407,15267,407,201201060544,90,201201060545,90,201201060545,N,"594,800"
+1,20120106052714,2044422,L,407,15267,407,201201060544,407,201201060544,0,201201060621,L,"594,801"
+1,20120106052714,2044423,L,407,15267,407,201201060544,407,201201060544,0,201201060727,L,"594,798"
+1,20120106052714,2044424,L,407,15267,407,201201060551,407,201201060551,0,201201060714,L,"594,802"
+1,20120106070243,2044439,L,504,15550,504,201201060721,504,201201060721,0,201201060753,L,"594,803"
+1,20120106070243,2044440,L,504,15550,504,201201060721,504,201201060721,0,201201060748,L,"594,807"
+1,20120106070243,2044441,L,504,15550,504,201201060721,504,201201060721,0,201201060748,L,"594,806"
+1,20120106070243,,Q,504,15550,504,201201060721,155,201201060735,155,201201060735,IV,"594,805"
+1,20120106070243,,Q,504,15550,504,201201060806,155,201201060813,155,201201060813,N,"594,820"
+1,20120106070243,2044524,L,504,15550,504,201201060806,504,201201060806,0,201201061004,L,"594,816"
+1,20120106070243,,Q,504,15550,504,201201060807,195,201201060813,195,201201060813,SEC,"594,822"
+1,20120106070243,2044522,L,504,15550,504,201201060806,504,201201060806,0,201201060959,L,"594,819"
+1,20120106070243,,Q,504,15550,504,201201060807,195,201201060811,195,201201060811,SEC,"594,821"
+1,20120106070243,,Q,504,15550,504,201201060806,155,201201060813,155,201201060813,N,"594,818"
+1,20120106070243,,Q,504,15550,504,201201060910,155,201201060916,155,201201060916,N,"594,831"
+1,20120106070243,2044716,X,504,15550,504,201201060928,504,201201060928,0,201201060953,X,"594,834"
+1,20120106073142,2044480,X,504,15550,504,201201060757,504,201201060757,0,201201060819,X,"594,815"
+1,20120106073757,2044475,L,504,15550,504,201201060749,155,201201060755,0,201201060925,URINE,"594,810"
+1,20120106073757,2044466,L,504,15550,504,201201060749,504,201201060749,0,201201060827,L,"594,808"
+1,20120106073757,2044470,X,504,15550,504,201201060749,504,201201060749,0,201201060818,X,"594,809"
+1,20120106073757,2044467,L,504,15550,504,201201060749,504,201201060749,0,201201060826,L,"594,813"
+1,20120106073757,2044468,L,504,15550,504,201201060749,504,201201060749,0,201201060839,L,"594,811"
+1,20120106073757,2044469,L,504,15550,504,201201060749,504,201201060749,0,201201060825,L,"594,814"
+1,20120106073757,,Q,504,15550,504,201201060749,155,201201060755,155,201201060755,IV,"594,812"
+1,20120106073757,,Q,504,15550,504,201201060911,76,201201060933,76,201201060933,IV,"594,832"
+1,20120106073757,,Q,504,15550,504,201201060928,34,201201060934,34,201201060934,SEC,"594,833"
+1,20120106073757,,Q,504,15550,504,201201061022,155,201201061108,155,201201061108,IV,"594,862"
+1,20120106073757,,Q,504,15550,504,201201061019,155,201201061025,155,201201061025,IV,"594,861"
+1,20120106073757,,Q,504,15550,504,201201061131,195,201201061133,195,201201061133,SEC,"594,896"
+1,20120106073757,,Q,504,15550,504,201201061131,195,201201061133,195,201201061133,SEC,"594,895"
+1,20120106073757,2045028,X,504,15550,504,201201061131,504,201201061131,0,201201061209,X,"594,898"
+1,20120106073757,2045029,X,504,15550,504,201201061131,504,201201061131,0,201201061345,X,"594,897"
+1,20120106073757,,Q,504,15550,504,201201061131,155,201201061223,155,201201061223,N,"594,894"
+1,20120106084347,2044639,X,504,15550,76,201201060850,76,201201060850,0,201201060931,X,"594,828"
+1,20120106084720,2044670,X,55,4644,55,201201060909,55,201201060909,0,201201060934,X,"594,829"
+1,20120106084720,,Q,55,4644,55,201201060910,66,201201060914,66,201201060914,N,"594,830"
+1,20120106085558,2044755,L,55,4644,55,201201060949,55,201201060949,0,201201061018,L,"594,846"
+1,20120106085558,2044756,L,55,4644,55,201201060949,55,201201060949,0,201201061038,L,"594,851"
+1,20120106085558,2044793,L,55,4644,55,201201060949,76,201201061003,0,201201061239,URINE,"594,848"
+1,20120106085558,,Q,55,4644,55,201201060949,76,201201061003,76,201201061003,IV,"594,850"
+1,20120106085558,,Q,55,4644,55,201201060949,76,201201061003,76,201201061003,IV,"594,847"
+1,20120106085558,2044757,L,55,4644,55,201201060949,55,201201060949,0,201201061040,L,"594,849"
+1,20120106085558,2044843,L,55,4644,55,201201061033,55,201201061033,0,201201071505,L,"594,864"
+1,20120106085558,2044841,X,55,4644,55,201201061032,55,201201061032,0,201201061136,X,"594,863"
+1,20120106085558,2044844,L,55,4644,55,201201061033,55,201201061033,0,201201061119,L,"594,865"
+1,20120106085558,,Q,55,4644,55,201201061228,195,201201061240,195,201201061240,SEC,"594,961"
+1,20120106091726,2044741,L,504,15550,504,201201060942,504,201201060942,0,201201061024,L,"594,839"
+1,20120106091726,2044745,X,504,15550,504,201201060942,504,201201060942,0,201201061016,X,"594,835"
+1,20120106091726,2044746,L,504,15550,504,201201060942,504,201201060942,0,201201061107,L,"594,842"
+1,20120106091726,2044740,L,504,15550,504,201201060942,504,201201060942,0,201201061017,L,"594,836"
+1,20120106091726,2044744,L,504,15550,504,201201060942,504,201201060942,0,201201061024,L,"594,838"
+1,20120106091726,2044742,L,504,15550,504,201201060942,504,201201060942,0,201201061016,L,"594,841"
+1,20120106091726,,Q,504,15550,504,201201060942,66,201201060944,66,201201060944,IV,"594,837"
+1,20120106091726,2044743,L,504,15550,504,201201060942,504,201201060942,0,201201061016,L,"594,840"
+1,20120106095129,2044814,X,55,4644,55,201201061010,55,201201061010,0,201201061037,X,"594,853"
+1,20120106100014,,Q,504,15550,504,201201061011,885,201201061037,885,201201061037,IV,"594,857"
+1,20120106100014,,Q,504,15550,504,201201061011,885,201201061037,885,201201061037,N,"594,858"
+1,20120106100014,,Q,504,15550,504,201201061011,885,201201061037,885,201201061037,N,"594,859"
+1,20120106100014,2044815,L,504,15550,504,201201061011,504,201201061011,0,201201061023,L,"594,854"
+1,20120106100014,2044817,L,504,15550,504,201201061011,504,201201061011,0,201201061049,L,"594,856"
+1,20120106100014,2044818,X,504,15550,504,201201061011,504,201201061011,0,201201061038,X,"594,855"
+1,20120106100014,2044816,L,504,15550,504,201201061011,504,201201061011,0,201201061049,L,"594,860"

data/spec/lib/csvify_spec.rb

@@ -11,7 +11,7 @@ describe "csvification" do
     output = File.read(tmp)
     tmp.unlink
     output.lines.count.should == 5
-    results[:input_count].should == 5
+    results[:input_count].should == 6
     results[:output_count].should == 5
   end
 end

data/spec/lib/gsub_spec.rb

@@ -0,0 +1,16 @@
+# spec for field regexp conversion
+
+require "spec_helper"
+require "tempfile"
+
+describe "gsubbing" do
+  it "should apply conversion to a single column" do
+    filename = File.dirname(__FILE__) + "/../data/badnums.csv"
+    tmp = Tempfile.new('gsubber')
+    results = Masticate.gsub(filename, :output => tmp, :field => 'AuditByID', :from => '/,|(.00$)/', :to => '')
+    output = File.read(tmp)
+    correct_output = File.read(File.dirname(__FILE__) + "/../data/badnums_fixed.csv")
+
+    output.should == correct_output
+  end
+end

data/spec/lib/mend_spec.rb

@@ -6,15 +6,24 @@ describe "mending" do
   it "should merge lines when delimiter counts don't match'" do
     filename = File.dirname(__FILE__) + "/../data/broken_psv.txt"
     results = Masticate.mend(filename, :col_sep => '|', :output => "/dev/null")
-    results[:input_records].should == 7
-    results[:output_records].should == 5
+    results[:input_count].should == 7
+    results[:output_count].should == 5
   end
 
   it "should strip trailer records" do
     filename = File.dirname(__FILE__) + "/../data/junk_trailer.txt"
     metadata = Masticate.sniff(filename)
     results = Masticate.mend(filename, metadata.merge(:output => "/dev/null"))
-    results[:input_records].should == 9
-    results[:output_records].should == 5
+    results[:input_count].should == 9
+    results[:output_count].should == 5
+    results[:headers].should == ['COL1', 'COL 2', 'Col 3', 'col-4', 'col5', 'col6']
+  end
+
+  it "should snip head fields" do
+    filename = File.dirname(__FILE__) + "/../data/junk_header.csv"
+    results = Masticate.mend(filename, :col_sep => ',', :snip => 1, :output => "/dev/null")
+    results[:input_count].should == 6
+    results[:output_count].should == 5
+    results[:headers].should == %w(hospid usrorder dteorder usrsend dtesend usrdone dtedone department)
   end
 end

data/spec/lib/plucker_spec.rb

@@ -0,0 +1,18 @@
+# spec for column-plucking functions
+
+require "spec_helper"
+require "tempfile"
+
+describe "plucker" do
+  it "should pull named columns" do
+    filename = File.dirname(__FILE__) + "/../data/namedcols.csv"
+    tmp = Tempfile.new('plucker')
+    results = Masticate.pluck(filename, :output => tmp, :fields => ['three', 'five'])
+    output = File.read(tmp)
+    correct_output = File.read(File.dirname(__FILE__) + "/../data/namedcols.csv.output")
+    tmp.unlink
+
+    results[:input_count].should == 5
+    output.should == correct_output
+  end
+end

data/spec/lib/sniffer_spec.rb

@@ -16,4 +16,12 @@ describe "delimiter sniffing" do
     results[:col_sep].should == '|'
     results[:field_counts].should == {6 => 5}
   end
+
+  it "should recognize quotes in CSV sources" do
+    filename = File.dirname(__FILE__) + "/../data/quoted_csv_data.txt"
+    results = Masticate.sniff(filename)
+    results[:col_sep].should == ','
+    results[:quote_char].should == '"'
+    results[:field_counts].should == {14 => 100}
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: masticate
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-04 00:00:00.000000000 Z
+date: 2012-04-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2153254280 !ruby/object:Gem::Requirement
+  requirement: &2153339260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 2.9.0
   type: :development
   prerelease: false
-  version_requirements: *2153254280
+  version_requirements: *2153339260
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &2153246900 !ruby/object:Gem::Requirement
+  requirement: &2153338760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 0.7.0
   type: :development
   prerelease: false
-  version_requirements: *2153246900
+  version_requirements: *2153338760
 - !ruby/object:Gem::Dependency
   name: ruby_gntp
-  requirement: &2153246180 !ruby/object:Gem::Requirement
+  requirement: &2153338100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,7 +43,7 @@ dependencies:
         version: 0.3.4
   type: :development
   prerelease: false
-  version_requirements: *2153246180
+  version_requirements: *2153338100
 description: Data file crunching
 email:
 - jmay@pobox.com
@@ -60,17 +60,29 @@ files:
 - Rakefile
 - bin/masticate
 - lib/masticate.rb
+- lib/masticate/base.rb
 - lib/masticate/csvify.rb
+- lib/masticate/datify.rb
+- lib/masticate/gsubber.rb
 - lib/masticate/mender.rb
+- lib/masticate/plucker.rb
 - lib/masticate/sniffer.rb
 - lib/masticate/version.rb
 - masticate.gemspec
+- spec/data/badnums.csv
+- spec/data/badnums_fixed.csv
 - spec/data/broken_psv.txt
+- spec/data/junk_header.csv
 - spec/data/junk_trailer.txt
+- spec/data/namedcols.csv
+- spec/data/namedcols.csv.output
 - spec/data/pipe_data.txt
+- spec/data/quoted_csv_data.txt
 - spec/data/tabbed_data.txt
 - spec/lib/csvify_spec.rb
+- spec/lib/gsub_spec.rb
 - spec/lib/mend_spec.rb
+- spec/lib/plucker_spec.rb
 - spec/lib/sniffer_spec.rb
 - spec/spec_helper.rb
 homepage: ''
@@ -98,12 +110,20 @@ signing_key:
 specification_version: 3
 summary: Utility functions for parsing incoming text data files.
 test_files:
+- spec/data/badnums.csv
+- spec/data/badnums_fixed.csv
 - spec/data/broken_psv.txt
+- spec/data/junk_header.csv
 - spec/data/junk_trailer.txt
+- spec/data/namedcols.csv
+- spec/data/namedcols.csv.output
 - spec/data/pipe_data.txt
+- spec/data/quoted_csv_data.txt
 - spec/data/tabbed_data.txt
 - spec/lib/csvify_spec.rb
+- spec/lib/gsub_spec.rb
 - spec/lib/mend_spec.rb
+- spec/lib/plucker_spec.rb
 - spec/lib/sniffer_spec.rb
 - spec/spec_helper.rb
 has_rdoc: