masticate 0.1.5 → 0.2

data/bin/masticate

@@ -1,119 +1,8 @@
 #!/usr/bin/env ruby
 
 require_relative "../lib/masticate"
-require "optparse"
 
-command = ARGV.shift
+masticator = Masticate::MyOptionParser.new
+command, options, filenames = masticator.parse
 
-options = {}
-OptionParser.new do |opts|
-  opts.banner = "Usage: example.rb [options]"
-
-  opts.on("--format FORMAT", String, "Specify format") do |v|
-    options[:format] = v
-  end
-
-  opts.on("--delim DELIMITER", String, "Specify field delimiter (character or TAB; default is ',')") do |v|
-    options[:col_sep] = v
-    options[:col_sep] = "\t" if options[:col_sep] == "TAB"
-  end
-
-  opts.on("--quote QUOTE-CHAR", String, "Specify character used for quoting fields (optional; default is no quoting)") do |char|
-    options[:quote_char] = char
-  end
-
-  opts.on("--stats", "(for *sniff*) collect & display input stats") do
-    options[:stats] = true
-  end
-
-  opts.on("--fields LIST", Array, "Specify fields to select") do |list|
-    options[:fields] = list
-  end
-
-  opts.on("--field FIELD", String, "Specify field to convert") do |f|
-    options[:field] = f
-  end
-
-  opts.on("--snip DIRECTIVE", String, "Specify header fields to snip: first N, or by name") do |f|
-    options[:snip] = f.to_i
-  end
-
-  opts.on("--from REGEXP", String, "Regular expression for gsub conversion") do |s|
-    options[:from] = s
-  end
-
-  opts.on("--to STRING", String, "Result string for gsub conversion") do |s|
-    options[:to] = s
-  end
-
-  opts.on("--inlined", "(for *mend* only) Source file has headers inlined on each line") do |b|
-    options[:inlined] = true
-  end
-
-  opts.on("--dejunk", "(for *mend* only) Expunge junk lines from source") do |b|
-    options[:dejunk] = true
-  end
-
-  opts.on("--by FIELD", String, "(for *maxrows* only) Field to group by") do |f|
-    options[:by] = f
-  end
-
-  opts.on("--max FIELD", String, "(for *maxrows* only) Field to find max value for") do |f|
-    options[:max] = f
-  end
-end.parse!
-
-filename = ARGV.shift # use stdin if no filename provided
-
-def logmessage(command, options, results)
-  $stderr.puts <<-EOT
-* masticate #{command} (#{options.keys.join(', ')})
-  Lines in input: #{results[:input_count]}
-  Lines in output: #{results[:output_count]}
-EOT
-  if results[:field_counts]
-    $stderr.puts "  Field counts: #{results[:field_counts].inspect}"
-  end
-end
-
-case command
-when 'sniff'
-  results = Masticate.sniff(filename, options)
-  col_sep = results[:col_sep]
-  col_sep = "TAB" if col_sep == "\t"
-  quote_char = results[:quote_char] || "NONE"
-  $stderr.puts <<-EOT
-Processing complete.
-  Input delimiter: #{col_sep}
-  Quote char: #{quote_char}
-  Field counts: #{results[:field_counts].inspect}
-  Headers: #{results[:headers].join(',')}
-EOT
-
-when 'mend'
-  results = Masticate.mend(filename, options)
-  logmessage(command, options, results)
-
-when 'csvify'
-  results = Masticate.csvify(filename, options)
-  logmessage(command, options, results)
-
-when 'pluck'
-  results = Masticate.pluck(filename, options)
-  logmessage(command, options, results)
-
-when 'datify'
-  results = Masticate.datify(filename, options)
-  logmessage(command, options, results)
-
-when 'gsub'
-  results = Masticate.gsub(filename, options)
-  logmessage(command, options, results)
-
-when 'maxrows'
-  results = Masticate.maxrows(filename, options)
-  logmessage(command, options, results)
-
-else
-  raise "unknown command #{command}"
-end
+masticator.execute(command, options, filenames)

data/lib/masticate/base.rb

@@ -4,8 +4,15 @@ class Masticate::Base
   attr_reader :input_count, :output_count
   attr_reader :csv_options
 
-  def initialize(filename)
-    @filename = filename
+  def initialize(args)
+    case args
+    when String
+      @filename = args
+    when Hash
+      configure(args)
+    else
+      raise "invalid initialization: #{args}"
+    end
   end
 
   def with_input
@@ -40,4 +47,8 @@ class Masticate::Base
       @csv_options[:quote_char] = opts[:quote_char] || "\0"
     end
   end
+
+  # def crunch(row)
+  #   # noop
+  # end
 end

data/lib/masticate/concat.rb

@@ -0,0 +1,21 @@
+# concatenate input files:
+# * assuming that each input file has a single header line
+# * writing a single header line to the output (just use the header line from the first file)
+# * trying that all the files have the same format (no validation)
+
+class Masticate::Concat #< Masticate::Base
+  def initialize(filenames)
+    @filenames = filenames
+  end
+
+  def concat(opts)
+    File.unlink(opts[:output]) if opts[:output] && File.exists?(opts[:output])
+    redirect = ">>#{opts[:output]}" if opts[:output]
+    
+    file1, *rest = @filenames
+    system "cat #{file1} #{redirect}"
+    rest.each do |file|
+      system "tail +2 #{file} #{redirect}"
+    end
+  end
+end

data/lib/masticate/cook.rb

@@ -0,0 +1,48 @@
+# cook up a recipe
+# * single file as input
+# * recipe from a file
+# * multiple steps
+# * single output
+
+require "shellwords"
+
+class Masticate::Cook < Masticate::Base
+  def initialize(filename)
+    @filename = filename
+  end
+
+  def cook(opts)
+    recipefile = opts[:recipe] or raise "missing recipe for cook"
+    recipe = File.read(recipefile).lines
+    standard_options(opts)
+
+    steps = recipe.map do |step|
+      # puts step
+      argv = Shellwords.split(step)
+      masticator = Masticate::MyOptionParser.new
+      command, options = masticator.parse(argv)
+      masticator.prepare(command, options)
+    end
+
+    @output_count = 0
+    headers = nil
+    with_input do |input|
+      while line = get
+        row = CSV.parse_line(line, csv_options)
+
+        steps.each do |step|
+          # puts "APPLY #{step} to #{row}"
+          row = step.crunch(row)
+        end
+
+        emit(row.to_csv) if row
+      end
+    end
+    @output.close if opts[:output]
+
+    {
+      :input_count => @input_count,
+      :output_count => @output_count
+    }
+  end
+end

data/lib/masticate/gsubber.rb

@@ -2,13 +2,21 @@
 require "csv"
 
 class Masticate::Gsubber < Masticate::Base
-  def gsub(opts)
+  def configure(opts)
     standard_options(opts)
 
-    field = opts[:field] or raise "missing field to gsub"
-    from = Regexp.new(opts[:from]) or raise "Invalid regex '#{opts[:from]}' for conversion"
-    to = opts[:to] or raise "missing 'to' string for gsub"
+    @field = opts[:field] or raise "missing field to gsub"
+    @from = Regexp.new(opts[:from]) or raise "Invalid regex '#{opts[:from]}' for conversion"
+    @to = opts[:to] or raise "missing 'to' string for gsub"
+  end
+
+  def set_headers(row)
+    @headers = row
+    @index = @headers.index(@field) or raise "Unable to find column '#{@field}' in headers"
+  end
 
+  def gsub(opts)
+    configure(opts)
     @output_count = 0
     headers = nil
     with_input do |input|
@@ -16,11 +24,11 @@ class Masticate::Gsubber < Masticate::Base
         row = CSV.parse_line(line, csv_options)
         if !headers
           headers = row
-          index = headers.index(field) or raise "Unable to find column '#{field}' in headers"
+          index = headers.index(@field) or raise "Unable to find column '#{@field}' in headers"
           emit(line)
         else
           oldval = row[index]
-          newval = oldval.gsub(from, to)
+          newval = oldval.gsub(@from, @to)
           row[index] = newval
           emit(row.to_csv)
         end
@@ -33,4 +41,13 @@ class Masticate::Gsubber < Masticate::Base
       :output_count => @output_count
     }
   end
+
+  def crunch(row)
+    if !@headers
+      set_headers(row)
+    else
+      row[@index] = row[@index].gsub(@from, @to)
+    end
+    row
+  end
 end

data/lib/masticate/max_rows.rb

@@ -2,11 +2,15 @@
 require "csv"
 
 class Masticate::MaxRows < Masticate::Base
-  def maxrows(opts)
+  def configure(opts)
     standard_options(opts)
 
-    groupby = opts[:by] or raise "missing field to group by"
-    maxon = opts[:max] or raise "missing field to max on"
+    @groupby = opts[:by] or raise "missing field to group by"
+    @maxon = opts[:max] or raise "missing field to max on"
+  end
+
+  def maxrows(opts)
+    configure(opts)
 
     @output_count = 0
     headers = nil
@@ -16,8 +20,8 @@ class Masticate::MaxRows < Masticate::Base
         row = CSV.parse_line(line, csv_options)
         if !headers
           headers = row
-          index_by = headers.index(groupby) or raise "Unable to find column '#{groupby}'"
-          index_max = headers.index(maxon) or raise "Unable to find column '#{maxon}'"
+          index_by = headers.index(@groupby) or raise "Unable to find column '#{@groupby}'"
+          index_max = headers.index(@maxon) or raise "Unable to find column '#{@maxon}'"
           emit(line)
         else
           key = row[index_by]
@@ -45,4 +49,30 @@ class Masticate::MaxRows < Masticate::Base
       :output_count => @output_count
     }
   end
+
+  def crunch(row)
+    if !@headers
+      @headers = row
+      @index_by = row.index(@groupby) or raise "Unable to find column '#{@groupby}'"
+      @index_max = row.index(@maxon) or raise "Unable to find column '#{@maxon}'"
+      @accum = {}
+      row
+    elsif row.nil?
+      # output the accumulated results
+      @accum.each do |k,row|
+        emit(row.to_csv)
+      end
+    else
+      key = row[@index_by]
+      if !@accum[key]
+        @accum[key] = row
+      else
+        oldscore = @accum[key][@index_max]
+        newscore = row[@index_max]
+        if newscore > oldscore
+          @accum[key] = row
+        end
+      end
+    end
+  end
 end

data/lib/masticate/myoptparse.rb

@@ -0,0 +1,163 @@
+require "optparse"
+
+class Masticate::MyOptionParser
+  attr_reader :command, :options
+
+  def initialize
+    @options = {}
+    @parser = OptionParser.new do |opts|
+      opts.banner = "Usage: masticate [command] [options]"
+
+      opts.on("--output FILENAME", String, "Redirect output from stdout to file") do |f|
+        @options[:output] = f
+      end
+
+      opts.on("--format FORMAT", String, "Specify format") do |v|
+        @options[:format] = v
+      end
+
+      opts.on("--delim DELIMITER", String, "Specify field delimiter (character or TAB; default is ',')") do |v|
+        @options[:col_sep] = v
+        @options[:col_sep] = "\t" if @options[:col_sep] == "TAB"
+      end
+
+      opts.on("--quote QUOTE-CHAR", String, "Specify character used for quoting fields (optional; default is no quoting)") do |char|
+        @options[:quote_char] = char
+      end
+
+      opts.on("--stats", "(for *sniff*) collect & display input stats") do
+        @options[:stats] = true
+      end
+
+      opts.on("--fields LIST", Array, "Specify fields to select") do |list|
+        @options[:fields] = list
+      end
+
+      opts.on("--field FIELD", String, "Specify field to convert") do |f|
+        @options[:field] = f
+      end
+
+      opts.on("--snip DIRECTIVE", String, "Specify header fields to snip: first N, or by name") do |f|
+        @options[:snip] = f.to_i
+      end
+
+      opts.on("--from REGEXP", String, "Regular expression for gsub conversion") do |s|
+        @options[:from] = s
+      end
+
+      # if I specify String here, then a blank string '' is considered invalid and triggers an exception.
+      opts.on("--to STRING", "Result string for gsub conversion") do |s|
+        @options[:to] = s
+      end
+
+      opts.on("--inlined", "(for *mend* only) Source file has headers inlined on each line") do |b|
+        @options[:inlined] = true
+      end
+
+      opts.on("--dejunk", "(for *mend* only) Expunge junk lines from source") do |b|
+        @options[:dejunk] = true
+      end
+
+      opts.on("--by FIELD", String, "(for *maxrows* only) Field to group by") do |f|
+        @options[:by] = f
+      end
+
+      opts.on("--max FIELD", String, "(for *maxrows* only) Field to find max value for") do |f|
+        @options[:max] = f
+      end
+
+      opts.on("--recipe FILENAME", String, "(*cook* only) Recipe file") do |f|
+        @options[:recipe] = f
+      end
+    end
+  end
+
+  def parse(argv = ARGV)
+    @command = argv.shift
+    filenames = @parser.parse(argv)
+    # argv remnants are filenames
+    [@command, @options, filenames]
+  end
+
+  def prepare(command, options)
+    klasses = {
+      'gsub' => Masticate::Gsubber,
+      'datify' => Masticate::Datify,
+      'maxrows' => Masticate::MaxRows,
+      'relabel' => Masticate::Relabel,
+      'pluck' => Masticate::Plucker
+    }
+
+    klass = klasses[command]
+    klass.new(options)
+  end
+
+  def execute(command, options, filenames = nil)
+    filename = filenames.first
+
+    case command
+    when 'sniff'
+      results = Masticate.sniff(filename, options)
+      col_sep = results[:col_sep]
+      col_sep = "TAB" if col_sep == "\t"
+      quote_char = results[:quote_char] || "NONE"
+      $stderr.puts <<-EOT
+Processing complete.
+  Input delimiter: #{col_sep}
+  Quote char: #{quote_char}
+  Field counts: #{results[:field_counts].inspect}
+  Headers: #{results[:headers].join(',')}
+EOT
+
+    when 'mend'
+      results = Masticate.mend(filename, options)
+      logmessage(command, options, results)
+
+    when 'csvify'
+      results = Masticate.csvify(filename, options)
+      logmessage(command, options, results)
+
+    when 'pluck'
+      results = Masticate.pluck(filename, options)
+      logmessage(command, options, results)
+
+    when 'datify'
+      results = Masticate.datify(filename, options)
+      logmessage(command, options, results)
+
+    when 'gsub'
+      results = Masticate.gsub(filename, options)
+      logmessage(command, options, results)
+
+    when 'maxrows'
+      results = Masticate.maxrows(filename, options)
+      logmessage(command, options, results)
+
+    when 'concat'
+      results = Masticate.concat(ARGV, options)
+      # logmessage(command, options, results)
+
+    when 'relabel'
+      results = Masticate.relabel(filename, options)
+      # logmessage(command, options, results)
+
+    when 'cook'
+      results = Masticate.cook(filename, options)
+      logmessage(command, options, results)
+
+    else
+      raise "unknown command #{command}"
+    end
+  end
+
+  def logmessage(command, options, results)
+    $stderr.puts <<-EOT
+* masticate #{command} (#{options.keys.join(', ')})
+  Lines in input: #{results[:input_count]}
+  Lines in output: #{results[:output_count]}
+EOT
+    if results[:field_counts]
+      $stderr.puts "  Field counts: #{results[:field_counts].inspect}"
+    end
+  end
+end

data/lib/masticate/plucker.rb

@@ -2,10 +2,17 @@
 require "csv"
 
 class Masticate::Plucker < Masticate::Base
-  def pluck(opts)
+  def configure(opts)
     standard_options(opts)
 
-    fields = opts[:fields] or raise "missing fields to pluck"
+    @fields = opts[:fields] or raise "missing fields to pluck"
+  end
+
+  def pluck(opts)
+    configure(opts)
+    # standard_options(opts)
+    # 
+    # fields = opts[:fields] or raise "missing fields to pluck"
 
     @output_count = 0
     headers = nil
@@ -14,7 +21,7 @@ class Masticate::Plucker < Masticate::Base
         row = CSV.parse_line(line, csv_options)
         if !headers
           headers = row
-          indexes = fields.map do |f|
+          indexes = @fields.map do |f|
             case f
             when String
               headers.index(f) or raise "Unable to find column '#{f}'"
@@ -41,4 +48,27 @@ class Masticate::Plucker < Masticate::Base
       :output_count => @output_count
     }
   end
+
+  def crunch(row)
+    if !@headers
+      @headers = row
+      @indexes = @fields.map do |f|
+        case f
+        when String
+          row.index(f) or raise "Unable to find column '#{f}'"
+        when Fixnum
+          if f > row.count
+            raise "Cannot pluck column #{f}, there are only #{row.count} fields"
+          else
+            f-1
+          end
+        else
+          raise "Invalid field descriptor '#{f}'"
+        end
+      end
+    end
+
+    # output is just the selected columns
+    @indexes.map {|i| row[i]}
+  end
 end

data/lib/masticate/relabel.rb

@@ -0,0 +1,44 @@
+# relabel a single input file
+# * assuming that input file has a single header line
+# * assuming that input file is in valid CSV format (no validation)
+
+class Masticate::Relabel < Masticate::Base
+  def configure(opts)
+    standard_options(opts)
+
+    @fields = opts[:fields] or raise "missing fieldnames for relabel"
+  end
+
+  def relabel(opts)
+    configure(opts)
+
+    @output_count = 0
+    headers = nil
+    with_input do |input|
+      while line = get
+        row = CSV.parse_line(line, csv_options)
+        if !headers
+          headers = @fields
+          emit(headers.to_csv)
+        else
+          emit(row.to_csv)
+        end
+      end
+    end
+    @output.close if opts[:output]
+
+    # File.unlink(opts[:output]) if opts[:output] && File.exists?(opts[:output])
+    # redirect = ">>#{opts[:output]}" if opts[:output]
+    # 
+    # system "/bin/echo -n '#{fields.to_csv}' #{redirect}"
+    # system "tail +2 #{@filename} #{redirect}"
+  end
+
+  def crunch(row)
+    if !@headers
+      @headers = @fields
+      row = @headers
+    end
+    row
+  end
+end

data/lib/masticate/sniffer.rb

@@ -4,7 +4,7 @@ class Masticate::Sniffer < Masticate::Base
   attr_reader :col_sep, :quote_char, :stats
   attr_reader :delimstats
 
-  CandidateDelimiters = [',', '|', "\t"]
+  CandidateDelimiters = [',', '|', "\t", "~"]
 
   def initialize(filename)
     @filename = filename

data/lib/masticate/version.rb

@@ -1,3 +1,3 @@
 module Masticate
-  VERSION = "0.1.5"
+  VERSION = "0.2"
 end

data/lib/masticate.rb

@@ -1,7 +1,10 @@
 require "open-uri"
+require "csv"
 
 require_relative "masticate/version"
 require_relative "masticate/base"
+require_relative "masticate/myoptparse"
+
 require_relative "masticate/sniffer"
 require_relative "masticate/mender"
 require_relative "masticate/csvify"
@@ -9,6 +12,9 @@ require_relative "masticate/plucker"
 require_relative "masticate/datify"
 require_relative "masticate/gsubber"
 require_relative "masticate/max_rows"
+require_relative "masticate/concat"
+require_relative "masticate/relabel"
+require_relative "masticate/cook"
 
 module Masticate
   def self.sniff(filename, opts = {})
@@ -38,4 +44,16 @@ module Masticate
   def self.maxrows(filename, opts)
     MaxRows.new(filename).maxrows(opts)
   end
+
+  def self.concat(filenames, opts)
+    Concat.new(filenames).concat(opts)
+  end
+
+  def self.relabel(filename, opts)
+    Relabel.new(filename).relabel(opts)
+  end
+
+  def self.cook(filename, opts)
+    Cook.new(filename).cook(opts)
+  end
 end

data/spec/data/concat_result.txt

@@ -0,0 +1,108 @@
+COL1	COL 2	Col 3   	col-4	   col5  	col6
+data	  data 	data  	d a t a	data	data
+data	  data 	data  	d a t a	data	data
+data	  data 	data  	d a t a	data	data
+data	  data 	data  	d a t a	data	data
+data|  data |data  |d a t a|data|data
+data|  data |data  |d a t a|data|data
+data|  data |data  |d a t a|data,data|data
+data|  data |data "more data"  |d a t a|data|data
+1,20120106003230,2044272,L,407,15267,407,201201060140,407,201201060140,0,201201060309,L,"594,756"
+1,20120106003230,2044277,X,407,15267,381,201201060222,381,201201060222,0,201201060647,X,"594,761"
+1,20120106003230,2044309,L,407,15267,407,201201060311,407,201201060311,0,201201060339,L,"594,766"
+1,20120106003230,,Q,407,15267,407,201201060514,108,201201060515,108,201201060515,SEC,"594,787"
+1,20120106024355,,Q,407,15267,407,201201060309,90,201201060316,90,201201060316,IV,"594,764"
+1,20120106024355,2044306,L,407,15267,407,201201060309,407,201201060309,0,201201060345,L,"594,763"
+1,20120106024355,2044308,X,407,15267,407,201201060310,407,201201060310,0,201201060556,X,"594,765"
+1,20120106024355,2044307,L,407,15267,407,201201060309,407,201201060309,0,201201060333,L,"594,762"
+1,20120106024355,,Q,407,15267,407,201201060520,108,201201060522,108,201201060522,SEC,"594,789"
+1,20120106024355,2044579,L,407,15267,68,201201060826,68,201201060826,0,201201071149,L,"594,823"
+1,20120106032719,2044345,L,407,15267,407,201201060348,407,201201060348,0,201201060442,L,"594,775"
+1,20120106032719,2044344,L,407,15267,407,201201060348,407,201201060348,0,201201060442,L,"594,777"
+1,20120106032719,2044343,L,407,15267,407,201201060348,407,201201060348,0,201201060428,L,"594,773"
+1,20120106032719,,Q,407,15267,407,201201060348,426,201201060408,426,201201060408,IV,"594,774"
+1,20120106032719,,Q,407,15267,407,201201060348,426,201201060634,426,201201060634,URINE,"594,776"
+1,20120106032719,2044386,L,407,15267,407,201201060445,407,201201060445,0,201201060519,L,"594,785"
+1,20120106032719,2044401,X,407,15267,407,201201060521,407,201201060521,0,201201060646,X,"594,790"
+1,20120106033235,,Q,407,15267,407,201201060347,74,201201060353,74,201201060353,IV,"594,769"
+1,20120106033235,2044349,L,407,15267,407,201201060347,74,201201060353,0,201201060443,L,"594,771"
+1,20120106033235,2044350,L,407,15267,407,201201060347,74,201201060353,0,201201060434,URINE,"594,770"
+1,20120106033235,2044347,L,407,15267,407,201201060347,74,201201060353,0,201201060428,L,"594,768"
+1,20120106033235,2044348,L,407,15267,407,201201060347,74,201201060353,0,201201060443,L,"594,772"
+1,20120106033235,2044372,X,407,15267,407,201201060429,407,201201060429,0,201201060649,X,"594,780"
+1,20120106035346,,Q,407,15267,407,201201060446,426,201201060448,426,201201060448,N,"594,786"
+1,20120106041426,2044383,L,407,15267,407,201201060445,407,201201060445,0,201201060657,L,"594,784"
+1,20120106041426,2044384,L,407,15267,407,201201060445,407,201201060445,0,201201060657,L,"594,782"
+1,20120106041426,2044382,L,407,15267,407,201201060445,407,201201060445,0,201201060522,L,"594,781"
+1,20120106041426,,Q,407,15267,407,201201060445,381,201201060452,381,201201060452,IV,"594,783"
+1,20120106043025,2044400,X,407,15267,407,201201060515,407,201201060515,0,201201060554,X,"594,788"
+1,20120106045326,2044411,R,407,15267,407,201201060535,407,201201060535,0,201201060630,RS,"594,791"
+1,20120106045326,,Q,407,15267,407,201201060535,108,201201060540,108,201201060540,SEC,"594,794"
+1,20120106045326,2044412,R,407,15267,407,201201060535,407,201201060535,0,201201060629,RS,"594,795"
+1,20120106045326,2044413,X,407,15267,407,201201060536,407,201201060536,0,201201060649,X,"594,796"
+1,20120106045326,,Q,407,15267,407,201201060535,108,201201060541,108,201201060541,SEC,"594,792"
+1,20120106045326,2044410,R,407,15267,407,201201060535,407,201201060535,0,201201060628,RS,"594,793"
+1,20120106052714,2044421,L,407,15267,407,201201060544,407,201201060544,0,201201060605,L,"594,797"
+1,20120106052714,,Q,407,15267,407,201201060544,90,201201060545,90,201201060545,IV,"594,799"
+1,20120106052714,,Q,407,15267,407,201201060544,90,201201060545,90,201201060545,N,"594,800"
+1,20120106052714,2044422,L,407,15267,407,201201060544,407,201201060544,0,201201060621,L,"594,801"
+1,20120106052714,2044423,L,407,15267,407,201201060544,407,201201060544,0,201201060727,L,"594,798"
+1,20120106052714,2044424,L,407,15267,407,201201060551,407,201201060551,0,201201060714,L,"594,802"
+1,20120106070243,2044439,L,504,15550,504,201201060721,504,201201060721,0,201201060753,L,"594,803"
+1,20120106070243,2044440,L,504,15550,504,201201060721,504,201201060721,0,201201060748,L,"594,807"
+1,20120106070243,2044441,L,504,15550,504,201201060721,504,201201060721,0,201201060748,L,"594,806"
+1,20120106070243,,Q,504,15550,504,201201060721,155,201201060735,155,201201060735,IV,"594,805"
+1,20120106070243,,Q,504,15550,504,201201060806,155,201201060813,155,201201060813,N,"594,820"
+1,20120106070243,2044524,L,504,15550,504,201201060806,504,201201060806,0,201201061004,L,"594,816"
+1,20120106070243,,Q,504,15550,504,201201060807,195,201201060813,195,201201060813,SEC,"594,822"
+1,20120106070243,2044522,L,504,15550,504,201201060806,504,201201060806,0,201201060959,L,"594,819"
+1,20120106070243,,Q,504,15550,504,201201060807,195,201201060811,195,201201060811,SEC,"594,821"
+1,20120106070243,,Q,504,15550,504,201201060806,155,201201060813,155,201201060813,N,"594,818"
+1,20120106070243,,Q,504,15550,504,201201060910,155,201201060916,155,201201060916,N,"594,831"
+1,20120106070243,2044716,X,504,15550,504,201201060928,504,201201060928,0,201201060953,X,"594,834"
+1,20120106073142,2044480,X,504,15550,504,201201060757,504,201201060757,0,201201060819,X,"594,815"
+1,20120106073757,2044475,L,504,15550,504,201201060749,155,201201060755,0,201201060925,URINE,"594,810"
+1,20120106073757,2044466,L,504,15550,504,201201060749,504,201201060749,0,201201060827,L,"594,808"
+1,20120106073757,2044470,X,504,15550,504,201201060749,504,201201060749,0,201201060818,X,"594,809"
+1,20120106073757,2044467,L,504,15550,504,201201060749,504,201201060749,0,201201060826,L,"594,813"
+1,20120106073757,2044468,L,504,15550,504,201201060749,504,201201060749,0,201201060839,L,"594,811"
+1,20120106073757,2044469,L,504,15550,504,201201060749,504,201201060749,0,201201060825,L,"594,814"
+1,20120106073757,,Q,504,15550,504,201201060749,155,201201060755,155,201201060755,IV,"594,812"
+1,20120106073757,,Q,504,15550,504,201201060911,76,201201060933,76,201201060933,IV,"594,832"
+1,20120106073757,,Q,504,15550,504,201201060928,34,201201060934,34,201201060934,SEC,"594,833"
+1,20120106073757,,Q,504,15550,504,201201061022,155,201201061108,155,201201061108,IV,"594,862"
+1,20120106073757,,Q,504,15550,504,201201061019,155,201201061025,155,201201061025,IV,"594,861"
+1,20120106073757,,Q,504,15550,504,201201061131,195,201201061133,195,201201061133,SEC,"594,896"
+1,20120106073757,,Q,504,15550,504,201201061131,195,201201061133,195,201201061133,SEC,"594,895"
+1,20120106073757,2045028,X,504,15550,504,201201061131,504,201201061131,0,201201061209,X,"594,898"
+1,20120106073757,2045029,X,504,15550,504,201201061131,504,201201061131,0,201201061345,X,"594,897"
+1,20120106073757,,Q,504,15550,504,201201061131,155,201201061223,155,201201061223,N,"594,894"
+1,20120106084347,2044639,X,504,15550,76,201201060850,76,201201060850,0,201201060931,X,"594,828"
+1,20120106084720,2044670,X,55,4644,55,201201060909,55,201201060909,0,201201060934,X,"594,829"
+1,20120106084720,,Q,55,4644,55,201201060910,66,201201060914,66,201201060914,N,"594,830"
+1,20120106085558,2044755,L,55,4644,55,201201060949,55,201201060949,0,201201061018,L,"594,846"
+1,20120106085558,2044756,L,55,4644,55,201201060949,55,201201060949,0,201201061038,L,"594,851"
+1,20120106085558,2044793,L,55,4644,55,201201060949,76,201201061003,0,201201061239,URINE,"594,848"
+1,20120106085558,,Q,55,4644,55,201201060949,76,201201061003,76,201201061003,IV,"594,850"
+1,20120106085558,,Q,55,4644,55,201201060949,76,201201061003,76,201201061003,IV,"594,847"
+1,20120106085558,2044757,L,55,4644,55,201201060949,55,201201060949,0,201201061040,L,"594,849"
+1,20120106085558,2044843,L,55,4644,55,201201061033,55,201201061033,0,201201071505,L,"594,864"
+1,20120106085558,2044841,X,55,4644,55,201201061032,55,201201061032,0,201201061136,X,"594,863"
+1,20120106085558,2044844,L,55,4644,55,201201061033,55,201201061033,0,201201061119,L,"594,865"
+1,20120106085558,,Q,55,4644,55,201201061228,195,201201061240,195,201201061240,SEC,"594,961"
+1,20120106091726,2044741,L,504,15550,504,201201060942,504,201201060942,0,201201061024,L,"594,839"
+1,20120106091726,2044745,X,504,15550,504,201201060942,504,201201060942,0,201201061016,X,"594,835"
+1,20120106091726,2044746,L,504,15550,504,201201060942,504,201201060942,0,201201061107,L,"594,842"
+1,20120106091726,2044740,L,504,15550,504,201201060942,504,201201060942,0,201201061017,L,"594,836"
+1,20120106091726,2044744,L,504,15550,504,201201060942,504,201201060942,0,201201061024,L,"594,838"
+1,20120106091726,2044742,L,504,15550,504,201201060942,504,201201060942,0,201201061016,L,"594,841"
+1,20120106091726,,Q,504,15550,504,201201060942,66,201201060944,66,201201060944,IV,"594,837"
+1,20120106091726,2044743,L,504,15550,504,201201060942,504,201201060942,0,201201061016,L,"594,840"
+1,20120106095129,2044814,X,55,4644,55,201201061010,55,201201061010,0,201201061037,X,"594,853"
+1,20120106100014,,Q,504,15550,504,201201061011,885,201201061037,885,201201061037,IV,"594,857"
+1,20120106100014,,Q,504,15550,504,201201061011,885,201201061037,885,201201061037,N,"594,858"
+1,20120106100014,,Q,504,15550,504,201201061011,885,201201061037,885,201201061037,N,"594,859"
+1,20120106100014,2044815,L,504,15550,504,201201061011,504,201201061011,0,201201061023,L,"594,854"
+1,20120106100014,2044817,L,504,15550,504,201201061011,504,201201061011,0,201201061049,L,"594,856"
+1,20120106100014,2044818,X,504,15550,504,201201061011,504,201201061011,0,201201061038,X,"594,855"
+1,20120106100014,2044816,L,504,15550,504,201201061011,504,201201061011,0,201201061049,L,"594,860"

data/spec/data/cooking_result.csv

@@ -0,0 +1,41 @@
+two,eight,fourteen
+20120106003230,201201060140,594756
+20120106003230,201201060222,594761
+20120106003230,201201060311,594766
+20120106003230,201201060514,594787
+20120106024355,201201060309,594764
+20120106024355,201201060310,594765
+20120106024355,201201060520,594789
+20120106024355,201201060826,594823
+20120106032719,201201060348,594775
+20120106032719,201201060445,594785
+20120106032719,201201060521,594790
+20120106033235,201201060347,594769
+20120106033235,201201060429,594780
+20120106035346,201201060446,594786
+20120106041426,201201060445,594784
+20120106043025,201201060515,594788
+20120106045326,201201060535,594791
+20120106045326,201201060536,594796
+20120106052714,201201060544,594797
+20120106052714,201201060551,594802
+20120106070243,201201060721,594803
+20120106070243,201201060806,594820
+20120106070243,201201060807,594822
+20120106070243,201201060910,594831
+20120106070243,201201060928,594834
+20120106073142,201201060757,594815
+20120106073757,201201060749,594810
+20120106073757,201201060911,594832
+20120106073757,201201060928,594833
+20120106073757,201201061022,594862
+20120106073757,201201061131,594896
+20120106084347,201201060850,594828
+20120106084720,201201060909,594829
+20120106084720,201201060910,594830
+20120106085558,201201060949,594846
+20120106085558,201201061033,594864
+20120106085558,201201061228,594961
+20120106091726,201201060942,594839
+20120106095129,201201061010,594853
+20120106100014,201201061011,594857

data/spec/data/recipe.txt

@@ -0,0 +1,4 @@
+gsub --field order_number --from ',|(.00$)' --to ''
+relabel --fields one,two,three,four,five,six,seven,eight,nine,ten,eleven,twelve,thirteen,fourteen
+pluck --fields two,eight,fourteen
+maxrows --by two --max eight

data/spec/data/relabel_result.csv

@@ -0,0 +1,4 @@
+happy,birth,day,to,you
+data1,data2,data3,data4,data5
+111,22,333,44,555
+91,92,93,94,95

data/spec/data/tilde_data.txt

@@ -0,0 +1,5 @@
+COL1~COL 2~Col 3   ~col-4~   col5  ~col6
+data~  data ~data  ~d a t a~data~data
+data~  data ~data  ~d a t a~data~data
+data~  data ~data  ~d a t a~data,data~data
+data~  data ~data "more data"  ~d a t a~data~data

data/spec/lib/concat_spec.rb

@@ -0,0 +1,17 @@
+# spec for file concatenation
+
+require "spec_helper"
+
+describe "concatenation" do
+  it "should leave just one header row in the result" do
+    file1 = File.dirname(__FILE__) + "/../data/tabbed_data.txt"
+    file2 = File.dirname(__FILE__) + "/../data/pipe_data.txt"
+    file3 = File.dirname(__FILE__) + "/../data/quoted_csv_data.txt"
+    tmp = Tempfile.new('concat')
+    results = Masticate.concat([file1, file2, file3], :output => tmp.path)
+    output = File.read(tmp)
+    tmp.unlink
+    correct_output = File.read(File.dirname(__FILE__) + "/../data/concat_result.txt")
+    output.should == correct_output
+  end
+end

data/spec/lib/cook_spec.rb

@@ -0,0 +1,16 @@
+# spec for cookery
+
+require "spec_helper"
+
+describe "cooking up a recipe" do
+  it "should handle multiple steps" do
+    input = File.dirname(__FILE__) + "/../data/quoted_csv_data.txt"
+    recipe = File.dirname(__FILE__) + "/../data/recipe.txt"
+    tmp = Tempfile.new('cooked')
+    results = Masticate.cook(input, :output => tmp, :recipe => recipe)
+    output = File.read(tmp)
+    correct_output = File.read(File.dirname(__FILE__) + "/../data/cooking_result.csv")
+  
+    output.should == correct_output
+  end
+end

data/spec/lib/gsub_spec.rb

@@ -1,7 +1,6 @@
 # spec for field regexp conversion
 
 require "spec_helper"
-require "tempfile"
 
 describe "gsubbing" do
   it "should apply conversion to a single column" do

data/spec/lib/relabel_spec.rb

@@ -0,0 +1,15 @@
+# spec for file concatenation
+
+require "spec_helper"
+
+describe "relabeling" do
+  it "result should be same as original" do
+    input = File.dirname(__FILE__) + "/../data/namedcols.csv"
+    tmp = Tempfile.new('relabel')
+    results = Masticate.relabel(input, :fields => %w{happy birth day to you}, :output => tmp.path)
+    output = File.read(tmp)
+    tmp.unlink
+    correct_output = File.read(File.dirname(__FILE__) + "/../data/relabel_result.csv")
+    output.should == correct_output
+  end
+end

data/spec/lib/sniffer_spec.rb

@@ -31,4 +31,11 @@ describe "delimiter sniffing" do
     results[:quote_char].should == '"'
     results[:field_counts].should == {14 => 100}
   end
+
+  it "should find tilde delimiter" do
+    filename = File.dirname(__FILE__) + "/../data/tilde_data.txt"
+    results = Masticate.sniff(filename, :stats => true)
+    results[:col_sep].should == '~'
+    results[:field_counts].should == {6 => 5}
+  end
 end

data/spec/spec_helper.rb

@@ -2,6 +2,7 @@
 # require 'rspec/rails'
 require 'rspec/autorun'
 # require 'capybara/rspec'
+require "tempfile"
 
 # Requires supporting ruby files with custom matchers and macros, etc,
 # in spec/support/ and its subdirectories.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: masticate
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: '0.2'
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-16 00:00:00.000000000 Z
+date: 2012-04-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2156618420 !ruby/object:Gem::Requirement
+  requirement: &2153649040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 2.9.0
   type: :development
   prerelease: false
-  version_requirements: *2156618420
+  version_requirements: *2153649040
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &2156617660 !ruby/object:Gem::Requirement
+  requirement: &2153648360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 0.7.0
   type: :development
   prerelease: false
-  version_requirements: *2156617660
+  version_requirements: *2153648360
 - !ruby/object:Gem::Dependency
   name: ruby_gntp
-  requirement: &2156616980 !ruby/object:Gem::Requirement
+  requirement: &2153647700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,7 +43,7 @@ dependencies:
         version: 0.3.4
   type: :development
   prerelease: false
-  version_requirements: *2156616980
+  version_requirements: *2153647700
 description: Data file crunching
 email:
 - jmay@pobox.com
@@ -61,18 +61,24 @@ files:
 - bin/masticate
 - lib/masticate.rb
 - lib/masticate/base.rb
+- lib/masticate/concat.rb
+- lib/masticate/cook.rb
 - lib/masticate/csvify.rb
 - lib/masticate/datify.rb
 - lib/masticate/gsubber.rb
 - lib/masticate/max_rows.rb
 - lib/masticate/mender.rb
+- lib/masticate/myoptparse.rb
 - lib/masticate/plucker.rb
+- lib/masticate/relabel.rb
 - lib/masticate/sniffer.rb
 - lib/masticate/version.rb
 - masticate.gemspec
 - spec/data/badnums.csv
 - spec/data/badnums_fixed.csv
 - spec/data/broken_psv.txt
+- spec/data/concat_result.txt
+- spec/data/cooking_result.csv
 - spec/data/events.csv
 - spec/data/events_reduced.csv
 - spec/data/inlined_headers.csv
@@ -83,12 +89,18 @@ files:
 - spec/data/namedcols.csv.output
 - spec/data/pipe_data.txt
 - spec/data/quoted_csv_data.txt
+- spec/data/recipe.txt
+- spec/data/relabel_result.csv
 - spec/data/tabbed_data.txt
+- spec/data/tilde_data.txt
+- spec/lib/concat_spec.rb
+- spec/lib/cook_spec.rb
 - spec/lib/csvify_spec.rb
 - spec/lib/gsub_spec.rb
 - spec/lib/maxrow_spec.rb
 - spec/lib/mender_spec.rb
 - spec/lib/plucker_spec.rb
+- spec/lib/relabel_spec.rb
 - spec/lib/sniffer_spec.rb
 - spec/spec_helper.rb
 homepage: ''
@@ -119,6 +131,8 @@ test_files:
 - spec/data/badnums.csv
 - spec/data/badnums_fixed.csv
 - spec/data/broken_psv.txt
+- spec/data/concat_result.txt
+- spec/data/cooking_result.csv
 - spec/data/events.csv
 - spec/data/events_reduced.csv
 - spec/data/inlined_headers.csv
@@ -129,12 +143,18 @@ test_files:
 - spec/data/namedcols.csv.output
 - spec/data/pipe_data.txt
 - spec/data/quoted_csv_data.txt
+- spec/data/recipe.txt
+- spec/data/relabel_result.csv
 - spec/data/tabbed_data.txt
+- spec/data/tilde_data.txt
+- spec/lib/concat_spec.rb
+- spec/lib/cook_spec.rb
 - spec/lib/csvify_spec.rb
 - spec/lib/gsub_spec.rb
 - spec/lib/maxrow_spec.rb
 - spec/lib/mender_spec.rb
 - spec/lib/plucker_spec.rb
+- spec/lib/relabel_spec.rb
 - spec/lib/sniffer_spec.rb
 - spec/spec_helper.rb
 has_rdoc: