masticate 0.1.3 → 0.1.4

data/bin/masticate

@@ -49,6 +49,14 @@ OptionParser.new do |opts|
   opts.on("--dejunk", "(for *mend* only) Expunge junk lines from source") do |v|
     options[:dejunk] = v
   end
+
+  opts.on("--by FIELD", "(for *maxrows* only) Field to group by") do |f|
+    options[:by] = f
+  end
+
+  opts.on("--max FIELD", "(for *maxrows* only) Field to find max value for") do |f|
+    options[:max] = f
+  end
 end.parse!
 
 filename = ARGV.shift # use stdin if no filename provided
@@ -98,6 +106,10 @@ when 'gsub'
   results = Masticate.gsub(filename, options)
   logmessage(command, options, results)
 
+when 'maxrows'
+  results = Masticate.maxrows(filename, options)
+  logmessage(command, options, results)
+
 else
   raise "unknown command #{command}"
 end

data/lib/masticate/mender.rb

@@ -87,7 +87,7 @@ class Masticate::Mender < Masticate::Base
   end
 
   def explode(line)
-    CSV.parse_line(line, :col_sep => @col_sep, :quote_char => @quote_char)
+    CSV.parse_line(line, :col_sep => @col_sep, :quote_char => @quote_char).map {|s| s && s.strip}
   end
 
   # a line is "junky" if it has 2 or fewer fields with any content

data/lib/masticate/plucker.rb

@@ -14,8 +14,21 @@ class Masticate::Plucker < Masticate::Base
         row = CSV.parse_line(line, csv_options)
         if !headers
           headers = row
-          indexes = fields.map {|f| headers.index(f) or raise "Unable to find column '#{f}'"}
-          emit(fields.to_csv)
+          indexes = fields.map do |f|
+            case f
+            when String
+              headers.index(f) or raise "Unable to find column '#{f}'"
+            when Fixnum
+              if f > headers.count
+                raise "Cannot pluck column #{f}, there are only #{headers.count} fields"
+              else
+                f-1
+              end
+            else
+              raise "Invalid field descriptor '#{f}'"
+            end
+          end
+          emit(indexes.map {|i| headers[i]}.to_csv)
         else
           emit(indexes.map {|i| row[i]}.to_csv) if row
         end

data/lib/masticate/sniffer.rb

@@ -10,10 +10,10 @@ class Masticate::Sniffer < Masticate::Base
     @filename = filename
   end
 
-  def sniff
+  def sniff(opts)
     @col_sep = find_col_sep
     @quote_char = delimstats[@col_sep][:quote_char]
-    @stats = stats
+    @stats = stats if opts[:stats]
     {
       :col_sep => @col_sep,
       :quote_char => @quote_char,

data/lib/masticate/version.rb

@@ -1,3 +1,3 @@
 module Masticate
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

data/lib/masticate.rb

@@ -11,8 +11,8 @@ require_relative "masticate/gsubber"
 require_relative "masticate/max_rows"
 
 module Masticate
-  def self.sniff(filename)
-    Sniffer.new(filename).sniff
+  def self.sniff(filename, opts = {})
+    Sniffer.new(filename).sniff(opts)
   end
 
   def self.mend(filename, opts)

data/spec/data/inlined_headers.csv

@@ -1,10 +1,10 @@
-LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,WASHINGTON,GEORGE,D,824,9556,09/10/2005,07/01/2006,TM,Surgical House Staff,M,09/23/1975,3/10/2012
-LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,JEFFERSON,TOM,,621,8052,07/23/2001,01/28/2011,TM,Telemetry,F,12/24/1976,3/10/2012
+LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT # ,Empl # ,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,WASHINGTON,GEORGE,D,824,9556,09/10/2005,07/01/2006,TM,Surgical House Staff,M,09/23/1975,3/10/2012
+LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,JEFFERSON,TOM ,,621,8052,07/23/2001,01/28/2011,TM,Telemetry,F,12/24/1976,3/10/2012
 LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,ADAMS,JOHN,,655,8834,09/22/2003,,WA,6 East,F,08/07/1978,3/10/2012
 LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,ADAMS,JOHN QUINCY,A,209,8637,02/24/2003,12/02/2007,TM,Imaging Svcs - MRI,F,11/03/1966,3/10/2012
 LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,HAMILTON,ANDREW,,278,10065,01/09/2007,11/16/2007,TM,Information Technology,M,09/16/1968,3/10/2012
-LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,MADISON,JAMES,F,672,10720,01/05/2009,02/16/2010,TM,Rehab Svcs - Outpatients,F,04/15/1985,3/10/2012
+LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,MADISON ,JAMES,F,672,10720,01/05/2009,02/16/2010,TM,Rehab Svcs - Outpatients,F,04/15/1985,3/10/2012
 LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,FRANKLIN,BENJAMIN,R,674,8340,05/01/2002,09/01/2003,TM,"Rehab Svcs, xyz",F,03/15/1973,3/10/2012
 LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,LINCOLN,ABRAHAM,M,634,11340,05/02/2011,,PN,Sibley Ambulatory Surgery Ctr,F,07/11/1960,3/10/2012
-LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,MONROE,JAMES,L,614,10757,02/16/2009,,RF,Labor & Delivery,F,11/06/1983,3/10/2012
-LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,REVERE,PAUL,B,424,8568,11/18/2002,06/27/2006,TM,Laundry & Linen,M,12/31/1976,3/10/2012
+LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl # ,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,MONROE,JAMES,L,614,10757,02/16/2009,,RF,Labor & Delivery,F,11/06/1983,3/10/2012
+LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE,REVERE,PAUL,B,424,8568,11/18/2002,06/27/2006,TM,Laundry & Linen ,M,12/31/1976,3/10/2012

data/spec/data/inlined_headers.csv.output

@@ -1,4 +1,4 @@
-LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date ,Status,R_NAME,SEX,BIRTHDATE
+LAST_NAME,FIRST_NAME,MIDDLE_INIT,DEPT #,Empl #,DATE_HIRED,Term Date,Status,R_NAME,SEX,BIRTHDATE
 WASHINGTON,GEORGE,D,824,9556,09/10/2005,07/01/2006,TM,Surgical House Staff,M,09/23/1975
 JEFFERSON,TOM,,621,8052,07/23/2001,01/28/2011,TM,Telemetry,F,12/24/1976
 ADAMS,JOHN,,655,8834,09/22/2003,,WA,6 East,F,08/07/1978

data/spec/data/junk_header.csv

@@ -1,4 +1,4 @@
-3/7/2012,hospid,usrorder,dteorder,usrsend,dtesend,usrdone,dtedone,department
+3/7/2012,hospid,usrorder ,dteorder,usrsend,dtesend,usrdone,dtedone,department
 15267,407,201201060140,407,201201060140,0,201201060309,L
 15267,381,201201060222,381,201201060222,"abc,def",201201060647,X
 15267,407,201201060311,407,201201060311,0,201201060339,L

data/spec/lib/plucker_spec.rb

@@ -15,4 +15,16 @@ describe "plucker" do
     results[:input_count].should == 5
     output.should == correct_output
   end
+
+  it "should pull numbered columns starting at 1" do
+    filename = File.dirname(__FILE__) + "/../data/namedcols.csv"
+    tmp = Tempfile.new('plucker')
+    results = Masticate.pluck(filename, :output => tmp, :fields => [3,5])
+    output = File.read(tmp)
+    correct_output = File.read(File.dirname(__FILE__) + "/../data/namedcols.csv.output")
+    tmp.unlink
+
+    results[:input_count].should == 5
+    output.should == correct_output
+  end
 end

data/spec/lib/sniffer_spec.rb

@@ -3,23 +3,30 @@
 require "spec_helper"
 
 describe "delimiter sniffing" do
-  it "should find tab delimiter" do
+  it "stats collection should default off" do
     filename = File.dirname(__FILE__) + "/../data/tabbed_data.txt"
     results = Masticate.sniff(filename)
     results[:col_sep].should == "\t"
+    results[:field_counts].should be_nil
+  end
+
+  it "should find tab delimiter" do
+    filename = File.dirname(__FILE__) + "/../data/tabbed_data.txt"
+    results = Masticate.sniff(filename, :stats => true)
+    results[:col_sep].should == "\t"
     results[:field_counts].should == {6 => 5}
   end
 
   it "should find pipe delimiter" do
     filename = File.dirname(__FILE__) + "/../data/pipe_data.txt"
-    results = Masticate.sniff(filename)
+    results = Masticate.sniff(filename, :stats => true)
     results[:col_sep].should == '|'
     results[:field_counts].should == {6 => 5}
   end
 
   it "should recognize quotes in CSV sources" do
     filename = File.dirname(__FILE__) + "/../data/quoted_csv_data.txt"
-    results = Masticate.sniff(filename)
+    results = Masticate.sniff(filename, :stats => true)
     results[:col_sep].should == ','
     results[:quote_char].should == '"'
     results[:field_counts].should == {14 => 100}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: masticate
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-06 00:00:00.000000000 Z
+date: 2012-04-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2152447240 !ruby/object:Gem::Requirement
+  requirement: &2152726520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 2.9.0
   type: :development
   prerelease: false
-  version_requirements: *2152447240
+  version_requirements: *2152726520
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &2152446740 !ruby/object:Gem::Requirement
+  requirement: &2152749240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 0.7.0
   type: :development
   prerelease: false
-  version_requirements: *2152446740
+  version_requirements: *2152749240
 - !ruby/object:Gem::Dependency
   name: ruby_gntp
-  requirement: &2152446280 !ruby/object:Gem::Requirement
+  requirement: &2152748720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,7 +43,7 @@ dependencies:
         version: 0.3.4
   type: :development
   prerelease: false
-  version_requirements: *2152446280
+  version_requirements: *2152748720
 description: Data file crunching
 email:
 - jmay@pobox.com