masticate 0.3 → 0.3.1

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+rvm:
+  - 1.9.2
+  - 1.9.3
+  - jruby-19mode # JRuby in 1.9 mode
+  - rbx-19mode
+# uncomment this line if your project needs to run something other than `rake`:
+# script: bundle exec rspec spec

data/Rakefile

@@ -5,4 +5,4 @@ require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-# task :default => :spec
+task :default => :spec

data/lib/masticate/base.rb

@@ -25,8 +25,14 @@ class Masticate::Base
 
   def get
     line = @input.gets
-    @input_count += 1
-    line && line.chomp
+    return nil if line.nil?
+    line.chomp!
+    if line.empty?
+      get
+    else
+      @input_count += 1
+      line
+    end
   end
 
   def emit(line)

data/lib/masticate/concat.rb

@@ -15,7 +15,7 @@ class Masticate::Concat #< Masticate::Base
     file1, *rest = @filenames
     system "cat #{file1} #{redirect}"
     rest.each do |file|
-      system "tail +2 #{file} #{redirect}"
+      system "tail -n +2 #{file} | sed '/^$/d' #{redirect}"
     end
   end
 end

data/lib/masticate/cook.rb

@@ -18,7 +18,8 @@ class Masticate::Cook < Masticate::Base
     recipe = File.read(recipefile).lines
     standard_options(opts)
 
-    steps = recipe.map do |step|
+    # ignore blank lines in recipe file
+    steps = recipe.grep(/\S/).map do |step|
       argv = Shellwords.split(step)
       masticator = Masticate::MyOptionParser.new
       command, options = masticator.parse(argv)
@@ -38,9 +39,16 @@ class Masticate::Cook < Masticate::Base
         emit(row) if row
       end
     end
+    more_rows = []
     steps.each do |step|
-      step.crunch(nil) {|row| emit(row)}
+      if more_rows.any?
+        more_rows = more_rows.map {|row| step.crunch(row)}
+      else
+        step.crunch(nil) {|row| more_rows << row}
+      end
     end
+    more_rows.each {|row| emit(row)}
+    # step.crunch(nil) {|row| emit(row)}
 
     @output.close if opts[:output]
 

data/lib/masticate/sniffer.rb

@@ -25,7 +25,8 @@ class Masticate::Sniffer < Masticate::Base
   def find_col_sep
     @delimstats = {}
     with_input do |input|
-      input.lines.take(10).each do |line|
+      lines = 10.times.map{get}.compact
+      lines.each do |line|
         @line1 = line unless @line1
 
         CandidateDelimiters.each do |delim|
@@ -67,8 +68,12 @@ class Masticate::Sniffer < Masticate::Base
   end
 
   def stats
+    counts = Hash.new(0)
     with_input do |input|
-      input.lines.each_with_object(Hash.new(0)) {|line, counts| counts[CSV.parse_line(line, :col_sep => col_sep, :quote_char => quote_char || "\0").count] += 1}
+      while line = get
+        counts[CSV.parse_line(line, :col_sep => col_sep, :quote_char => quote_char || "\0").count] += 1
+      end
     end
+    counts
   end
 end

data/lib/masticate/version.rb

@@ -1,3 +1,3 @@
 module Masticate
-  VERSION = "0.3"
+  VERSION = "0.3.1"
 end

data/masticate.gemspec

@@ -16,6 +16,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = Masticate::VERSION
 
+  gem.add_development_dependency "rake", "~> 0.9.2"
   gem.add_development_dependency "rspec", "~> 2.9.0"
   gem.add_development_dependency "guard-rspec", "~> 0.7.0"
   gem.add_development_dependency "ruby_gntp", "~> 0.3.4"

data/spec/data/cookery_input.psv

@@ -0,0 +1,8 @@
+DTTM|AuditCode|AuditDesc|AuditEventTypeID|AuditByID
+3/12/2012  11:20:32AM|EXCEPTADD|Exception Added|10|9,123.00
+3/13/2012   8:12:39AM|EXCEPTADD|Exception Added|10|9,234.00
+3/13/2012   8:13:25AM|EXCEPTADD|Exception Added|10|9,123.00
+3/12/2012  11:20:32AM|EXCEPTADD|Exception Added|10|9,345.00
+3/12/2012  11:20:32AM|EXCEPTADD|Exception Added|10|9,234.00
+3/12/2012  11:20:32AM|EXCEPTADD|Exception Added|10|9,678.00
+3/12/2012  11:20:32AM|1

data/spec/data/cookery_result.csv

@@ -0,0 +1,5 @@
+id,ts,desc
+9123,1331626405,Exception Added
+9234,1331626359,Exception Added
+9345,1331551232,Exception Added
+9678,1331551232,Exception Added

data/spec/data/quoted_csv_data.txt

@@ -98,3 +98,4 @@ site,ibex,unit,face,doctor,hospid,usrorder,dteorder,usrsend,dtesend,usrdone,dted
 1,20120106100014,2044817,L,504,15550,504,201201061011,504,201201061011,0,201201061049,L,"594,856"
 1,20120106100014,2044818,X,504,15550,504,201201061011,504,201201061011,0,201201061038,X,"594,855"
 1,20120106100014,2044816,L,504,15550,504,201201061011,504,201201061011,0,201201061049,L,"594,860"
+

data/spec/data/recipe_cookery.txt

@@ -0,0 +1,6 @@
+exclude --field AuditCode --value '1'
+pluck --fields AuditByID,DTTM,AuditDesc
+datify --field DTTM --format "%m/%d/%Y %H:%M:%S%p"
+maxrows --by AuditByID --max DTTM
+gsub --field AuditByID --from ',|(.00$)' --to ''
+relabel --fields id,ts,desc

data/spec/data/recipe_mend.txt

@@ -1,3 +1,5 @@
 mend
 exclude --field 1 --value 'data2'
 pluck --fields 3
+
+

data/spec/lib/cook_spec.rb

@@ -24,4 +24,15 @@ describe "cooking up a recipe" do
   
     output.should == correct_output
   end
+
+  it "should do correct datify & gsub in recipe" do
+    input = File.dirname(__FILE__) + "/../data/cookery_input.psv"
+    recipe = File.dirname(__FILE__) + "/../data/recipe_cookery.txt"
+    tmp = Tempfile.new('cooked')
+    results = Masticate.cook(input, :col_sep => '|', :output => tmp, :recipe => recipe)
+    output = File.read(tmp)
+    correct_output = File.read(File.dirname(__FILE__) + "/../data/cookery_result.csv")
+  
+    output.should == correct_output
+  end
 end

data/spec/lib/csvify_spec.rb

@@ -10,7 +10,7 @@ describe "csvification" do
     output = File.read(tmp)
     tmp.unlink
     output.lines.count.should == 5
-    results[:input_count].should == 6
+    results[:input_count].should == 5
     results[:output_count].should == 5
   end
 end

data/spec/lib/mender_spec.rb

@@ -7,14 +7,14 @@ describe "mending" do
   it "should merge lines when delimiter counts don't match'" do
     filename = File.dirname(__FILE__) + "/../data/broken_psv.txt"
     results = Masticate.mend(filename, :col_sep => '|', :output => "/dev/null")
-    results[:input_count].should == 7
+    results[:input_count].should == 6
     results[:output_count].should == 5
   end
 
   it "should strip trailer records" do
     filename = File.dirname(__FILE__) + "/../data/junk_trailer.txt"
     results = Masticate.mend(filename, :col_sep => '|', :output => "/dev/null", :dejunk => true)
-    results[:input_count].should == 10
+    results[:input_count].should == 7
     results[:output_count].should == 5
     results[:headers].should == ['COL1', 'COL 2', 'Col 3', 'col-4', 'col5', 'col6']
   end
@@ -22,7 +22,7 @@ describe "mending" do
   it "should snip head fields" do
     filename = File.dirname(__FILE__) + "/../data/junk_header.csv"
     results = Masticate.mend(filename, :col_sep => ',', :snip => 1, :output => "/dev/null")
-    results[:input_count].should == 6
+    results[:input_count].should == 5
     results[:output_count].should == 5
     results[:headers].should == %w(hospid usrorder dteorder usrsend dtesend usrdone dtedone department)
   end
@@ -34,7 +34,7 @@ describe "mending" do
     output = File.read(tmp)
     correct_output = File.read(File.dirname(__FILE__) + "/../data/inlined_headers.csv.output")
 
-    results[:input_count].should == 11
+    results[:input_count].should == 10
     results[:output_count].should == 11
     # results[:field_counts].should == {11 => 11}
     output.should == correct_output

data/spec/lib/plucker_spec.rb

@@ -12,7 +12,7 @@ describe "plucker" do
     correct_output = File.read(File.dirname(__FILE__) + "/../data/namedcols.csv.output")
     tmp.unlink
 
-    results[:input_count].should == 5
+    results[:input_count].should == 4
     output.should == correct_output
   end
 
@@ -24,7 +24,7 @@ describe "plucker" do
     correct_output = File.read(File.dirname(__FILE__) + "/../data/namedcols.csv.output")
     tmp.unlink
 
-    results[:input_count].should == 5
+    results[:input_count].should == 4
     output.should == correct_output
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: masticate
 version: !ruby/object:Gem::Version
-  version: '0.3'
+  version: 0.3.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-11 00:00:00.000000000 Z
+date: 2012-05-30 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &2153051960 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: *2153051960
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2152079220 !ruby/object:Gem::Requirement
+  requirement: &2153051460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +32,10 @@ dependencies:
         version: 2.9.0
   type: :development
   prerelease: false
-  version_requirements: *2152079220
+  version_requirements: *2153051460
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &2152076120 !ruby/object:Gem::Requirement
+  requirement: &2153051000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +43,10 @@ dependencies:
         version: 0.7.0
   type: :development
   prerelease: false
-  version_requirements: *2152076120
+  version_requirements: *2153051000
 - !ruby/object:Gem::Dependency
   name: ruby_gntp
-  requirement: &2152074480 !ruby/object:Gem::Requirement
+  requirement: &2153050540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,7 +54,7 @@ dependencies:
         version: 0.3.4
   type: :development
   prerelease: false
-  version_requirements: *2152074480
+  version_requirements: *2153050540
 description: Data file crunching
 email:
 - jmay@pobox.com
@@ -53,6 +64,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - Gemfile
 - Guardfile
 - LICENSE
@@ -79,6 +91,8 @@ files:
 - spec/data/badnums_fixed.csv
 - spec/data/broken_psv.txt
 - spec/data/concat_result.txt
+- spec/data/cookery_input.psv
+- spec/data/cookery_result.csv
 - spec/data/cooking_mend_result.csv
 - spec/data/cooking_result.csv
 - spec/data/datify_input.csv
@@ -96,6 +110,7 @@ files:
 - spec/data/pipe_data.txt
 - spec/data/quoted_csv_data.txt
 - spec/data/recipe.txt
+- spec/data/recipe_cookery.txt
 - spec/data/recipe_mend.txt
 - spec/data/relabel_result.csv
 - spec/data/tabbed_data.txt
@@ -141,6 +156,8 @@ test_files:
 - spec/data/badnums_fixed.csv
 - spec/data/broken_psv.txt
 - spec/data/concat_result.txt
+- spec/data/cookery_input.psv
+- spec/data/cookery_result.csv
 - spec/data/cooking_mend_result.csv
 - spec/data/cooking_result.csv
 - spec/data/datify_input.csv
@@ -158,6 +175,7 @@ test_files:
 - spec/data/pipe_data.txt
 - spec/data/quoted_csv_data.txt
 - spec/data/recipe.txt
+- spec/data/recipe_cookery.txt
 - spec/data/recipe_mend.txt
 - spec/data/relabel_result.csv
 - spec/data/tabbed_data.txt