masticate 0.2 → 0.2.1

data/lib/masticate/base.rb

@@ -30,9 +30,15 @@ class Masticate::Base
   end
 
   def emit(line)
+    @output_count ||= 0
     @output_count += 1
     begin
-      @output.puts line
+      case line
+      when Array
+        @output.puts line.to_csv
+      else
+        @output.puts line
+      end
     rescue Errno::EPIPE
       # output was closed, e.g. ran piped into `head`
       # silently ignore this condition, it's not fatal and doesn't need a warning
@@ -48,7 +54,23 @@ class Masticate::Base
     end
   end
 
-  # def crunch(row)
-  #   # noop
-  # end
+  def execute(opts)
+    configure(opts)
+
+    @output_count = 0
+    with_input do |input|
+      while line = get
+        row = CSV.parse_line(line, csv_options)
+        output = crunch(row)
+        emit(output) if output
+      end
+    end
+    crunch(nil) {|row| emit(row)}
+    @output.close if opts[:output]
+
+    {
+      :input_count => input_count,
+      :output_count => @output_count
+    }
+  end
 end

data/lib/masticate/cook.rb

@@ -12,6 +12,8 @@ class Masticate::Cook < Masticate::Base
   end
 
   def cook(opts)
+    standard_options(opts)
+
     recipefile = opts[:recipe] or raise "missing recipe for cook"
     recipe = File.read(recipefile).lines
     standard_options(opts)
@@ -31,13 +33,16 @@ class Masticate::Cook < Masticate::Base
         row = CSV.parse_line(line, csv_options)
 
         steps.each do |step|
-          # puts "APPLY #{step} to #{row}"
-          row = step.crunch(row)
+          row = step.crunch(row) if row
         end
 
-        emit(row.to_csv) if row
+        emit(row) if row
       end
     end
+    steps.each do |step|
+      step.crunch(nil) {|row| emit(row)}
+    end
+
     @output.close if opts[:output]
 
     {

data/lib/masticate/datify.rb

@@ -2,32 +2,23 @@
 require "csv"
 
 class Masticate::Datify < Masticate::Base
-  def datify(opts)
+  def configure(opts)
     standard_options(opts)
+    @field = opts[:field] or raise "missing field to datify"
+    @format = opts[:format] or raise "strptime format required for parsing timestamps"
+  end
 
-    field = opts[:field] or raise "missing field to datify"
-    format = opts[:format] or raise "strptime format required for parsing timestamps"
+  def datify(opts)
+    execute(opts)
+  end
 
-    @output_count = 0
-    headers = nil
-    with_input do |input|
-      while line = get
-        row = CSV.parse_line(line, csv_options)
-        if !headers
-          headers = row
-          index = headers.index(field) or raise "Unable to find column '#{field}'"
-          emit(headers.to_csv)
-        else
-          row[index] = DateTime.strptime(row[index], format).to_time.to_i rescue nil
-          emit(row.to_csv)
-        end
-      end
+  def crunch(row)
+    if !@index
+      @index = row.index(@field) or raise "Unable to find column '#{@field}'"
+    elsif row
+      ts = DateTime.strptime(row[@index], @format).to_time
+      row[@index] = ts.to_i rescue nil
     end
-    @output.close if opts[:output]
-
-    {
-      :input_count => @input_count,
-      :output_count => @output_count
-    }
+    row
   end
 end

data/lib/masticate/gsubber.rb

@@ -16,36 +16,13 @@ class Masticate::Gsubber < Masticate::Base
   end
 
   def gsub(opts)
-    configure(opts)
-    @output_count = 0
-    headers = nil
-    with_input do |input|
-      while line = get
-        row = CSV.parse_line(line, csv_options)
-        if !headers
-          headers = row
-          index = headers.index(@field) or raise "Unable to find column '#{@field}' in headers"
-          emit(line)
-        else
-          oldval = row[index]
-          newval = oldval.gsub(@from, @to)
-          row[index] = newval
-          emit(row.to_csv)
-        end
-      end
-    end
-    @output.close if opts[:output]
-
-    {
-      :input_count => input_count,
-      :output_count => @output_count
-    }
+    execute(opts)
   end
 
   def crunch(row)
     if !@headers
       set_headers(row)
-    else
+    elsif row
       row[@index] = row[@index].gsub(@from, @to)
     end
     row

data/lib/masticate/max_rows.rb

@@ -10,44 +10,7 @@ class Masticate::MaxRows < Masticate::Base
   end
 
   def maxrows(opts)
-    configure(opts)
-
-    @output_count = 0
-    headers = nil
-    accum = {}
-    with_input do |input|
-      while line = get
-        row = CSV.parse_line(line, csv_options)
-        if !headers
-          headers = row
-          index_by = headers.index(@groupby) or raise "Unable to find column '#{@groupby}'"
-          index_max = headers.index(@maxon) or raise "Unable to find column '#{@maxon}'"
-          emit(line)
-        else
-          key = row[index_by]
-          if !accum[key]
-            accum[key] = row
-          else
-            oldscore = accum[key][index_max]
-            newscore = row[index_max]
-            if newscore > oldscore
-              accum[key] = row
-            end
-          end
-        end
-      end
-    end
-
-    accum.each do |k,row|
-      emit(row.to_csv)
-    end
-
-    @output.close if opts[:output]
-
-    {
-      :input_count => @input_count,
-      :output_count => @output_count
-    }
+    execute(opts)
   end
 
   def crunch(row)
@@ -60,7 +23,7 @@ class Masticate::MaxRows < Masticate::Base
     elsif row.nil?
       # output the accumulated results
       @accum.each do |k,row|
-        emit(row.to_csv)
+        yield row
       end
     else
       key = row[@index_by]
@@ -73,6 +36,7 @@ class Masticate::MaxRows < Masticate::Base
           @accum[key] = row
         end
       end
+      nil
     end
   end
 end

data/lib/masticate/plucker.rb

@@ -9,44 +9,7 @@ class Masticate::Plucker < Masticate::Base
   end
 
   def pluck(opts)
-    configure(opts)
-    # standard_options(opts)
-    # 
-    # fields = opts[:fields] or raise "missing fields to pluck"
-
-    @output_count = 0
-    headers = nil
-    with_input do |input|
-      while line = get
-        row = CSV.parse_line(line, csv_options)
-        if !headers
-          headers = row
-          indexes = @fields.map do |f|
-            case f
-            when String
-              headers.index(f) or raise "Unable to find column '#{f}'"
-            when Fixnum
-              if f > headers.count
-                raise "Cannot pluck column #{f}, there are only #{headers.count} fields"
-              else
-                f-1
-              end
-            else
-              raise "Invalid field descriptor '#{f}'"
-            end
-          end
-          emit(indexes.map {|i| headers[i]}.to_csv)
-        else
-          emit(indexes.map {|i| row[i]}.to_csv) if row
-        end
-      end
-    end
-    @output.close if opts[:output]
-
-    {
-      :input_count => input_count,
-      :output_count => @output_count
-    }
+    execute(opts)
   end
 
   def crunch(row)
@@ -66,9 +29,10 @@ class Masticate::Plucker < Masticate::Base
           raise "Invalid field descriptor '#{f}'"
         end
       end
+      @indexes.map {|i| row[i]}
+    elsif row
+      # output is just the selected columns
+      @indexes.map {|i| row[i]}
     end
-
-    # output is just the selected columns
-    @indexes.map {|i| row[i]}
   end
 end

data/lib/masticate/relabel.rb

@@ -10,28 +10,7 @@ class Masticate::Relabel < Masticate::Base
   end
 
   def relabel(opts)
-    configure(opts)
-
-    @output_count = 0
-    headers = nil
-    with_input do |input|
-      while line = get
-        row = CSV.parse_line(line, csv_options)
-        if !headers
-          headers = @fields
-          emit(headers.to_csv)
-        else
-          emit(row.to_csv)
-        end
-      end
-    end
-    @output.close if opts[:output]
-
-    # File.unlink(opts[:output]) if opts[:output] && File.exists?(opts[:output])
-    # redirect = ">>#{opts[:output]}" if opts[:output]
-    # 
-    # system "/bin/echo -n '#{fields.to_csv}' #{redirect}"
-    # system "tail +2 #{@filename} #{redirect}"
+    execute(opts)
   end
 
   def crunch(row)

data/lib/masticate/version.rb

@@ -1,3 +1,3 @@
 module Masticate
-  VERSION = "0.2"
+  VERSION = "0.2.1"
 end

data/spec/data/cooking_result.csv

@@ -1,40 +1,18 @@
 two,eight,fourteen
-20120106003230,201201060140,594756
-20120106003230,201201060222,594761
-20120106003230,201201060311,594766
 20120106003230,201201060514,594787
-20120106024355,201201060309,594764
-20120106024355,201201060310,594765
-20120106024355,201201060520,594789
 20120106024355,201201060826,594823
-20120106032719,201201060348,594775
-20120106032719,201201060445,594785
 20120106032719,201201060521,594790
-20120106033235,201201060347,594769
 20120106033235,201201060429,594780
 20120106035346,201201060446,594786
 20120106041426,201201060445,594784
 20120106043025,201201060515,594788
-20120106045326,201201060535,594791
 20120106045326,201201060536,594796
-20120106052714,201201060544,594797
 20120106052714,201201060551,594802
-20120106070243,201201060721,594803
-20120106070243,201201060806,594820
-20120106070243,201201060807,594822
-20120106070243,201201060910,594831
 20120106070243,201201060928,594834
 20120106073142,201201060757,594815
-20120106073757,201201060749,594810
-20120106073757,201201060911,594832
-20120106073757,201201060928,594833
-20120106073757,201201061022,594862
 20120106073757,201201061131,594896
 20120106084347,201201060850,594828
-20120106084720,201201060909,594829
 20120106084720,201201060910,594830
-20120106085558,201201060949,594846
-20120106085558,201201061033,594864
 20120106085558,201201061228,594961
 20120106091726,201201060942,594839
 20120106095129,201201061010,594853

data/spec/data/datify_input.txt

@@ -0,0 +1,20 @@
+ChangeDTTM|AuditEventID|AuditEventCode|AuditEventTypeID|AuditEventName|AuditEventDesc|AuditByID
+12/31/2011   3:41:54PM|57,314,552.00|LOGGEDOUT|30|Logged out|Logged out|4,426.00
+1/8/2012   9:21:23AM|57,486,988.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
+1/8/2012   9:21:24AM|57,486,989.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
+2/19/2012   6:08:59PM|58,597,521.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
+2/19/2012   7:02:50PM|58,597,816.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
+2/19/2012   7:02:50PM|58,597,817.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
+2/4/2012   9:29:20AM|58,201,259.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
+2/4/2012  12:19:30AM|58,196,973.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
+2/4/2012  12:20:53AM|58,196,975.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
+2/3/2012  10:44:25PM|58,196,507.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
+1/20/2012   8:01:07PM|57,833,496.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
+1/22/2012   4:20:48AM|57,845,095.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
+1/22/2012   4:20:49AM|57,845,096.00|LOGGEDOUT|30|Logged out|Logged out|4,573.00
+1/14/2012   7:50:40AM|57,661,613.00|LOGGEDOUT|30|Logged out|Logged out|5,019.00
+2/26/2012   7:48:07AM|58,776,243.00|LOGGEDOUT|30|Logged out|Logged out|5,019.00
+2/23/2012   7:47:39AM|58,699,535.00|LOGGEDOUT|30|Logged out|Logged out|5,019.00
+2/23/2012   7:39:56AM|58,699,387.00|LOGGEDOUT|30|Logged out|Logged out|5,019.00
+2/23/2012   7:39:56AM|58,699,388.00|LOGGEDOUT|30|Logged out|Logged out|5,019.00
+2/15/2012   3:15:06PM|58,502,037.00|LOGGEDOUT|30|Logged out|Logged out|5,019.00

data/spec/data/recipe.txt

@@ -1,4 +1,5 @@
 gsub --field order_number --from ',|(.00$)' --to ''
+datify --field dtedone --format "%Y%m%d%H%M"
 relabel --fields one,two,three,four,five,six,seven,eight,nine,ten,eleven,twelve,thirteen,fourteen
 pluck --fields two,eight,fourteen
 maxrows --by two --max eight

data/spec/lib/csvify_spec.rb

@@ -1,7 +1,6 @@
 # spec for file-sniffing functions
 
 require "spec_helper"
-require "tempfile"
 
 describe "csvification" do
   it "should convert pipes to standard commas" do

data/spec/lib/datify_spec.rb

@@ -0,0 +1,16 @@
+# spec for data translation functions
+
+require "spec_helper"
+
+describe "csvification" do
+  it "should convert pipes to standard commas" do
+    filename = File.dirname(__FILE__) + "/../data/pipe_data.txt"
+    tmp = Tempfile.new('csvify')
+    results = Masticate.csvify(filename, :output => tmp, :col_sep => '|')
+    output = File.read(tmp)
+    tmp.unlink
+    output.lines.count.should == 5
+    results[:input_count].should == 6
+    results[:output_count].should == 5
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: masticate
 version: !ruby/object:Gem::Version
-  version: '0.2'
+  version: 0.2.1
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-04-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2153649040 !ruby/object:Gem::Requirement
+  requirement: &2157087600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 2.9.0
   type: :development
   prerelease: false
-  version_requirements: *2153649040
+  version_requirements: *2157087600
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &2153648360 !ruby/object:Gem::Requirement
+  requirement: &2157087060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 0.7.0
   type: :development
   prerelease: false
-  version_requirements: *2153648360
+  version_requirements: *2157087060
 - !ruby/object:Gem::Dependency
   name: ruby_gntp
-  requirement: &2153647700 !ruby/object:Gem::Requirement
+  requirement: &2157086580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,7 +43,7 @@ dependencies:
         version: 0.3.4
   type: :development
   prerelease: false
-  version_requirements: *2153647700
+  version_requirements: *2157086580
 description: Data file crunching
 email:
 - jmay@pobox.com
@@ -79,6 +79,7 @@ files:
 - spec/data/broken_psv.txt
 - spec/data/concat_result.txt
 - spec/data/cooking_result.csv
+- spec/data/datify_input.txt
 - spec/data/events.csv
 - spec/data/events_reduced.csv
 - spec/data/inlined_headers.csv
@@ -96,6 +97,7 @@ files:
 - spec/lib/concat_spec.rb
 - spec/lib/cook_spec.rb
 - spec/lib/csvify_spec.rb
+- spec/lib/datify_spec.rb
 - spec/lib/gsub_spec.rb
 - spec/lib/maxrow_spec.rb
 - spec/lib/mender_spec.rb
@@ -133,6 +135,7 @@ test_files:
 - spec/data/broken_psv.txt
 - spec/data/concat_result.txt
 - spec/data/cooking_result.csv
+- spec/data/datify_input.txt
 - spec/data/events.csv
 - spec/data/events_reduced.csv
 - spec/data/inlined_headers.csv
@@ -150,6 +153,7 @@ test_files:
 - spec/lib/concat_spec.rb
 - spec/lib/cook_spec.rb
 - spec/lib/csvify_spec.rb
+- spec/lib/datify_spec.rb
 - spec/lib/gsub_spec.rb
 - spec/lib/maxrow_spec.rb
 - spec/lib/mender_spec.rb