masticate 0.0.3 → 0.0.4

data/Guardfile

@@ -0,0 +1,10 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard 'rspec', :version => 2 do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+
+  watch(%r{^spec/support/(.+)\.rb$})                  { "spec" }
+end

data/bin/masticate

@@ -1,31 +1,25 @@
 #!/usr/bin/env ruby
 
-require "masticate"
+require_relative "../lib/masticate"
 
 command, filename = ARGV
 
 case ARGV.shift
 when 'sniff'
-  file = File.open(filename)
-  results = Masticate.sniff(file)
+  results = Masticate.sniff(filename)
   col_sep = results[:col_sep]
   col_sep = "TAB" if col_sep == "\t"
-  file.close
   $stderr.puts <<-EOT
 Processing complete.
   Input delimiter: #{col_sep}
-  Field counts: #{results[:field_counts].join(',')}
+  Field counts: #{results[:field_counts].inspect}
 EOT
 
 when 'mend'
-  file = File.open(filename)
-  metadata = Masticate.sniff(file)
+  metadata = Masticate.sniff(filename)
   col_sep = metadata[:col_sep]
   col_sep = "TAB" if col_sep == "\t"
-  file.close
-  file = File.open(filename)
-  results = Masticate.mend(file, metadata.merge(:output => $stdout))
-  file.close
+  results = Masticate.mend(filename, metadata)
   $stderr.puts <<-EOT
 Processing complete.
   Input delimiter: #{col_sep}
@@ -34,11 +28,14 @@ Processing complete.
 EOT
 
 when 'csvify'
-  file = File.open(filename)
-  metadata = Masticate.sniff(file)
-  file.close
-  file = File.open(filename)
-  Masticate.csvify(file, metadata.merge(:output => $stdout))
+  metadata = Masticate.sniff(filename)
+  results = Masticate.csvify(filename, metadata)
+  $stderr.puts <<-EOT
+Processing complete.
+  Input delimiter: #{metadata[:col_sep]}
+  Lines in input: #{results[:input_count]}
+  Lines in output: #{results[:output_count]}
+EOT
 
 else
   raise "unknown command #{command}"

data/lib/masticate/csvify.rb

@@ -2,19 +2,38 @@
 require "csv"
 
 class Masticate::Csvify
-  attr_reader :file
+  attr_reader :input
 
-  def initialize(file)
-    @file = file
+  def initialize(filename)
+    @input = File.open(filename)
   end
 
   def csvify(opts)
+    @output = opts[:output] ? File.open(opts[:output], "w") : $stdout
     csv_options = {}
     csv_options[:col_sep] = opts[:col_sep] if opts[:col_sep]
     csv_options[:quote_char] = opts[:quote_char] || opts[:col_sep] if opts[:quote_char] || opts[:col_sep]
-    
-    CSV.foreach(file, csv_options) do |row|
-      opts[:output] << row.to_csv
+
+    input_count = @output_count = 0
+    CSV.foreach(input, csv_options) do |row|
+      input_count += 1
+      emit(row.to_csv)
+    end
+    @output.close if opts[:output]
+    @input.close
+    {
+      :input_count => input_count,
+      :output_count => @output_count
+    }
+  end
+
+  def emit(line)
+    @output_count += 1
+    begin
+      @output.puts line
+    rescue Errno::EPIPE
+      # output was closed, e.g. ran piped into `head`
+      # silently ignore this condition, it's not fatal and doesn't need a warning
     end
   end
 end

data/lib/masticate/mender.rb

@@ -4,46 +4,60 @@
 # (due to a newline embedded in a field).  Glue those two lines into a single line in the output.
 
 class Masticate::Mender
-  attr_reader :file
+  attr_reader :input
 
-  def initialize(file)
-    @file = file
+  def initialize(filename)
+    @input = open(filename)
   end
 
   def mend(opts)
-    output = opts[:output]
+    @output = opts[:output] ? File.open(opts[:output], "w") : $stdout
     col_sep = opts[:col_sep]
 
-    expected_count = nil
-    @input_count = output_count = 0
+    expected_delim_count = nil
+    @input_count = @output_count = 0
     while (line = get) do
-      if !expected_count
-        # trust the first row
-        expected_count = line.count(col_sep)
-      else
-        running_count = line.count(col_sep)
-        until line.nil? || running_count >= expected_count
-          nextbit = get
-          if nextbit
-            line = line.chomp + ' ' + nextbit
-            running_count = line.count(col_sep)
-          else
-            line = nil
+      unless line =~ /^\s*$/
+        if !expected_delim_count
+          # trust the first row
+          expected_delim_count = line.count(col_sep)
+        else
+          running_count = line.count(col_sep)
+          while !input.eof? && running_count < expected_delim_count do
+            nextbit = get
+            if nextbit
+              line = line + ' ' + nextbit
+              running_count = line.count(col_sep)
+            end
           end
         end
+        if line.count(col_sep) > 2
+          emit(line)
+        end
       end
-      output_count += 1
-      output << line
     end
 
+    @input.close
+    @output.close if opts[:output]
     {
       :input_records => @input_count,
-      :output_records => output_count
+      :output_records => @output_count
     }
   end
 
   def get
-    (line = file.gets) && @input_count += 1
-    line
+    line = input.gets
+    @input_count += 1
+    line && line.chomp
+  end
+
+  def emit(line)
+    @output_count += 1
+    begin
+      @output.puts line
+    rescue Errno::EPIPE
+      # output was closed, e.g. ran piped into `head`
+      # silently ignore this condition, it's not fatal and doesn't need a warning
+    end
   end
 end

data/lib/masticate/sniffer.rb

@@ -1,32 +1,29 @@
 class Masticate::Sniffer
-  attr_reader :file
-  attr_reader :col_sep
+  attr_reader :col_sep, :stats
 
   CandidateDelimiters = [',', '|', "\t"]
 
-  def initialize(file)
-    @file = file
-  end
-
-  def self.sniff(file)
-    sniffer = new(file)
-    sniffer.sniff
+  def initialize(filename)
+    @filename = filename
   end
 
   def sniff
     @col_sep = find_col_sep
+    @stats = stats
     {
-      :col_sep => col_sep,
-      :field_counts => stats
+      :col_sep => @col_sep,
+      :field_counts => @stats,
+      :line1 => @line1
     }
   end
 
   def find_col_sep
-    line1 = file.lines.first
+    input = open(@filename)
+    @line1 = input.lines.first
     delimcounts = CandidateDelimiters.each_with_object({}) do |delim,h|
-      h[delim] = consider_delim(line1, delim)
+      h[delim] = consider_delim(@line1, delim)
     end
-    file.seek(0) # reset file pointer
+    input.close
     delimcounts.sort_by{|h,v| -v}.first.first
   end
 
@@ -35,6 +32,9 @@ class Masticate::Sniffer
   end
 
   def stats
-    file.lines.map {|line| line.split(col_sep).count}.uniq
+    input = open(@filename)
+    counts = input.lines.each_with_object(Hash.new(0)) {|line, counts| counts[line.split(col_sep).count] += 1}
+    input.close
+    counts
   end
 end

data/lib/masticate/version.rb

@@ -1,3 +1,3 @@
 module Masticate
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

data/lib/masticate.rb

@@ -1,18 +1,20 @@
-require "masticate/version"
-require "masticate/sniffer"
-require "masticate/mender"
-require "masticate/csvify"
+require "open-uri"
+
+require_relative "masticate/version"
+require_relative "masticate/sniffer"
+require_relative "masticate/mender"
+require_relative "masticate/csvify"
 
 module Masticate
-  def self.sniff(file)
-    Sniffer.new(file).sniff
+  def self.sniff(filename)
+    Sniffer.new(filename).sniff
   end
 
-  def self.mend(file, opts)
-    Mender.new(file).mend(opts)
+  def self.mend(filename, opts)
+    Mender.new(filename).mend(opts)
   end
 
-  def self.csvify(file, opts)
-    Csvify.new(file).csvify(opts)
+  def self.csvify(filename, opts)
+    Csvify.new(filename).csvify(opts)
   end
 end

data/masticate.gemspec

@@ -16,5 +16,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = Masticate::VERSION
 
-  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "rspec", "~> 2.9.0"
+  gem.add_development_dependency "guard-rspec", "~> 0.7.0"
+  gem.add_development_dependency "ruby_gntp", "~> 0.3.4"
 end

data/spec/data/broken_psv.txt

@@ -1,6 +1,6 @@
 COL1|COL 2|Col 3   |col-4|   col5  |col6
-data|  data |data  |d a t a|data|data
-data|  data |data  |d a t a|data|data
-data|  data |this long row
+data1|  data |data  |d a t a|data|data
+data2|  data |data  |d a t a|data|data
+data3|  data |this long row
 is split across lines|d a t a|data|data
-data|  data |data  |d a t a|data|data
+data4|  data |data  |d a t a|data|data

data/spec/data/junk_trailer.txt

@@ -0,0 +1,8 @@
+COL1|COL 2|Col 3   |col-4|   col5  |col6
+one|  data |data  |d a t a|data|data
+two|  data |data  |d a t a|data|data
+three|  data |data  |d a t a|data,data|data
+four|  data |data "more data"  |d a t a|data|data
+trailer|123
+
+

data/spec/lib/csvify_spec.rb

@@ -0,0 +1,17 @@
+# spec for file-sniffing functions
+
+require "spec_helper"
+require "tempfile"
+
+describe "csvification" do
+  it "should convert pipes to standard commas" do
+    filename = File.dirname(__FILE__) + "/../data/pipe_data.txt"
+    tmp = Tempfile.new('csvify')
+    results = Masticate.csvify(filename, :output => tmp, :col_sep => '|')
+    output = File.read(tmp)
+    tmp.unlink
+    output.lines.count.should == 5
+    results[:input_count].should == 5
+    results[:output_count].should == 5
+  end
+end

data/spec/lib/mend_spec.rb

@@ -0,0 +1,20 @@
+# spec for file-sniffing functions
+
+require "spec_helper"
+
+describe "mending" do
+  it "should merge lines when delimiter counts don't match'" do
+    filename = File.dirname(__FILE__) + "/../data/broken_psv.txt"
+    results = Masticate.mend(filename, :col_sep => '|', :output => "/dev/null")
+    results[:input_records].should == 7
+    results[:output_records].should == 5
+  end
+
+  it "should strip trailer records" do
+    filename = File.dirname(__FILE__) + "/../data/junk_trailer.txt"
+    metadata = Masticate.sniff(filename)
+    results = Masticate.mend(filename, metadata.merge(:output => "/dev/null"))
+    results[:input_records].should == 9
+    results[:output_records].should == 5
+  end
+end

data/spec/{spec → lib}/sniffer_spec.rb

@@ -5,17 +5,15 @@ require "spec_helper"
 describe "delimiter sniffing" do
   it "should find tab delimiter" do
     filename = File.dirname(__FILE__) + "/../data/tabbed_data.txt"
-    file = File.open(filename)
-    results = Masticate.sniff(file)
+    results = Masticate.sniff(filename)
     results[:col_sep].should == "\t"
-    results[:field_counts].should == [6]
+    results[:field_counts].should == {6 => 5}
   end
 
   it "should find pipe delimiter" do
     filename = File.dirname(__FILE__) + "/../data/pipe_data.txt"
-    file = File.open(filename)
-    results = Masticate.sniff(file)
+    results = Masticate.sniff(filename)
     results[:col_sep].should == '|'
-    results[:field_counts].should == [6]
+    results[:field_counts].should == {6 => 5}
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: masticate
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,19 +9,41 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-03 00:00:00.000000000 Z
+date: 2012-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2151740780 !ruby/object:Gem::Requirement
+  requirement: &2153254280 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.9.0
   type: :development
   prerelease: false
-  version_requirements: *2151740780
+  version_requirements: *2153254280
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: &2153246900 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+  type: :development
+  prerelease: false
+  version_requirements: *2153246900
+- !ruby/object:Gem::Dependency
+  name: ruby_gntp
+  requirement: &2153246180 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.4
+  type: :development
+  prerelease: false
+  version_requirements: *2153246180
 description: Data file crunching
 email:
 - jmay@pobox.com
@@ -32,6 +54,7 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - Gemfile
+- Guardfile
 - LICENSE
 - README.md
 - Rakefile
@@ -43,11 +66,12 @@ files:
 - lib/masticate/version.rb
 - masticate.gemspec
 - spec/data/broken_psv.txt
+- spec/data/junk_trailer.txt
 - spec/data/pipe_data.txt
 - spec/data/tabbed_data.txt
-- spec/spec/csvify_spec.rb
-- spec/spec/mend_spec.rb
-- spec/spec/sniffer_spec.rb
+- spec/lib/csvify_spec.rb
+- spec/lib/mend_spec.rb
+- spec/lib/sniffer_spec.rb
 - spec/spec_helper.rb
 homepage: ''
 licenses: []
@@ -75,10 +99,11 @@ specification_version: 3
 summary: Utility functions for parsing incoming text data files.
 test_files:
 - spec/data/broken_psv.txt
+- spec/data/junk_trailer.txt
 - spec/data/pipe_data.txt
 - spec/data/tabbed_data.txt
-- spec/spec/csvify_spec.rb
-- spec/spec/mend_spec.rb
-- spec/spec/sniffer_spec.rb
+- spec/lib/csvify_spec.rb
+- spec/lib/mend_spec.rb
+- spec/lib/sniffer_spec.rb
 - spec/spec_helper.rb
 has_rdoc: 

data/spec/spec/csvify_spec.rb

@@ -1,14 +0,0 @@
-# spec for file-sniffing functions
-
-require "spec_helper"
-
-describe "csvification" do
-  it "should convert pipes to standard commas" do
-    filename = File.dirname(__FILE__) + "/../data/pipe_data.txt"
-    file = File.open(filename)
-    strio = StringIO.new
-    Masticate.csvify(file, :output => strio, :col_sep => '|')
-    strio.close
-    strio.string.lines.count.should == 5
-  end
-end

data/spec/spec/mend_spec.rb

@@ -1,14 +0,0 @@
-# spec for file-sniffing functions
-
-require "spec_helper"
-
-describe "mending" do
-  it "should merge lines when delimiter counts don't match'" do
-    filename = File.dirname(__FILE__) + "/../data/broken_psv.txt"
-    file = File.open(filename)
-    devnull = File.open('/dev/null', 'w')
-    results = Masticate.mend(file, :output => devnull, :col_sep => '|')
-    results[:input_records].should == 6
-    results[:output_records].should == 5
-  end
-end