masticate 0.0.1 → 0.0.2

data/.gitignore

@@ -16,3 +16,4 @@ test/tmp
 test/version_tmp
 tmp
 .rspec
+.irb_history

data/README.md

@@ -1,6 +1,6 @@
 # Masticate
 
-TODO: Write a gem description
+Data file crunching
 
 ## Installation
 
@@ -18,7 +18,15 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+    masticate sniff filename
+    masticate mend filename
+
+or
+
+    > require 'masticate'
+    > f = File.open(filename)
+    > Masticate.sniff(f)
+    > Masticate.mend(f, :output => $stdout, :col_sep => "\t")
 
 ## Contributing
 

data/bin/masticate

@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby
+
+require "masticate"
+
+command, filename = ARGV
+
+case ARGV.shift
+when 'sniff'
+  file = File.open(filename)
+  results = Masticate.sniff(file)
+  col_sep = results[:col_sep]
+  col_sep = "TAB" if col_sep == "\t"
+  file.close
+  $stderr.puts <<-EOT
+Processing complete.
+  Input delimiter: #{col_sep}
+  Field counts: #{results[:field_counts].join(',')}
+EOT
+
+when 'mend'
+  file = File.open(filename)
+  metadata = Masticate.sniff(file)
+  col_sep = metadata[:col_sep]
+  col_sep = "TAB" if col_sep == "\t"
+  file.close
+  file = File.open(filename)
+  results = Masticate.mend(file, metadata.merge(:output => $stdout))
+  file.close
+  $stderr.puts <<-EOT
+Processing complete.
+  Input delimiter: #{col_sep}
+  Lines in input: #{results[:input_records]}
+  Lines in output: #{results[:output_records]}
+EOT
+
+when 'csvify'
+  file = File.open(filename)
+  metadata = Masticate.sniff(file)
+  file.close
+  file = File.open(filename)
+  Masticate.csvify(file, metadata.merge(:output => $stdout))
+
+else
+  raise "unknown command #{command}"
+end

data/lib/masticate/csvify.rb

@@ -0,0 +1,16 @@
+# convert input to clean standard CSV
+require "csv"
+
+class Masticate::Csvify
+  attr_reader :file
+
+  def initialize(file)
+    @file = file
+  end
+
+  def csvify(opts)
+    CSV.foreach(file, :col_sep => opts[:col_sep]) do |row|
+      opts[:output] << row.to_csv
+    end
+  end
+end

data/lib/masticate/mender.rb

@@ -0,0 +1,49 @@
+# repair delimited input files
+#
+# A row that contains fewer delimiters than expected has been split across two lines
+# (due to a newline embedded in a field).  Glue those two lines into a single line in the output.
+
+class Masticate::Mender
+  attr_reader :file
+
+  def initialize(file)
+    @file = file
+  end
+
+  def mend(opts)
+    output = opts[:output]
+    col_sep = opts[:col_sep]
+
+    expected_count = nil
+    @input_count = output_count = 0
+    while (line = get) do
+      if !expected_count
+        # trust the first row
+        expected_count = line.count(col_sep)
+      else
+        running_count = line.count(col_sep)
+        until line.nil? || running_count >= expected_count
+          nextbit = get
+          if nextbit
+            line = line.chomp + ' ' + nextbit
+            running_count = line.count(col_sep)
+          else
+            line = nil
+          end
+        end
+      end
+      output_count += 1
+      output << line
+    end
+
+    {
+      :input_records => @input_count,
+      :output_records => output_count
+    }
+  end
+
+  def get
+    (line = file.gets) && @input_count += 1
+    line
+  end
+end

data/lib/masticate/version.rb

@@ -1,3 +1,3 @@
 module Masticate
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/masticate.rb

@@ -1,8 +1,18 @@
 require "masticate/version"
 require "masticate/sniffer"
+require "masticate/mender"
+require "masticate/csvify"
 
 module Masticate
   def self.sniff(file)
     Sniffer.new(file).sniff
   end
+
+  def self.mend(file, opts)
+    Mender.new(file).mend(opts)
+  end
+
+  def self.csvify(file, opts)
+    Csvify.new(file).csvify(opts)
+  end
 end

data/spec/data/broken_psv.txt

@@ -0,0 +1,6 @@
+COL1|COL 2|Col 3   |col-4|   col5  |col6
+data|  data |data  |d a t a|data|data
+data|  data |data  |d a t a|data|data
+data|  data |this long row
+is split across lines|d a t a|data|data
+data|  data |data  |d a t a|data|data

data/spec/data/pipe_data.txt

@@ -1,5 +1,5 @@
 COL1|COL 2|Col 3   |col-4|   col5  |col6
 data|  data |data  |d a t a|data|data
 data|  data |data  |d a t a|data|data
-data|  data |data  |d a t a|data|data
+data|  data |data  |d a t a|data,data|data
 data|  data |data  |d a t a|data|data

data/spec/spec/csvify_spec.rb

@@ -0,0 +1,14 @@
+# spec for file-sniffing functions
+
+require "spec_helper"
+
+describe "csvification" do
+  it "should convert pipes to standard commas" do
+    filename = File.dirname(__FILE__) + "/../data/pipe_data.txt"
+    file = File.open(filename)
+    strio = StringIO.new
+    Masticate.csvify(file, :output => strio, :col_sep => '|')
+    strio.close
+    strio.string.lines.count.should == 5
+  end
+end

data/spec/spec/mend_spec.rb

@@ -0,0 +1,14 @@
+# spec for file-sniffing functions
+
+require "spec_helper"
+
+describe "mending" do
+  it "should merge lines when delimiter counts don't match'" do
+    filename = File.dirname(__FILE__) + "/../data/broken_psv.txt"
+    file = File.open(filename)
+    devnull = File.open('/dev/null', 'w')
+    results = Masticate.mend(file, :output => devnull, :col_sep => '|')
+    results[:input_records].should == 6
+    results[:output_records].should == 5
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: masticate
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2153533260 !ruby/object:Gem::Requirement
+  requirement: &2151845500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,11 +21,12 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2153533260
+  version_requirements: *2151845500
 description: Data file crunching
 email:
 - jmay@pobox.com
-executables: []
+executables:
+- masticate
 extensions: []
 extra_rdoc_files: []
 files:
@@ -34,12 +35,18 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- bin/masticate
 - lib/masticate.rb
+- lib/masticate/csvify.rb
+- lib/masticate/mender.rb
 - lib/masticate/sniffer.rb
 - lib/masticate/version.rb
 - masticate.gemspec
+- spec/data/broken_psv.txt
 - spec/data/pipe_data.txt
 - spec/data/tabbed_data.txt
+- spec/spec/csvify_spec.rb
+- spec/spec/mend_spec.rb
 - spec/spec/sniffer_spec.rb
 - spec/spec_helper.rb
 homepage: ''
@@ -67,8 +74,11 @@ signing_key:
 specification_version: 3
 summary: Utility functions for parsing incoming text data files.
 test_files:
+- spec/data/broken_psv.txt
 - spec/data/pipe_data.txt
 - spec/data/tabbed_data.txt
+- spec/spec/csvify_spec.rb
+- spec/spec/mend_spec.rb
 - spec/spec/sniffer_spec.rb
 - spec/spec_helper.rb
 has_rdoc: