heuristic-csv 0.0.2.alpha

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'http://rubygems.org'
+
+# Specify your gem's dependencies in columns-matcher.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,24 @@
+PATH
+  remote: .
+  specs:
+    heuristic-csv (0.0.2.alpha)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.3)
+    rspec (2.6.0)
+      rspec-core (~> 2.6.0)
+      rspec-expectations (~> 2.6.0)
+      rspec-mocks (~> 2.6.0)
+    rspec-core (2.6.4)
+    rspec-expectations (2.6.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.6.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  heuristic-csv!
+  rspec (~> 2.6.0)

data/README.md

@@ -0,0 +1,5 @@
+HeuristicCsv
+=============
+
+Standard CSV library raise an exception when you try to read some malformed CSV data.
+Sometimes you need to read readable data anyway. This gem adds to standard CSV library some methods to try to overcome the CSV::MalformedCSVError exception and read the data using RegExp to fix the malformed data.

data/Rakefile

@@ -0,0 +1,6 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+task default: :spec

data/heuristic-csv.gemspec

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/heuristic-csv/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Andrea Mostosi"]
+  gem.email         = ["andrea.mostosi@zenkay.net"]
+  gem.description   = %q{Try to overcome the exception CSV::MalformedCSVError and read the data anyway}
+  gem.summary       = %q{CSV extension}
+  gem.homepage      = "https://github.com/zenkay/heuristic-csv"
+
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.name          = "heuristic-csv"
+  gem.require_paths = ["lib"]
+  gem.version       = HeuristicCsv::VERSION
+  
+  gem.add_development_dependency "rspec", "~> 2.6.0"
+end

data/lib/heuristic-csv.rb

@@ -0,0 +1,30 @@
+require "heuristic-csv/version"
+
+class CSV
+  def self.heuristic_parse(data = $stdout, options = Hash.new, &block)
+    if block_given?
+      begin
+        self.parse(data, options, &block)
+      rescue Exception => e
+        data = self.remove_misplaced_quotes(data, options)
+        self.parse(data, options, &block)
+      end
+    end
+  end
+  
+  private
+  
+  # Strategy 1: Replace misplaced quotes using regular expressions and remove it
+  def self.remove_misplaced_quotes(data, options)
+    if options.nil? or options[:col_sep].nil?
+      sep = DEFAULT_OPTIONS[:col_sep]
+    else
+      sep = options[:col_sep]
+    end
+    strings_with_problems = data.scan(/\"([^\n#{sep}]*\"[^\n#{sep}]*)\"[\n#{sep}]+/)
+    strings_with_problems.each do |string|
+      data.gsub!(string.first, string.first.gsub("\"", ""))
+    end
+    return data
+  end
+end

data/lib/heuristic-csv/version.rb

@@ -0,0 +1,3 @@
+module HeuristicCsv
+  VERSION = "0.0.2.alpha"
+end

data/spec/heuristic-csv_spec.rb

@@ -0,0 +1,51 @@
+# encoding: utf-8
+require 'spec_helper'
+describe HeuristicCsv do
+  
+  describe "Heuristic behavior" do
+    
+    it "should parse unquoted CSV strings" do
+      data = "argument1,argument2,argument3"
+      CSV.heuristic_parse(data) do |row|
+        row.count.should eq(3)
+      end
+    end
+
+    it "should parse unquoted CSV strings with different col_sep" do
+      data = "argument1;argument2;argument3"
+      CSV.heuristic_parse(data, {col_sep: ";"}) do |row|
+        row.count.should eq(3)
+      end
+    end
+
+    it "should parse quoted CSV strings" do
+      data = '"argument1","argument2","argument3"'
+      CSV.heuristic_parse(data) do |row|
+        row.count.should eq(3)
+      end
+    end
+    
+    it "should parse quoted CSV strings with different col_sep" do
+      data = '"argument1";"argument2";"argument3"'
+      CSV.heuristic_parse(data, {col_sep: ";"}) do |row|
+        row.count.should eq(3)
+      end
+    end
+    
+    it "should parse CSV strings with misplaced quotes" do
+      data = '"argument1","argum"WRONG"ent2","argument3"'
+      CSV.heuristic_parse(data) do |row|
+        row.count.should eq(3)
+      end
+    end
+    
+    it "should parse CSV strings with misplaced quotes with different col_sep" do
+      data = '"argument1";"argum"WRONG"ent2";"argument3"'
+      CSV.heuristic_parse(data, {col_sep: ";"}) do |row|
+        row.count.should eq(3)
+      end
+    end
+    
+  end
+  
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+require 'csv'
+require 'heuristic-csv'

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification
+name: heuristic-csv
+version: !ruby/object:Gem::Version
+  version: 0.0.2.alpha
+  prerelease: 6
+platform: ruby
+authors:
+- Andrea Mostosi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-06-17 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70290990509180 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+  type: :development
+  prerelease: false
+  version_requirements: *70290990509180
+description: Try to overcome the exception CSV::MalformedCSVError and read the data
+  anyway
+email:
+- andrea.mostosi@zenkay.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- heuristic-csv.gemspec
+- lib/.DS_Store
+- lib/heuristic-csv.rb
+- lib/heuristic-csv/version.rb
+- spec/.DS_Store
+- spec/heuristic-csv_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/zenkay/heuristic-csv
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: CSV extension
+test_files:
+- spec/heuristic-csv_spec.rb
+- spec/spec_helper.rb