csv_sniffer 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0c4ec9e742582c39e7ee5b57d4dce731292bc0e0
+  data.tar.gz: 3aaa46390b01a030b317724be479da6724a2612c
+SHA512:
+  metadata.gz: 976a4029144696e77fdb7904a9be48270b10ca4e4b03b99c26a99875c0845a4032e5cf587ce32dddf8b1332789fd5af0062c95a73b6e10c34a404be262311b79
+  data.tar.gz: 232fb8c1acacbf5932962e0429bf0f287ec510f96e7d7f731b613662155a636dd89b3f87a5d1659aea8e0d760a648c6fd1a33563a0486fc83e240b72db005620

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Tim Ojo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,65 @@
+# CSV Sniffer
+
+CSV Sniffer is intended to provide utilities that will allow a user heuristically detect the delimiter character in use, whether the values in the CSV file are quote enclosed, whether the file contains a header, and more. The library is intended to detect information to be used as configuration inputs for CSV parsers. For delimiter detection the following delimiters are currently supported `[",", "\t", "|", ";"]`
+
+To ensure high performance and a low memory footprint, the library uses as little information as needed to make accurate decisions. Contributors are welcome to
+improve the algorithms in use.
+
+
+## Installation
+
+```
+$ gem install csv_sniffer
+```
+
+## Usage
+
+Given a `some_data.csv` file:
+
+```csv
+Name;Phone
+John Doe ;555-481-2345
+Jane C. Doe;555-123-4567
+```
+
+Detection usage is as follows:
+
+```rb
+require "csv_sniffer"
+
+delim = CsvSniffer.detect_delimiter("/path/to/some_data.csv") #=> ";"
+is_quote_enclosed = CsvSniffer.is_quote_enclosed?("/path/to/some_data.csv") #=> False
+```
+
+See [`test.rb`](test.rb) for more examples.
+
+
+## Tests
+
+```
+$ ruby test.rb
+```
+
+## License
+
+The MIT License (MIT)
+
+Copyright © 2015 Tim Ojo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc "Run tests"
+task :default => :test

data/csv_sniffer.gemspec

@@ -0,0 +1,15 @@
+Gem::Specification.new do |s|
+  s.name        = 'csv_sniffer'
+  s.version     = '0.0.1'
+  s.date        = '2015-10-09'
+  s.summary     = "CSV library for heuristic detection of CSV properties"
+  s.description = "CSV Sniffer is intended to provide utilities that will allow a user detect the delimiter character in use, whether the values in the CSV file are quote enclosed, whether the file contains a header, and more. The library is intended to detect information to be used as configuration inputs for CSV parsers."
+  s.authors     = ["Tim Ojo"]
+  s.email       = 'ojo.tim@gmail.com'
+  s.homepage    = 'https://github.com/tim-ojo/csv_sniffer'
+  s.license     = 'MIT'
+
+  s.files       = `git ls-files`.split($/)
+  s.test_files  = s.files.grep(/^test/)
+  s.add_development_dependency "test-unit"
+end

data/lib/csv_sniffer.rb

@@ -0,0 +1,106 @@
+# This class contains functions to heuristically decipher certain information from a CSV file
+class CsvSniffer
+
+    # Reads the first line of the csv and returns true if the line starts and ends with " or '
+    def self.is_quote_enclosed?(filepath)
+      line = File.open(filepath, &:readline)
+      line.chomp!.strip!
+      return line.start_with?('"') && line.end_with?('"') || line.start_with?("'") && line.end_with?("'")
+    end
+
+    def self.get_quote_char(filepath)
+      if is_quote_enclosed?(filepath)
+        line = File.open(filepath, &:readline)
+        line.chomp!.strip!
+        return line[0]
+      else
+        return nil
+      end
+    end
+
+    # If the csv is quote enclosed then just get the delimiter after the first cell. Otherwise...
+    # Get the first line and count how many of the possible delimiters are present. If there is >1 of one of the
+    # delimiters and 0 of the others then, then we pick the max. If there are more than 0 of any of the others then
+    # we repeat the counting procedure for the next 50 lines until the condition is satisfied.
+    # If the condition is never satisfied then we simply pick the delimiter that occurs the most frequently, defaulting
+    # to the comma. Unless that delimeter's count is equal to the tab or pipe delimiter's count. In that case we return \t or |
+    def self.detect_delimiter (filepath)
+
+      if is_quote_enclosed?(filepath)
+        line = File.open(filepath, &:readline)
+        line.chomp!.strip!
+        m = /["'].+?["']([,|;\t])/.match(line)
+        if (m)
+          return m[1]
+        end
+      end
+
+      lineCount = 0
+      File.foreach(filepath) do |line|
+        detectedDelim = max_delim_when_others_are_zero(line)
+        if detectedDelim != '0' #=> '0' is a sentinel value that indicates no delim found
+          return detectedDelim
+        end
+
+        lineCount += 1;
+        break if lineCount == 50
+      end
+
+      # If I got here I'm going to pick the default by counting the delimiters on the first line and returning the max
+      line = File.open(filepath, &:readline)
+      freqOfPossibleDelims = get_freq_of_possible_delims(line)
+
+      maxFreq = 0
+      maxFreqIndex = 0
+      freqOfPossibleDelims.each_with_index do |delimFreq, i|
+        if (delimFreq > maxFreq)
+          maxFreq = delimFreq
+          maxFreqIndex = i
+        end
+      end
+
+      # Favor "\t" and "|" over ","
+      if (maxFreq == freqOfPossibleDelims[1])
+        return "\t"
+      elsif (maxFreq == freqOfPossibleDelims[3])
+        return "|"
+      else
+        return [",", "\t", ";", "|"][maxFreqIndex]
+      end
+    end
+
+    def self.max_delim_when_others_are_zero (line)
+      freqOfPossibleDelims = get_freq_of_possible_delims(line)
+
+      maxFreq = 0
+      maxFreqIndex = 0
+      zeroCount = 0
+      freqOfPossibleDelims.each_with_index do |delimFreq, i|
+        if (delimFreq > maxFreq)
+          maxFreq = delimFreq
+          maxFreqIndex = i
+        end
+        zeroCount += 1 if delimFreq == 0
+      end
+
+      if zeroCount >= 3
+        return [',', '\t', ';', '|'][maxFreqIndex]
+      else
+        return '0' #=> '0' is a sentinel value that indicates no delim found
+      end
+    end
+
+    def self.get_freq_of_possible_delims (line)
+      freqOfPossibleDelims = Array.new(4) #=> [0 = ','] [1 = '\t'] [2 = ';'] [3 = '|']
+      freqOfPossibleDelims[0] = line.count ","
+      freqOfPossibleDelims[1] = line.count "\t"
+      freqOfPossibleDelims[2] = line.count ";"
+      freqOfPossibleDelims[3] = line.count "|"
+
+      return freqOfPossibleDelims
+    end
+
+    private_class_method :max_delim_when_others_are_zero
+    private_class_method :get_freq_of_possible_delims
+
+end

data/test/test_csv_sniffer.rb

@@ -0,0 +1,71 @@
+require 'minitest/autorun'
+require 'tempfile'
+require 'csv_sniffer'
+
+class CsvSnifferTest < Minitest::Test
+
+  @@file1 = Tempfile.new('file1')
+  @@file1.puts "Name,Number"
+  @@file1.puts "John Doe,555-123-4567"
+  @@file1.puts "Jane C. Doe,555-000-1234"
+  @@file1.rewind
+
+  @@file2 = Tempfile.new('file2')
+  @@file2.puts "'Name' |'Number'\t"
+  @@file2.puts "'John Doe'|'555-123-4567'"
+  @@file2.puts "'Jane C. Doe'|'555-000-1234'"
+  @@file2.rewind
+
+  @@file3 = Tempfile.new('file3')
+  @@file3.puts "John Doe;555-123-4567;Good\tdude"
+  @@file3.puts "Jane C. Doe;555-000-1234   ; Great gal"
+  @@file3.rewind
+
+  @@file4 = Tempfile.new('file4')
+  @@file4.puts "Doe, John\t555-123-4567"
+  @@file4.puts "Jane C. Doe\t555-000-1234\t"
+  @@file4.rewind
+
+  @@file5 = Tempfile.new('file5')
+  @@file5.puts '"Doe,,,,,, John"|"555-123-4567"'
+  @@file5.puts '"Jane C. Doe"|"555-000-1234\t"'
+  @@file5.rewind
+
+  @@file6 = Tempfile.new('file6')
+  @@file6.puts 'Doe, John|555-123-4567'
+  @@file6.puts 'Doe, Jane C. |555-000-1234'
+  @@file6.rewind
+
+  def test_file1
+    assert_equal ",", CsvSniffer.detect_delimiter(@@file1.path)
+    assert_equal false, CsvSniffer.is_quote_enclosed?(@@file1.path)
+    assert_equal nil, CsvSniffer.get_quote_char(@@file1.path)
+  end
+
+  def test_file2
+    assert_equal "|", CsvSniffer.detect_delimiter(@@file2.path)
+    assert_equal true, CsvSniffer.is_quote_enclosed?(@@file2.path)
+    assert_equal "'", CsvSniffer.get_quote_char(@@file2.path)
+  end
+
+  def test_file3
+    assert_equal ";", CsvSniffer.detect_delimiter(@@file3.path)
+    assert_equal false, CsvSniffer.is_quote_enclosed?(@@file3.path)
+  end
+
+  def test_file4
+    assert_equal "\\t", CsvSniffer.detect_delimiter(@@file4.path)
+    assert_equal nil, CsvSniffer.get_quote_char(@@file4.path)
+  end
+
+  def test_file5
+    assert_equal "|", CsvSniffer.detect_delimiter(@@file5.path)
+    assert_equal true, CsvSniffer.is_quote_enclosed?(@@file5.path)
+    assert_equal '"', CsvSniffer.get_quote_char(@@file5.path)
+  end
+
+  def test_file6
+    assert_equal "|", CsvSniffer.detect_delimiter(@@file6.path)
+  end
+
+end

metadata

@@ -0,0 +1,67 @@
+--- !ruby/object:Gem::Specification
+name: csv_sniffer
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Tim Ojo
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-10-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: CSV Sniffer is intended to provide utilities that will allow a user detect
+  the delimiter character in use, whether the values in the CSV file are quote enclosed,
+  whether the file contains a header, and more. The library is intended to detect
+  information to be used as configuration inputs for CSV parsers.
+email: ojo.tim@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- Rakefile
+- csv_sniffer.gemspec
+- lib/csv_sniffer.rb
+- test/test_csv_sniffer.rb
+homepage: https://github.com/tim-ojo/csv_sniffer
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.7
+signing_key: 
+specification_version: 4
+summary: CSV library for heuristic detection of CSV properties
+test_files:
+- test/test_csv_sniffer.rb