csv_sniffer 0.0.2 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b9e303592691083e6ac15f46740577447dfeea43
-  data.tar.gz: 0d7c5ab5c866d2b16d5591a20db8ff62ac0abeb3
+  metadata.gz: e14c6e37b6559305010247b4ba71c6d1b87d4f5a
+  data.tar.gz: 909f073414ada386352ae1a7462d4b227681551f
 SHA512:
-  metadata.gz: c7bd0b58ee2ae274f149e0212fb9f8434b2a240304b0b37f1237d815ff573a2ae14459cb27f573ce39622d88680460a8c4a859dda94e7297556eca72fc510608
-  data.tar.gz: caa05cdbb7763a1e72b85468f2720f10b2354ba7b97c7b6fb05582807b1f3f0bcdf9f078a8888bdfb811f30022904307f98989e252d059f51e8c75206dcb886e
+  metadata.gz: 045d74a5dc4c0a134e157f2b0c18bd713c5e85cfcdb696c77b74b0d33f78489db9d172936d25979bf1709a2f45dd662a32469df7da7059eefc5d511d26388dce
+  data.tar.gz: 01f3db3f4953cbd489b4ef90f0c5771e4253da659dc9bb398e4fc2d1a9d421f560f03bb88dd35e6bbf30b3f23e92ec34994ed0563bd6bb891b4ab2454dda4645

data/csv_sniffer.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name        = 'csv_sniffer'
-  s.version     = '0.0.2'
-  s.date        = '2015-10-10'
+  s.version     = '0.1.0'
+  s.date        = '2015-10-15'
   s.summary     = "CSV library for heuristic detection of CSV properties"
   s.description = "CSV Sniffer is intended to provide utilities that will allow a user detect the delimiter character in use, whether the values in the CSV file are quote enclosed, whether the file contains a header, and more. The library is intended to detect information to be used as configuration inputs for CSV parsers."
   s.authors     = ["Tim Ojo"]

data/lib/csv_sniffer.rb

@@ -89,14 +89,110 @@ class CsvSniffer
 
       # Favor "\t" and "|" over ","
       if (maxFreq == freqOfPossibleDelims[1])
-        return "\t"
+        return '\t'
       elsif (maxFreq == freqOfPossibleDelims[3])
         return "|"
       else
-        return [",", "\t", ";", "|"][maxFreqIndex]
+        return [",", '\t', ";", "|"][maxFreqIndex]
       end
     end
 
+    # Heuristically detects whether or not the csv file uses the first line as a header
+    #
+    # Example:
+    #   CsvSniffer.has_header?("path/to/file")
+    #   =>  false
+    #
+    # Arguments:
+    #   filepath: (String)
+
+    def self.has_header?(filepath)
+      # Creates a dictionary of types of data in each column. If any
+      # column is of a single type (say, integers), *except* for the first
+      # row, then the first row is presumed to be labels. If the type
+      # can't be determined, it is assumed to be a string in which case
+      # the length of the string is the determining factor: if all of the
+      # rows except for the first are the same length, it's a header.
+      # Finally, a 'vote' is taken at the end for each column, adding or
+      # subtracting from the likelihood of the first row being a header.
+      delim = detect_delimiter(filepath)
+      if (delim == "\\t")
+        delim = "\t"
+      end
+
+      headerRow = nil
+      lineCount = 0
+      columnTypes = Hash.new
+      File.foreach(filepath) do |line|
+        if (!headerRow) # assume the first row is a header
+          headerRow = line.split(delim)
+
+          headerRow.each_index do |colIndex|
+            columnTypes[colIndex] = nil
+          end
+          next
+        end
+
+        lineCount += 1
+        break if lineCount == 50
+
+        row = line.split(delim)
+        columnTypes.each_key do |colIndex|
+          thisColType = nil
+          if (row[colIndex].strip.to_i.to_s == row[colIndex])
+            thisColType = Integer
+          elsif (row[colIndex].strip.to_f.to_s == row[colIndex])
+            thisColType = Float
+          else
+            # fallback to the length of the string
+            thisColType = row[colIndex].strip.length
+          end
+
+          if (thisColType != columnTypes[colIndex])
+            if (columnTypes[colIndex] == nil)
+              # add new column type
+              columnTypes[colIndex] = thisColType
+            else
+              # type is inconsistent, remove from consideration
+              columnTypes[colIndex] = nil
+            end
+          end
+
+        end # end iterate through each row column to determine columnType
+      end # end iterate through each row
+
+      # finally, compare results against first row and "vote" on whether its a header
+      hasHeader = 0
+      columnTypes.each do |colIndex, colVal|
+        if colVal.class == NilClass
+          # ignore
+        elsif (colVal.class != Class) # it's a length
+          if (headerRow[colIndex].strip.length != colVal)
+            hasHeader += 1
+          else
+            hasHeader -= 1
+          end
+        else
+          # determine the type of the header and compare it to the type in the Hash
+          # if the type is the same then vote down otherwise vote up
+          if headerRow[colIndex].strip.to_i.to_s == headerRow[colIndex]
+            if colVal == Integer
+              hasHeader -= 1
+            else
+              hasHeader += 1
+            end
+          elsif headerRow[colIndex].strip.to_f.to_s == headerRow[colIndex]
+            if colVal == Float
+              hasHeader -= 1
+            else
+              hasHeader += 1
+            end
+          end
+        end # end type comparison voting branch
+      end # end voting loop
+
+      return hasHeader > 0
+    end
 
     def self.max_delim_when_others_are_zero (line)
       freqOfPossibleDelims = get_freq_of_possible_delims(line)

data/test/test_csv_sniffer.rb

@@ -19,6 +19,7 @@ class CsvSnifferTest < Minitest::Test
   @@file3 = Tempfile.new('file3')
   @@file3.puts "John Doe;555-123-4567;Good\tdude"
   @@file3.puts "Jane C. Doe;555-000-1234   ; Great gal"
+  @@file3.puts "John Smith;555-999-1234;Don't know about him"
   @@file3.rewind
 
   @@file4 = Tempfile.new('file4')
@@ -32,30 +33,35 @@ class CsvSnifferTest < Minitest::Test
   @@file5.rewind
 
   @@file6 = Tempfile.new('file6')
-  @@file6.puts 'Doe, John|555-123-4567'
-  @@file6.puts 'Doe, Jane C. |555-000-1234'
+  @@file6.puts 'Name|Phone No.|Age'
+  @@file6.puts 'Doe, John|555-123-4567|31'
+  @@file6.puts 'Doe, Jane C. |555-000-1234|30'
   @@file6.rewind
 
   def test_file1
     assert_equal ",", CsvSniffer.detect_delimiter(@@file1.path)
     assert_equal false, CsvSniffer.is_quote_enclosed?(@@file1.path)
     assert_equal nil, CsvSniffer.get_quote_char(@@file1.path)
+    assert_equal true, CsvSniffer.has_header?(@@file1.path)
   end
 
   def test_file2
     assert_equal "|", CsvSniffer.detect_delimiter(@@file2.path)
     assert_equal true, CsvSniffer.is_quote_enclosed?(@@file2.path)
     assert_equal "'", CsvSniffer.get_quote_char(@@file2.path)
+    assert_equal true, CsvSniffer.has_header?(@@file2.path)
   end
 
   def test_file3
     assert_equal ";", CsvSniffer.detect_delimiter(@@file3.path)
     assert_equal false, CsvSniffer.is_quote_enclosed?(@@file3.path)
+    assert_equal false, CsvSniffer.has_header?(@@file3.path)
   end
 
   def test_file4
-    assert_equal "\\t", CsvSniffer.detect_delimiter(@@file4.path)
+    assert_equal '\t', CsvSniffer.detect_delimiter(@@file4.path)
     assert_equal nil, CsvSniffer.get_quote_char(@@file4.path)
+    assert_equal false, CsvSniffer.has_header?(@@file4.path)
   end
 
   def test_file5
@@ -66,6 +72,7 @@ class CsvSnifferTest < Minitest::Test
 
   def test_file6
     assert_equal "|", CsvSniffer.detect_delimiter(@@file6.path)
+    assert_equal true, CsvSniffer.has_header?(@@file6.path)
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csv_sniffer
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors:
 - Tim Ojo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-10 00:00:00.000000000 Z
+date: 2015-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: test-unit