hippie_csv 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 25ff6543c2223eed975723e6b5718081b2f03e33
-  data.tar.gz: 5f62a6c390164ade3e7880ed59171f56f6d608fe
+  metadata.gz: f53a562b556d85f2c893855cf0aa34ea6126afab
+  data.tar.gz: 9c1a2954a38b2c01b1a9dcbc13c25fb21e9799be
 SHA512:
-  metadata.gz: 43c0883650a3091512b8c60e55c882a22ea60fe0f946a9e74280739e0765306296fc53b8807189f9aecf71b76f41c7d23758056e025281d73e83639a301a4147
-  data.tar.gz: 8318a4e154ddce8ef9d5d835faf30b5ae08a94a252350380d59538a34e01a7de7c07ee968e521c1efd34d53a15720f21372beda179d0ca3513c2a02a644abc35
+  metadata.gz: 7db7d04670863b9bffc7c66d8c1410b57892b4729819c6755816bd3e5deffcf0acc2af75870d6e26b41dc5643f1ef7eb8ec9cce86aa9868fff49129c2872e204
+  data.tar.gz: 2a587740e0ee24c021a373c71e4c3038447de70595d7ff1e89d86dc51aa696083a238cef221bdb2e6970a53eca11d407b6a0c5ee11df1449e32305ce25113055

data/lib/hippie_csv/constants.rb

@@ -1,10 +1,12 @@
-require 'csv'
+require "csv"
 
 module HippieCSV
   QUOTE_CHARACTERS = %w(" ' |).freeze
   DELIMETERS = %W(, ; \t).freeze
   ENCODING = "utf-8".freeze
   ALTERNATE_ENCODING = "utf-16".freeze
-  FIELD_SAMPLE_COUNT = 10
+  FIELD_SAMPLE_COUNT = 10.freeze
+  ENCODING_SAMPLE_CHARACTER_COUNT = 10000.freeze
   ENCODING_WITH_BOM = "bom|#{ENCODING}".freeze
+  BLANK_LINE_REGEX = /^,+\r+$/.freeze
 end

data/lib/hippie_csv/support.rb

@@ -10,16 +10,18 @@ module HippieCSV
 
       def encode(string)
         unless string.valid_encoding?
-          current_encoding = CharDet.detect(string)["encoding"]
-
-          string = if !current_encoding.nil?
-            string.encode(ENCODING, current_encoding)
-          else
-            string.encode(ALTERNATE_ENCODING, ENCODING, invalid: :replace, replace: "")
-                  .encode(ENCODING, ALTERNATE_ENCODING)
+          string = begin
+            current_encoding = detect_encoding(string)
+            if !current_encoding.nil?
+              string.encode(ENCODING, current_encoding)
+            else
+              magical_encode(string)
+            end
+          rescue Encoding::InvalidByteSequenceError
+            magical_encode(string)
           end
         end
-
+        string.gsub!(BLANK_LINE_REGEX, "")
         string.encode(string.encoding, universal_newline: true)
       end
 
@@ -59,6 +61,15 @@ module HippieCSV
 
       private
 
+      def detect_encoding(string)
+        CharDet.detect(string[0..ENCODING_SAMPLE_CHARACTER_COUNT])["encoding"]
+      end
+
+      def magical_encode(string)
+        string.encode(ALTERNATE_ENCODING, ENCODING, invalid: :replace, replace: "")
+              .encode(ENCODING, ALTERNATE_ENCODING)
+      end
+
       def field_count(file, delimeter, quote_character)
         csv = CSV.new(file, col_sep: delimeter, quote_char: quote_character)
         csv.lazy.take(FIELD_SAMPLE_COUNT).map(&:size).inject(:+)

data/lib/hippie_csv/version.rb

@@ -1,3 +1,3 @@
 module HippieCSV
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end