hippie_csv 0.0.1 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 99c371eea25788cf2b155a089ef2378ce564a039
-  data.tar.gz: 53c18182171b267f4c6a13f50b8672d39e177296
+  metadata.gz: 662fc68658433183bde7f943fff313555281db4a
+  data.tar.gz: c0202320766061ee14a7fc7077e3d85d9f0548d7
 SHA512:
-  metadata.gz: 30ece254d7ba4e882d529c839fafa551979b3974d2af14ced46587d369a99ff94d9a3528bddac6ea174761a4430bd1d3b223717a5791cf8559218024da14f507
-  data.tar.gz: d9ce69502180d757d890d69b30985a7200ab9611f1694bbb140c399c2e577764c07d2839a7e9cea856d8767de23dbe6a463e52f44645c3a3b089a8e49e3acf45
+  metadata.gz: 659b01340829dc9056ef86f5df2ad8eb6c70f5ed66977f39bc7720149c5274d0ea26861b4bc7db20c2bc72ff3ca9e9165e09dde31812deb486b0974cbeb2a4d8
+  data.tar.gz: 5c9118714c431e0385598759b22d6b7d3f90c5b490afb1b8aaf911620a9ae5031c9bce34d0a591b602e395bd4ab8fd35784ef9ecc364f6e3aabbab68fc41344a

data/README.md

@@ -3,7 +3,7 @@
 [ ![Codeship Status for intercom/hippie_csv](https://codeship.com/projects/f3b188e0-f312-0132-75cb-5ed004d44c71/status?branch=master)](https://codeship.com/projects/85324)
 
 Ruby's `CSV` is great. It complies with the [proposed CSV spec](https://www.ietf.org/rfc/rfc4180.txt)
-pretty well. If you pass its methods bad or incompliant CSVs, it’ll rightfully
+pretty well. If you pass its methods bad or non-compliant CSVs, it’ll rightfully
 and loudly complain. It’s great 👍
 
 Except…if you want to be able to deal with files from the real world. At

data/hippie_csv.gemspec

@@ -21,4 +21,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "pry"
+
+  spec.add_dependency "charlock_holmes"
 end

data/lib/hippie_csv/errors.rb

@@ -1,7 +1,7 @@
 module HippieCSV
   class UnableToParseError < StandardError
-    def self.explain
-      raise new "Something went wrong. Report this CSV: https://github.com/intercom/hippie_csv"
+    def initialize(msg = "Something went wrong. Report this CSV: https://github.com/intercom/hippie_csv")
+      super(msg)
     end
   end
 end

data/lib/hippie_csv/support.rb

@@ -1,4 +1,5 @@
 require "hippie_csv/constants"
+require "charlock_holmes"
 
 module HippieCSV
   module Support
@@ -7,13 +8,19 @@ module HippieCSV
         File.read(file_path, encoding: ENCODING_WITH_BOM)
       end
 
-      def encode!(string)
+      def encode(string)
         unless string.valid_encoding?
-          string.encode!(ALTERNATE_ENCODING, ENCODING, invalid: :replace, replace: "")
-          string.encode!(ENCODING, ALTERNATE_ENCODING)
+          current_encoding = CharlockHolmes::EncodingDetector.detect(string)[:encoding]
+
+          string = if !current_encoding.nil?
+            CharlockHolmes::Converter.convert(string, current_encoding, ENCODING)
+          else
+            string.encode(ALTERNATE_ENCODING, ENCODING, invalid: :replace, replace: "")
+                  .encode(ENCODING, ALTERNATE_ENCODING)
+          end
         end
 
-        string.encode!(string.encoding, universal_newline: true)
+        string.encode(string.encoding, universal_newline: true)
       end
 
       def maybe_parse(string)

data/lib/hippie_csv/version.rb

@@ -1,3 +1,3 @@
 module HippieCSV
-  VERSION = "0.0.1"
+  VERSION = "0.0.3"
 end

data/lib/hippie_csv.rb

@@ -9,7 +9,7 @@ module HippieCSV
   end
 
   def self.parse(string)
-    Support.encode!(string)
-    Support.maybe_parse(string) || UnableToParseError.explain
+    string = Support.encode(string)
+    Support.maybe_parse(string) || (raise UnableToParseError)
   end
 end

data/spec/fixtures/accents_semicolon_windows_1252.csv

@@ -0,0 +1,3 @@
+Vincent;J�rome;jvincent@example.com;06 02 21 32 91;Vincent J�rome;
+H�lo�se;Martin ;martinheloise@example.com;06 12 81 12 28;;
+Guzole;Virgile ;v.guzole@example.com ;02 23 23 20 10;;

data/spec/hippie_csv/support_spec.rb

@@ -23,7 +23,7 @@ describe HippieCSV::Support do
     end
   end
 
-  describe ".encode!" do
+  describe ".encode" do
     context "with invalid byte sequence" do
       let(:string) { "\u0014\xFE\u000E\u0000" }
 
@@ -34,9 +34,7 @@ describe HippieCSV::Support do
       it "works" do
         expect(string).not_to be_valid_encoding
 
-        subject.encode!(string)
-
-        expect(string).to be_valid_encoding
+        expect(subject.encode(string)).to be_valid_encoding
       end
     end
 
@@ -48,9 +46,8 @@ describe HippieCSV::Support do
       end
 
       it "works" do
-        subject.encode!(string)
+        result = CSV.parse(subject.encode(string))
 
-        result = CSV.parse(string)
         rows, columns = result.size, result.first.size
 
         expect(rows).to eq(2)

data/spec/hippie_csv/version_spec.rb

@@ -3,7 +3,7 @@ require "spec_helper"
 describe HippieCSV do
 
   it "defines a version" do
-    expect(HippieCSV::VERSION).to eq('0.0.1')
+    expect(HippieCSV::VERSION).to eq("0.0.3")
   end
 
 end

data/spec/hippie_csv_spec.rb

@@ -17,7 +17,8 @@ describe HippieCSV do
 
   describe ".parse" do
     it "encodes the string" do
-      expect(subject::Support).to receive(:encode!).with(string)
+      expect(subject::Support).to receive(:encode).with(string)
+      allow(subject::Support).to receive(:maybe_parse).and_return(double)
 
       subject.parse(string)
     end
@@ -100,5 +101,13 @@ describe HippieCSV do
       import = subject.read(path)
       expect(import[0].count).to eq(9)
     end
+
+    it "works for a hard case" do
+      path = fixture_path(:accents_semicolon_windows_1252)
+
+      import = subject.read(path)
+      expect(import[0][1]).to eq("Jérome")
+      expect(import[1][0]).to eq("Héloïse")
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hippie_csv
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.3
 platform: ruby
 authors:
 - Stephen O'Brien
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-12 00:00:00.000000000 Z
+date: 2015-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,6 +66,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: charlock_holmes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - stephen@intercom.io
@@ -85,6 +99,7 @@ files:
 - lib/hippie_csv/errors.rb
 - lib/hippie_csv/support.rb
 - lib/hippie_csv/version.rb
+- spec/fixtures/accents_semicolon_windows_1252.csv
 - spec/fixtures/dos_line_ending.csv
 - spec/fixtures/encoding.csv
 - spec/fixtures/escaped_quotes.csv
@@ -123,6 +138,7 @@ signing_key:
 specification_version: 4
 summary: Tolerant, liberal CSV parsing
 test_files:
+- spec/fixtures/accents_semicolon_windows_1252.csv
 - spec/fixtures/dos_line_ending.csv
 - spec/fixtures/encoding.csv
 - spec/fixtures/escaped_quotes.csv