hippie_csv 0.0.11 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 07464f9018551eea7e183ef2ffd0e855b8a46d6a
-  data.tar.gz: 98b28c4757768af0fe785127914bdc964a067cff
+  metadata.gz: aca3f93c2f4a521f946abc23a9bf258fa32e1e5f
+  data.tar.gz: e7bb585167d4a9cc7b351593650991e7383359a5
 SHA512:
-  metadata.gz: 346448b5193bc9beeddee5ec1dfe066a795d0a8820b2dbf28910925bf461feaec1cbbcd886394bf8323d47d931cbfda6e540febe5d5a51935b6fc127c0b61573
-  data.tar.gz: ec3278642148bc8597af5d5128fe2c9ac2231fa7b87c19dc40d3a0933ba0a41c0407464bcbe938609e3ea55c72587d39c3c50d56b04c280e4300bd8f5f27e44e
+  metadata.gz: 37be4a8ccff26295f9af6109207b8337c9bb0b9bddc392beb4ad7a668ff1102525dfb36e7eacdc462d5da6b61ac50de2681b2e6c3e3b3d2c8dbd46231dcd5748
+  data.tar.gz: bd0a83a02adf59c29fb7ebc8644b500d728552e902d04b4d5cb8262e8b485fa0c92d42b569ee6be6cafeeec5e65af566b9b809543fc0a7f96bc0d899f4f374cc

data/README.md

@@ -31,14 +31,23 @@ Or install it yourself as:
 
 ## Usage
 
-Exposes two public methods: `read` (for paths to files), and `parse` (for
-strings).
+Exposes three public methods:
+1. `.read` a file path to an array. Reads from the file all at once, building the whole CSV object in memory.
+2. `.parse` an in memory string to an array.
+3. `.stream` from a file path and parse line by line, calling a given block on each row.
+
+**Note**: Processing large files using read or parse is a memory intensive operation. Use stream for parsing a CSV file line by line from the file to save memory. This method will use less memory but take longer, as we run each line through parse.
+
 
 ```ruby
 require 'hippie_csv'
 
 HippieCSV.read("path/to/data.csv")
 
+HippieCSV.stream("path/to/data.csv") do |row|
+  # use row here...
+end
+
 HippieCSV.parse(csv_string)
 ```
 

data/lib/hippie_csv.rb

@@ -4,12 +4,15 @@ require "hippie_csv/errors"
 
 module HippieCSV
   def self.read(path)
-    string = Support.file_path_to_string(path)
+    string = File.read(path, encoding: ENCODING_WITH_BOM)
     parse(string)
   end
 
   def self.parse(string)
-    string = Support.encode(string)
     Support.maybe_parse(string) || (raise UnableToParseError)
   end
+
+  def self.stream(path, &block)
+    Support.maybe_stream(path, &block)
+  end
 end

data/lib/hippie_csv/support.rb

@@ -4,10 +4,6 @@ require "rchardet"
 module HippieCSV
   module Support
     class << self
-      def file_path_to_string(file_path)
-        File.read(file_path, encoding: ENCODING_WITH_BOM)
-      end
-
       def encode(string)
         string = ensure_valid_encoding(string)
 
@@ -19,8 +15,10 @@ module HippieCSV
       end
 
       def maybe_parse(string)
+        encoded_string = encode(string)
+
         QUOTE_CHARACTERS.find do |quote_character|
-          [string, tolerate_escaping(string, quote_character), dump_quotes(string, quote_character)].find do |string_to_parse|
+          [encoded_string, tolerate_escaping(encoded_string, quote_character), dump_quotes(encoded_string, quote_character)].find do |string_to_parse|
             rescuing_malformed do
               return parse_csv(string_to_parse.squeeze("\n").strip, quote_character)
             end
@@ -36,6 +34,13 @@ module HippieCSV
         )
       end
 
+      def maybe_stream(path, &block)
+        File.foreach(path, encoding: ENCODING_WITH_BOM) do |line|
+          row = maybe_parse(line)
+          block.call(row.first) if row.first
+        end
+      end
+
       def dump_quotes(string, quote_character)
         string.gsub(quote_character, "")
       end

data/lib/hippie_csv/version.rb

@@ -1,3 +1,3 @@
 module HippieCSV
-  VERSION = "0.0.11"
+  VERSION = "0.1.0"
 end

data/spec/fixtures/small_file.csv

@@ -0,0 +1,2 @@
+name,email
+stephen,test@example.com

data/spec/hippie_csv/support_spec.rb

@@ -2,27 +2,6 @@ require "spec_helper"
 
 describe HippieCSV::Support do
 
-  describe ".file_to_string" do
-    let(:file_path) { fixture_path(:normal) }
-    let(:result) { HippieCSV::Support.file_path_to_string(file_path) }
-
-    it "provides a string" do
-      expect(result.class).to eq String
-    end
-
-    it "reads the file" do
-      expect(result.slice(0,8)).to eq 'id,email'
-    end
-
-    context "with a byte order mark" do
-      let(:file_path) { fixture_path(:with_byte_order_mark) }
-
-      it "works" do
-        expect(result).to eq '"Name","Email Address","Date Added"'
-      end
-    end
-  end
-
   describe ".encode" do
     context "with invalid byte sequence" do
       let(:string) { "\u0014\xFE\u000E\u0000" }
@@ -68,8 +47,23 @@ describe HippieCSV::Support do
   end
 
   describe ".maybe_parse" do
-    it "needs to be written" do
-      skip # TODO write this test
+    let(:file_path) { fixture_path(:small_file) }
+    it "works" do
+      expect(subject.maybe_parse(File.read(file_path))).to eq(
+        [["name", "email"], ["stephen", "test@example.com"]]
+      )
+    end
+  end
+
+  describe ".maybe_stream" do
+    let(:file_path) { fixture_path(:small_file) }
+    it "works" do
+      result = []
+      subject.maybe_stream(file_path) { |row| result << row }
+
+      expect(result).to eq(
+        [["name", "email"], ["stephen", "test@example.com"]]
+      )
     end
   end
 

data/spec/hippie_csv/version_spec.rb

@@ -3,7 +3,7 @@ require "spec_helper"
 describe HippieCSV do
 
   it "defines a version" do
-    expect(HippieCSV::VERSION).to eq("0.0.11")
+    expect(HippieCSV::VERSION).to eq("0.1.0")
   end
 
 end

data/spec/hippie_csv_spec.rb

@@ -2,27 +2,28 @@ require "spec_helper"
 require "csv"
 
 describe HippieCSV do
+
   let(:string) { "test" }
 
   describe ".read" do
-    let(:path) { double }
 
-    it "converts to string and parses" do
-      expect(subject::Support).to receive(:file_path_to_string).with(path).and_return(string)
-      expect(subject).to receive(:parse).with(string)
+    it "reads and parses the file" do
+      path = fixture_path(:normal)
 
-      subject.read(path)
+      result = subject.read(path)
+      expect(result.first[0..1]).to eq(["id", "email"])
     end
-  end
 
-  describe ".parse" do
-    it "encodes the string" do
-      expect(subject::Support).to receive(:encode).with(string)
-      allow(subject::Support).to receive(:maybe_parse).and_return(double)
+    it "reads and parses the file with a byte order mark" do
+      path = fixture_path(:with_byte_order_mark)
 
-      subject.parse(string)
+      result = subject.read(path)
+      expect(result).to eq([["Name", "Email Address", "Date Added"]])
     end
 
+  end
+
+  describe ".parse" do
     it "defers to support parse method" do
       result = double
       expect(subject::Support).to receive(:maybe_parse).with(string).and_return(result)
@@ -46,115 +47,156 @@ describe HippieCSV do
     end
   end
 
-  context "integration cases: hard/encountered problems" do
-    it "works when a BOM is present in the file" do
-      path = fixture_path(:with_byte_order_mark)
-      expect { CSV.read(path) }.to raise_error(CSV::MalformedCSVError)
+  describe ".stream" do
+    path = fixture_path(:normal)
+    let(:proc) { Proc.new {} }
 
-      import = subject.read(path)
-      expect(import[0]).to eq(["Name", "Email Address", "Date Added"])
-    end
+    it "encodes the string" do
+      allow(subject::Support).to receive(:maybe_stream).and_return(double)
 
-    it "works with a malformed CSV" do
-      path = fixture_path(:malformed)
-      expect { CSV.read(path) }.to raise_error(CSV::MalformedCSVError)
+      subject.stream(path, &proc)
+    end
 
-      import = subject.read(path)
-      expect(import[0]).to eq(%w(site lon lat max min precip snow snowdepth))
+    it "defers to support stream method" do
+      result = double
+      expect(subject::Support).to receive(:maybe_stream).with(path, &proc).and_return(result)
+      expect(subject.stream(path, &proc)).to eq(result)
     end
 
-    it "works with odd encoding & emoji!" do
-      path = fixture_path(:encoding)
-      expect { CSV.read(path) }.to raise_error(ArgumentError)
+    it "works" do
+      path = fixture_path(:normal)
 
-      import = subject.read(path)
-      expect(import[0].count).to eq(4)
+      result = []
+      subject.stream(path) { |row| result << row }
+      expect(result[0]).to eq(["id", "email", "name", "country", "city", "created_at", "admin"])
     end
+  end
 
-    it "works with an excel export" do
-      path = fixture_path(:excel)
+  context "integration cases: hard/encountered problems" do
 
-      import = subject.read(path)
-      expect(import[0].count).to eq(24)
+    def read(path)
+      subject.read(path)
     end
 
-    it "works with unescaped internal quotes" do
-      path = fixture_path(:internal_quotes)
-      expect { CSV.read(path) }.to raise_error(CSV::MalformedCSVError)
+    def stream(path)
+      [].tap do |rows|
+        subject.stream(path) do |row|
+          rows << row
+        end
+      end
+    end
 
-      import = subject.read(path)
-      expect(import[1][1]).to eq("123")
-      expect(import[1][2]).to eq("James Jimmy Doe")
+    def subject_call_method(method, path)
+      send(method, path)
     end
 
-    it "works with escaped quotes" do
-      path = fixture_path(:escaped_quotes)
+    it "::read deals with a long, challenging file (and quickly)" do
+      start_time = Time.now
+      path = fixture_path(:never_ordered)
 
       import = subject.read(path)
-      expect(import[0][1]).to eq("Lalo \"ElPapi\" Neymar")
-      expect(import[0][2]).to eq("lalo@example.com")
+
+      expect(import[0].count).to eq(10)
+      expect(import.count).to eq(32803)
+      expect(Time.now).to be_within(5).of(start_time)
     end
 
-    it "works with an invalid escaped quotes case" do
-      path = fixture_path(:escaped_quotes_semicolons)
+    %w[read stream].each do |method|
+      it "::#{method} works when a BOM is present in the file" do
+        path = fixture_path(:with_byte_order_mark)
 
-      import = subject.read(path)
-      expect(import[0][0]).to eq("133")
-      expect(import[0][1]).to eq("z3268856")
-      expect(import[0][2]).to eq("stephen@example.com")
-    end
+        import = subject_call_method(method, path)
+        expect(import[0]).to eq(["Name", "Email Address", "Date Added"])
+      end
 
-    it "works for a complicated case involving bad newlines and quote chars" do
-      path = fixture_path(:dos_line_ending)
+      it "::#{method} works with a malformed CSV" do
+        path = fixture_path(:malformed)
+        expect { CSV.read(path) }.to raise_error(CSV::MalformedCSVError)
 
-      import = subject.read(path)
-      expect(import[0].count).to eq(9)
-    end
+        import = subject_call_method(method, path)
+        expect(import[0]).to eq(%w(site lon lat max min precip snow snowdepth))
+      end
 
-    it "works for a hard case" do
-      path = fixture_path(:accents_semicolon_windows_1252)
+      it "::#{method} works with odd encoding & emoji!" do
+        path = fixture_path(:encoding)
+        expect { CSV.read(path) }.to raise_error(ArgumentError)
 
-      import = subject.read(path)
-      expect(import[0][1]).to eq("Jérome")
-      expect(import[1][0]).to eq("Héloise")
-    end
+        import = subject_call_method(method, path)
+        expect(import[0].count).to eq(4)
+      end
 
-    it "deals with a long, challenging file (and quickly)" do
-      start_time = Time.now
-      path = fixture_path(:never_ordered)
+      it "::#{method} works with an excel export" do
+        path = fixture_path(:excel)
 
-      import = subject.read(path)
+        import = subject_call_method(method, path)
+        expect(import[0].count).to eq(24)
+      end
 
-      expect(import[0].count).to eq(10)
-      expect(import.count).to eq(32803)
-      expect(Time.now).to be_within(5).of(start_time)
-    end
+      it "::#{method} works with unescaped internal quotes" do
+        path = fixture_path(:internal_quotes)
+
+        import = subject_call_method(method, path)
+        expect(import[1][1]).to eq("123")
+        expect(import[1][2]).to eq("James Jimmy Doe")
+      end
 
-    it "works when many invalid quote types contained" do
-      path = fixture_path(:bad_quoting)
+      it "::#{method} works with escaped quotes" do
+        path = fixture_path(:escaped_quotes)
 
-      expect { CSV.read(path) }.to raise_error(CSV::MalformedCSVError)
-      expect {
-        import = subject.read(path)
-        expect(import.map(&:count).uniq).to eq([11])
-        expect(import.count).to eq(8)
-      }.not_to raise_error
-    end
+        import = subject_call_method(method, path)
+        expect(import[0][1]).to eq("Lalo \"ElPapi\" Neymar")
+        expect(import[0][2]).to eq("lalo@example.com")
+      end
 
-    it "strips leading/trailing blank lines" do
-      path = fixture_path(:trailing_leading_blank_lines)
+      it "::#{method} works with an invalid escaped quotes case" do
+        path = fixture_path(:escaped_quotes_semicolons)
 
-      import = subject.read(path)
-      expect(import.first).not_to be_empty
-      expect(import.last).not_to be_empty
-    end
+        import = subject_call_method(method, path)
+        expect(import[0][0]).to eq("133")
+        expect(import[0][1]).to eq("z3268856")
+        expect(import[0][2]).to eq("stephen@example.com")
+      end
+
+      it "::#{method} works for a complicated case involving bad newlines and quote chars" do
+        path = fixture_path(:dos_line_ending)
+
+        import = subject_call_method(method, path)
+        expect(import[0].count).to eq(9)
+      end
+
+      it "::#{method} works for a hard case" do
+        path = fixture_path(:accents_semicolon_windows_1252)
+
+        import = subject_call_method(method, path)
+        expect(import[0][1]).to eq("Jérome")
+        expect(import[1][0]).to eq("Héloise")
+      end
+
+      it "::#{method} works when many invalid quote types contained" do
+        path = fixture_path(:bad_quoting)
+
+        expect {
+          import = subject_call_method(method, path)
+          expect(import.map(&:count).uniq).to eq([11])
+          expect(import.count).to eq(8)
+        }.not_to raise_error
+      end
+
+      it "::#{method} strips leading/trailing blank lines" do
+        path = fixture_path(:trailing_leading_blank_lines)
+
+        import = subject_call_method(method, path)
+        expect(import.first).not_to be_empty
+        expect(import.last).not_to be_empty
+      end
 
-    it "maintains coherent column count when stripping blank lines" do
-      [:blank_lines_crlf, :trailing_leading_blank_lines].each do |fixture_name|
-        path = fixture_path(fixture_name)
+      it "::#{method} maintains coherent column count when stripping blank lines" do
+        [:blank_lines_crlf, :trailing_leading_blank_lines].each do |fixture_name|
+          path = fixture_path(fixture_name)
 
-        import = subject.read(path)
-        expect(import.map(&:length).uniq.size).to eq(1)
+          import = subject_call_method(method, path)
+          expect(import.map(&:length).uniq.size).to eq(1)
+        end
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hippie_csv
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.1.0
 platform: ruby
 authors:
 - Stephen O'Brien
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-01 00:00:00.000000000 Z
+date: 2017-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -111,6 +111,7 @@ files:
 - spec/fixtures/malformed.csv
 - spec/fixtures/never_ordered.csv
 - spec/fixtures/normal.csv
+- spec/fixtures/small_file.csv
 - spec/fixtures/trailing_leading_blank_lines.csv
 - spec/fixtures/with_byte_order_mark.csv
 - spec/hippie_csv/constants_spec.rb
@@ -138,7 +139,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Tolerant, liberal CSV parsing
@@ -155,6 +156,7 @@ test_files:
 - spec/fixtures/malformed.csv
 - spec/fixtures/never_ordered.csv
 - spec/fixtures/normal.csv
+- spec/fixtures/small_file.csv
 - spec/fixtures/trailing_leading_blank_lines.csv
 - spec/fixtures/with_byte_order_mark.csv
 - spec/hippie_csv/constants_spec.rb