postgres-copy 1.4.0 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2d64b9718c2f756ad9c55bb61a173ee66b0ed5f7
-  data.tar.gz: b8a084b47fa7136bce031104d11b873482e13071
+  metadata.gz: 6477dfff9506950fc1f3a728138a48ce5f669a0c
+  data.tar.gz: 6a048abf7e41bbec754e4603b6f0fce48c6c3aa7
 SHA512:
-  metadata.gz: d8b2b7d20479b9bc5e63a50d2c2ee256f26da73e3baf7d526e7f7976ca61684e022c62e2034acffeeb82ec184e1195c12f08940bb78def6dad9af29c068d558c
-  data.tar.gz: 865261adbe21c5e16d2c928336b99b2e21bc49f5dd0025d49b3edeb89eccaff38f55a5af2ed6ed52e4ff0e01017c975d63ee8ebccebd41b44e4bb2ad2506e9d2
+  metadata.gz: 29f73c848994f32475f6652fe07788180efa8ab3faede9e98bc4d13d1f5afd9a95d165c2c8eb9c077fa9c8780aab876c167b1aa0fef49875b4a27c554785ee61
+  data.tar.gz: dba63f2014d5caad8e5f3623e85abf684f97ed8fd6ccee207f10033a4b090d2fa469d401b61173e1d0967988bc2e370434dee13b33525a863c35f864c0bdb987

data/.travis.yml

@@ -2,6 +2,12 @@ language: ruby
 rvm:
   - 2.3.1
 
+services:
+  - postgresql
+
+addons:
+  postgresql: '9.4'
+
 branches:
   only:
     - master

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    postgres-copy (1.4.0)
+    postgres-copy (1.4.1)
       activerecord (>= 5.1)
       pg (>= 0.17)
       responders
@@ -9,63 +9,66 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (5.1.4)
-      actionview (= 5.1.4)
-      activesupport (= 5.1.4)
+    actionpack (5.2.2)
+      actionview (= 5.2.2)
+      activesupport (= 5.2.2)
       rack (~> 2.0)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    actionview (5.1.4)
-      activesupport (= 5.1.4)
+    actionview (5.2.2)
+      activesupport (= 5.2.2)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.3)
-    activemodel (5.1.4)
-      activesupport (= 5.1.4)
-    activerecord (5.1.4)
-      activemodel (= 5.1.4)
-      activesupport (= 5.1.4)
-      arel (~> 8.0)
-    activesupport (5.1.4)
+    activemodel (5.2.2)
+      activesupport (= 5.2.2)
+    activerecord (5.2.2)
+      activemodel (= 5.2.2)
+      activesupport (= 5.2.2)
+      arel (>= 9.0)
+    activesupport (5.2.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (~> 0.7)
+      i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    arel (8.0.0)
+    arel (9.0.0)
     builder (3.2.3)
-    concurrent-ruby (1.0.5)
+    concurrent-ruby (1.1.4)
+    crass (1.0.4)
     diff-lcs (1.3)
-    erubi (1.6.1)
-    i18n (0.8.6)
-    loofah (2.0.3)
+    erubi (1.8.0)
+    i18n (1.5.3)
+      concurrent-ruby (~> 1.0)
+    loofah (2.2.3)
+      crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
-    method_source (0.8.2)
-    mini_portile2 (2.3.0)
-    minitest (5.10.3)
-    nokogiri (1.8.1)
-      mini_portile2 (~> 2.3.0)
-    pg (0.21.0)
-    rack (2.0.3)
-    rack-test (0.7.0)
+    method_source (0.9.2)
+    mini_portile2 (2.4.0)
+    minitest (5.11.3)
+    nokogiri (1.10.1)
+      mini_portile2 (~> 2.4.0)
+    pg (1.1.4)
+    rack (2.0.6)
+    rack-test (1.1.0)
       rack (>= 1.0, < 3)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.0.3)
-      loofah (~> 2.0)
-    railties (5.1.4)
-      actionpack (= 5.1.4)
-      activesupport (= 5.1.4)
+    rails-html-sanitizer (1.0.4)
+      loofah (~> 2.2, >= 2.2.2)
+    railties (5.2.2)
+      actionpack (= 5.2.2)
+      activesupport (= 5.2.2)
       method_source
       rake (>= 0.8.7)
-      thor (>= 0.18.1, < 2.0)
+      thor (>= 0.19.0, < 2.0)
     rake (11.2.2)
     rdoc (5.0.0)
-    responders (2.4.0)
-      actionpack (>= 4.2.0, < 5.3)
-      railties (>= 4.2.0, < 5.3)
+    responders (2.4.1)
+      actionpack (>= 4.2.0, < 6.0)
+      railties (>= 4.2.0, < 6.0)
     rspec (2.99.0)
       rspec-core (~> 2.99.0)
       rspec-expectations (~> 2.99.0)
@@ -74,9 +77,9 @@ GEM
     rspec-expectations (2.99.2)
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.99.4)
-    thor (0.20.0)
+    thor (0.20.3)
     thread_safe (0.3.6)
-    tzinfo (1.2.3)
+    tzinfo (1.2.5)
       thread_safe (~> 0.1)
 
 PLATFORMS
@@ -90,4 +93,4 @@ DEPENDENCIES
   rspec (~> 2.12)
 
 BUNDLED WITH
-   1.14.6
+   1.16.3

data/README.md

@@ -55,8 +55,8 @@ This will execute in the database the command:
 COPY (SELECT "users".* FROM "users" ) TO '/tmp/users.csv' WITH DELIMITER ',' CSV HEADER
 ```
 
-Remark that the file will be created in the database server disk.  
-But what if you want to write the lines in a file on the server that is running Rails, instead of the database?  
+Remark that the file will be created in the database server disk.
+But what if you want to write the lines in a file on the server that is running Rails, instead of the database?
 In this case you can pass a block and retrieve the generated lines and then write them to a file:
 
 ```ruby
@@ -166,6 +166,12 @@ To specify NULL value you can pass the null option parameter.
 User.copy_from "/tmp/users.csv", :null => 'null'
 ```
 
+Match the specified columns' values against the null string, even if it has been quoted, and if a match is found set the value to NULL (Postgres 9.4+ only).
+
+```ruby
+User.copy_from "/tmp/users.csv", :null => '', :force_null => [:name, :city]
+```
+
 
 To copy a binary formatted data file or IO object you can specify the format as binary
 

data/lib/postgres-copy/acts_as_copy_target.rb

@@ -77,8 +77,9 @@ module PostgresCopy
                            "BINARY"
                          else
                            quote = options[:quote] == "'" ? "''" : options[:quote]
-                           null = options.key?(:null) ? "NULL '#{options[:null]}'" : ''
-                           "DELIMITER '#{options[:delimiter]}' QUOTE '#{quote}' #{null} CSV"
+                           null = options.key?(:null) ? "NULL '#{options[:null]}'" : nil
+                           force_null = options.key?(:force_null) ? "FORCE_NULL(#{options[:force_null].join(',')})" : nil
+                           "WITH (" + ["DELIMITER '#{options[:delimiter]}'", "QUOTE '#{quote}'", null, force_null, "FORMAT CSV"].compact.join(', ') + ")"
                          end
         io = path_or_io.instance_of?(String) ? File.open(path_or_io, 'r') : path_or_io
 
@@ -110,15 +111,31 @@ module PostgresCopy
             rescue EOFError
             end
           else
+            line_buffer = ''
+
             while line = io.gets do
               next if line.strip.size == 0
-              if block_given?
-                row = CSV.parse_line(line.strip, {:col_sep => options[:delimiter]})
-                yield(row)
-                next if row.all?{|f| f.nil? }
-                line = CSV.generate_line(row, {:col_sep => options[:delimiter]})
+
+              line_buffer += line
+
+              # If line is incomplete, get the next line until it terminates
+              if line_buffer =~ /\n$/ || line_buffer =~ /\Z/
+                if block_given?
+                  begin
+                    row = CSV.parse_line(line_buffer.strip, {:col_sep => options[:delimiter]})
+                    yield(row)
+                    next if row.all?{|f| f.nil? }
+                    line_buffer = CSV.generate_line(row, {:col_sep => options[:delimiter]})
+                  rescue CSV::MalformedCSVError
+                    next
+                  end
+                end
+
+                connection.raw_connection.put_copy_data(line_buffer)
+
+                # Clear the buffer
+                line_buffer = ''
               end
-              connection.raw_connection.put_copy_data line
             end
           end
         end

data/postgres-copy.gemspec

@@ -5,7 +5,7 @@ $:.unshift lib unless $:.include?(lib)
 
 Gem::Specification.new do |s|
   s.name                  = "postgres-copy"
-  s.version               = "1.4.0"
+  s.version               = "1.4.1"
   s.platform              = Gem::Platform::RUBY
   s.required_ruby_version = ">= 1.9.3"
   s.authors               = ["Diogo Biazus"]

data/spec/copy_from_spec.rb

@@ -100,20 +100,20 @@ describe "COPY FROM" do
     ReservedWordModel.copy_from File.expand_path('spec/fixtures/reserved_words.csv'), :delimiter => "\t"
     ReservedWordModel.order(:id).map{|r| r.attributes}.should == [{"group"=>"group name", "id"=>1, "select"=>"test select"}]
   end
-  
+
   it "should import even last columns have empty values" do
     TestExtendedModel.copy_from File.expand_path('spec/fixtures/comma_with_header_empty_values_at_the_end.csv')
-    TestExtendedModel.order(:id).map{|r| r.attributes}.should == 
+    TestExtendedModel.order(:id).map{|r| r.attributes}.should ==
       [{"id"=>1, "data"=>"test data 1", "more_data"=>nil, "other_data"=>nil, "final_data"=>nil},
        {"id"=>2, "data"=>"test data 2", "more_data"=>"9", "other_data"=>nil, "final_data"=>nil},
        {"id"=>3, "data"=>"test data 2", "more_data"=>"9", "other_data"=>nil, "final_data"=>"0"}]
   end
-  
+
   it "should import even last columns have empty values with block" do
     TestExtendedModel.copy_from File.expand_path('spec/fixtures/comma_with_header_empty_values_at_the_end.csv') do |row|
       row[4]="666"
     end
-    TestExtendedModel.order(:id).map{|r| r.attributes}.should == 
+    TestExtendedModel.order(:id).map{|r| r.attributes}.should ==
       [{"id"=>1, "data"=>"test data 1", "more_data"=>nil, "other_data"=>nil, "final_data"=>"666"},
        {"id"=>2, "data"=>"test data 2", "more_data"=>"9", "other_data"=>nil, "final_data"=>"666"},
        {"id"=>3, "data"=>"test data 2", "more_data"=>"9", "other_data"=>nil, "final_data"=>"666"}]
@@ -150,4 +150,14 @@ describe "COPY FROM" do
     TestModel.copy_from File.open(File.expand_path('spec/fixtures/special_null_with_header.csv'), 'r'), :null => 'NULL'
     TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => nil}]
   end
+
+  it "should import with a carriage return in the value" do
+    TestModel.copy_from File.expand_path('spec/fixtures/comma_with_carriage_returns.csv')
+    TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => "test\ndata 1"}]
+  end
+
+  it "should import custom force null expressions from path" do
+    TestModel.copy_from File.expand_path('spec/fixtures/comma_with_empty_string.csv'), :null => '', :force_null => [:data]
+    TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => nil}]
+  end
 end

data/spec/fixtures/comma_with_carriage_returns.csv

@@ -0,0 +1,3 @@
+id,data
+1,"test
+data 1"

data/spec/fixtures/comma_with_empty_string.csv

@@ -0,0 +1,2 @@
+id,data
+1,""

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: postgres-copy
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.4.1
 platform: ruby
 authors:
 - Diogo Biazus
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-28 00:00:00.000000000 Z
+date: 2019-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -124,7 +124,6 @@ files:
 - LICENSE
 - README.md
 - Rakefile
-- VERSION
 - lib/postgres-copy.rb
 - lib/postgres-copy/acts_as_copy_target.rb
 - lib/postgres-copy/csv_responder.rb
@@ -135,6 +134,8 @@ files:
 - spec/copy_to_spec.rb
 - spec/fixtures/2_col_binary_data.dat
 - spec/fixtures/comma_inside_field.csv
+- spec/fixtures/comma_with_carriage_returns.csv
+- spec/fixtures/comma_with_empty_string.csv
 - spec/fixtures/comma_with_header.csv
 - spec/fixtures/comma_with_header_and_scope.csv
 - spec/fixtures/comma_with_header_empty_values_at_the_end.csv
@@ -177,7 +178,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Put COPY command functionality in ActiveRecord's model class
@@ -188,6 +189,8 @@ test_files:
 - spec/copy_to_spec.rb
 - spec/fixtures/2_col_binary_data.dat
 - spec/fixtures/comma_inside_field.csv
+- spec/fixtures/comma_with_carriage_returns.csv
+- spec/fixtures/comma_with_empty_string.csv
 - spec/fixtures/comma_with_header.csv
 - spec/fixtures/comma_with_header_and_scope.csv
 - spec/fixtures/comma_with_header_empty_values_at_the_end.csv

data/VERSION

@@ -1 +0,0 @@
-0.5.6