postgres-copy 1.1.2 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3cc595142d45d6053c54bac79e0855c4d8136856
-  data.tar.gz: 548bbff5d3505a7c9721d0d69f20313072770dd3
+  metadata.gz: 1bf6eb9e338ede86cf702bcd1841142a54abf542
+  data.tar.gz: e187a19feb3b625151b21ac71d5a4325874f3fc3
 SHA512:
-  metadata.gz: e15905d7bcd481f38153decfc9350d0220642c62b35ea7d285293b3ccf2f50fd8cf4e87d1e4bc8c49bcb120e15b6a422b78f05313e3587ca94e0b578561fd28a
-  data.tar.gz: 1da873e2f16c582afb38eef9d4dba009067446df3c6835228ec31c1cf2253a8690dd3e716f162a05032b6f0d0becf0d8defcca575cf29c101755c9f651c88935
+  metadata.gz: f633038cbe888b0db2bb72c3c6611716ad7bf57cae50b94a784eeee402c1c2d0e2de6d3f142da7d1093afa00ab4825791b4a36c2c6fd27404eb22f4cb0846b25
+  data.tar.gz: 5c050adfe50105e1f2016665d5efd8b7190edfc6cdd878745edb37c2eeb2eacd098ab851141bda53337cb33ebbaf8eda258b358f6767141bf409d7a5a1672169

data/Gemfile.lock

@@ -1,70 +1,71 @@
 PATH
   remote: .
   specs:
-    postgres-copy (1.1.2)
-      activerecord (>= 4.0)
+    postgres-copy (1.2.0)
+      activerecord (>= 4.0, < 5.1)
       pg (>= 0.17)
       responders
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (5.0.1)
-      actionview (= 5.0.1)
-      activesupport (= 5.0.1)
+    actionpack (5.0.3)
+      actionview (= 5.0.3)
+      activesupport (= 5.0.3)
       rack (~> 2.0)
       rack-test (~> 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    actionview (5.0.1)
-      activesupport (= 5.0.1)
+    actionview (5.0.3)
+      activesupport (= 5.0.3)
       builder (~> 3.1)
       erubis (~> 2.7.0)
       rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    activemodel (5.0.1)
-      activesupport (= 5.0.1)
-    activerecord (5.0.1)
-      activemodel (= 5.0.1)
-      activesupport (= 5.0.1)
+      rails-html-sanitizer (~> 1.0, >= 1.0.3)
+    activemodel (5.0.3)
+      activesupport (= 5.0.3)
+    activerecord (5.0.3)
+      activemodel (= 5.0.3)
+      activesupport (= 5.0.3)
       arel (~> 7.0)
-    activesupport (5.0.1)
+    activesupport (5.0.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (~> 0.7)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
     arel (7.1.4)
     builder (3.2.3)
-    concurrent-ruby (1.0.4)
+    concurrent-ruby (1.0.5)
     diff-lcs (1.3)
     erubis (2.7.0)
-    i18n (0.8.1)
+    i18n (0.8.4)
     loofah (2.0.3)
       nokogiri (>= 1.5.9)
     method_source (0.8.2)
-    mini_portile2 (2.1.0)
-    minitest (5.10.1)
-    nokogiri (1.7.0.1)
-      mini_portile2 (~> 2.1.0)
-    pg (0.19.0)
-    rack (2.0.1)
+    mini_portile2 (2.2.0)
+    minitest (5.10.2)
+    nokogiri (1.8.0)
+      mini_portile2 (~> 2.2.0)
+    pg (0.21.0)
+    rack (2.0.3)
     rack-test (0.6.3)
       rack (>= 1.0)
-    rails-dom-testing (2.0.2)
-      activesupport (>= 4.2.0, < 6.0)
-      nokogiri (~> 1.6)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
     rails-html-sanitizer (1.0.3)
       loofah (~> 2.0)
-    railties (5.0.1)
-      actionpack (= 5.0.1)
-      activesupport (= 5.0.1)
+    railties (5.0.3)
+      actionpack (= 5.0.3)
+      activesupport (= 5.0.3)
       method_source
       rake (>= 0.8.7)
       thor (>= 0.18.1, < 2.0)
     rake (11.2.2)
     rdoc (5.0.0)
-    responders (2.3.0)
-      railties (>= 4.2.0, < 5.1)
+    responders (2.4.0)
+      actionpack (>= 4.2.0, < 5.3)
+      railties (>= 4.2.0, < 5.3)
     rspec (2.99.0)
       rspec-core (~> 2.99.0)
       rspec-expectations (~> 2.99.0)
@@ -75,7 +76,7 @@ GEM
     rspec-mocks (2.99.4)
     thor (0.19.4)
     thread_safe (0.3.6)
-    tzinfo (1.2.2)
+    tzinfo (1.2.3)
       thread_safe (~> 0.1)
 
 PLATFORMS
@@ -89,4 +90,4 @@ DEPENDENCIES
   rspec (~> 2.12)
 
 BUNDLED WITH
-   1.14.3
+   1.14.6

data/README.md

@@ -5,7 +5,7 @@ If you need to tranfer data between a PostgreSQL database and CSV files, the Pos
 will give you a greater performance than using the ruby CSV+INSERT commands.
 I have not found time to make accurate benchmarks, but in the use scenario where I have developed the gem
 I have had a four-fold performance gain.
-This gem was written having the Rails framework in mind, I think it could work only with active-record, 
+This gem was written having the Rails framework in mind, I think it could work only with active-record,
 but I will assume in this README that you are using Rails.
 
 ## Install
@@ -34,8 +34,9 @@ end
 
 This will add the aditiontal class methods to your model:
 
-* copy_to 
+* copy_to
 * copy_to_string
+* copy_to_enumerator
 * copy_from
 
 ### Using copy_to and copy_to_string
@@ -66,13 +67,23 @@ File.open('/tmp/users.csv', 'w') do |f|
 end
 ```
 
+Instead of yielding each line, you could return an enumerator with all users:
+```ruby
+enumerator = User.copy_to_enumerator
+```
+
+And for better performance when rendering the result of the enumerator, you can return an enumerator with blocks of 100 lines joined:
+```ruby
+enumerator = User.copy_to_enumerator(:buffer_lines => 100)
+```
+
 Or, if you have enough memory, you can read all table contents to a string using .copy_to_string
 
 ```ruby
 puts User.copy_to_string
 ```
 
-Another insteresting feature of copy_to is that it uses the scoped relation, it means that you can use ARel 
+Another insteresting feature of copy_to is that it uses the scoped relation, it means that you can use ARel
 operations to generate different CSV files according to your needs.
 Assuming we want to generate a file only with the names of users 1, 2 and 3:
 

data/lib/postgres-copy/acts_as_copy_target.rb

@@ -30,6 +30,32 @@ module PostgresCopy
         return self
       end
 
+      # Create an enumerator with each line from the CSV.
+      # Note that using this directly in a controller response
+      # will perform very poorly as each line will get put
+      # into its own chunk. Joining every (eg) 100 rows together
+      # is much, much faster.
+      def copy_to_enumerator(options={})
+        buffer_lines = options.delete(:buffer_lines)
+        # Somehow, self loses its scope once inside the Enumerator
+        scope = self.current_scope || self
+        result = Enumerator.new do |y|
+          scope.copy_to(nil, options) do |line|
+            y << line
+          end
+        end
+
+        if buffer_lines.to_i > 0
+          Enumerator.new do |y|
+            result.each_slice(buffer_lines.to_i) do |slice|
+              y << slice.join
+            end
+          end
+        else
+          result
+        end
+      end
+
       # Copy all data to a single string
       def copy_to_string options = {}
         data = ''

data/postgres-copy.gemspec

@@ -5,7 +5,7 @@ $:.unshift lib unless $:.include?(lib)
 
 Gem::Specification.new do |s|
   s.name                  = "postgres-copy"
-  s.version               = "1.1.2"
+  s.version               = "1.2.0"
   s.platform              = Gem::Platform::RUBY
   s.required_ruby_version = ">= 1.9.3"
   s.authors               = ["Diogo Biazus"]
@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
   s.summary               = "Put COPY command functionality in ActiveRecord's model class"
 
   s.add_dependency "pg", ">= 0.17"
-  s.add_dependency "activerecord", '>= 4.0'
+  s.add_dependency "activerecord", '>= 4.0', '< 5.1'
   s.add_dependency "responders"
   s.add_development_dependency "bundler"
   s.add_development_dependency "rdoc"

data/spec/copy_to_spec.rb

@@ -1,7 +1,7 @@
 require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 
 describe "COPY TO" do
-  before(:all) do
+  before(:each) do
     ActiveRecord::Base.connection.execute %{
       TRUNCATE TABLE test_models;
       SELECT setval('test_models_id_seq', 1, false);
@@ -21,6 +21,40 @@ describe "COPY TO" do
     end
   end
 
+  describe ".copy_to_enumerator" do
+    before(:each) do
+      TestModel.create :data => 'test data 2'
+      TestModel.create :data => 'test data 3'
+      TestModel.create :data => 'test data 4'
+    end
+
+    context "with no options" do
+      subject{ TestModel.copy_to_enumerator.to_a }
+      it{ should == File.open('spec/fixtures/comma_with_header_multi.csv', 'r').read.lines }
+    end
+
+    context "with tab as delimiter" do
+      subject{ TestModel.copy_to_enumerator(:delimiter => "\t").to_a }
+      it{ should == File.open('spec/fixtures/tab_with_header_multi.csv', 'r').read.lines }
+    end
+
+    context "with many records" do
+      context "enumerating in batches" do
+        subject{ TestModel.copy_to_enumerator(:buffer_lines => 2).to_a }
+        it do
+          expected = []
+          File.open('spec/fixtures/comma_with_header_multi.csv', 'r').read.lines.each_slice(2){|s| expected << s.join }
+          should == expected
+        end
+      end
+
+      context "excluding some records via a scope" do
+        subject{ TestModel.where("data not like '%3'").copy_to_enumerator.to_a }
+        it{ should == File.open('spec/fixtures/comma_with_header_and_scope.csv', 'r').read.lines }
+      end
+    end
+  end
+
   describe ".copy_to" do
     it "should copy and pass data to block if block is given and no path is passed" do
       File.open('spec/fixtures/comma_with_header.csv', 'r') do |f|

data/spec/fixtures/comma_with_header_and_scope.csv

@@ -0,0 +1,4 @@
+id,data
+1,test data 1
+2,test data 2
+4,test data 4

data/spec/fixtures/comma_with_header_multi.csv

@@ -0,0 +1,5 @@
+id,data
+1,test data 1
+2,test data 2
+3,test data 3
+4,test data 4

data/spec/fixtures/tab_with_header_multi.csv

@@ -0,0 +1,5 @@
+id	data
+1	test data 1
+2	test data 2
+3	test data 3
+4	test data 4

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: postgres-copy
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.2.0
 platform: ruby
 authors:
 - Diogo Biazus
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-09 00:00:00.000000000 Z
+date: 2017-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -31,6 +31,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -38,6 +41,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5.1'
 - !ruby/object:Gem::Dependency
   name: responders
   requirement: !ruby/object:Gem::Requirement
@@ -136,7 +142,9 @@ files:
 - spec/fixtures/2_col_binary_data.dat
 - spec/fixtures/comma_inside_field.csv
 - spec/fixtures/comma_with_header.csv
+- spec/fixtures/comma_with_header_and_scope.csv
 - spec/fixtures/comma_with_header_empty_values_at_the_end.csv
+- spec/fixtures/comma_with_header_multi.csv
 - spec/fixtures/comma_without_header.csv
 - spec/fixtures/extra_field.rb
 - spec/fixtures/reserved_word_model.rb
@@ -150,6 +158,7 @@ files:
 - spec/fixtures/tab_with_error.csv
 - spec/fixtures/tab_with_extra_line.csv
 - spec/fixtures/tab_with_header.csv
+- spec/fixtures/tab_with_header_multi.csv
 - spec/fixtures/tab_with_two_lines.csv
 - spec/fixtures/test_extended_model.rb
 - spec/fixtures/test_model.rb
@@ -174,7 +183,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: Put COPY command functionality in ActiveRecord's model class
@@ -186,7 +195,9 @@ test_files:
 - spec/fixtures/2_col_binary_data.dat
 - spec/fixtures/comma_inside_field.csv
 - spec/fixtures/comma_with_header.csv
+- spec/fixtures/comma_with_header_and_scope.csv
 - spec/fixtures/comma_with_header_empty_values_at_the_end.csv
+- spec/fixtures/comma_with_header_multi.csv
 - spec/fixtures/comma_without_header.csv
 - spec/fixtures/extra_field.rb
 - spec/fixtures/reserved_word_model.rb
@@ -200,6 +211,7 @@ test_files:
 - spec/fixtures/tab_with_error.csv
 - spec/fixtures/tab_with_extra_line.csv
 - spec/fixtures/tab_with_header.csv
+- spec/fixtures/tab_with_header_multi.csv
 - spec/fixtures/tab_with_two_lines.csv
 - spec/fixtures/test_extended_model.rb
 - spec/fixtures/test_model.rb