postgres-copy 1.1.2 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +33 -32
  3. data/README.md +14 -3
  4. data/lib/postgres-copy/acts_as_copy_target.rb +26 -0
  5. data/postgres-copy.gemspec +2 -2
  6. data/spec/copy_to_spec.rb +35 -1
  7. data/spec/fixtures/comma_with_header_and_scope.csv +4 -0
  8. data/spec/fixtures/comma_with_header_multi.csv +5 -0
  9. data/spec/fixtures/tab_with_header_multi.csv +5 -0
  10. metadata +15 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 3cc595142d45d6053c54bac79e0855c4d8136856
4
- data.tar.gz: 548bbff5d3505a7c9721d0d69f20313072770dd3
3
+ metadata.gz: 1bf6eb9e338ede86cf702bcd1841142a54abf542
4
+ data.tar.gz: e187a19feb3b625151b21ac71d5a4325874f3fc3
5
5
  SHA512:
6
- metadata.gz: e15905d7bcd481f38153decfc9350d0220642c62b35ea7d285293b3ccf2f50fd8cf4e87d1e4bc8c49bcb120e15b6a422b78f05313e3587ca94e0b578561fd28a
7
- data.tar.gz: 1da873e2f16c582afb38eef9d4dba009067446df3c6835228ec31c1cf2253a8690dd3e716f162a05032b6f0d0becf0d8defcca575cf29c101755c9f651c88935
6
+ metadata.gz: f633038cbe888b0db2bb72c3c6611716ad7bf57cae50b94a784eeee402c1c2d0e2de6d3f142da7d1093afa00ab4825791b4a36c2c6fd27404eb22f4cb0846b25
7
+ data.tar.gz: 5c050adfe50105e1f2016665d5efd8b7190edfc6cdd878745edb37c2eeb2eacd098ab851141bda53337cb33ebbaf8eda258b358f6767141bf409d7a5a1672169
data/Gemfile.lock CHANGED
@@ -1,70 +1,71 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- postgres-copy (1.1.2)
5
- activerecord (>= 4.0)
4
+ postgres-copy (1.2.0)
5
+ activerecord (>= 4.0, < 5.1)
6
6
  pg (>= 0.17)
7
7
  responders
8
8
 
9
9
  GEM
10
10
  remote: https://rubygems.org/
11
11
  specs:
12
- actionpack (5.0.1)
13
- actionview (= 5.0.1)
14
- activesupport (= 5.0.1)
12
+ actionpack (5.0.3)
13
+ actionview (= 5.0.3)
14
+ activesupport (= 5.0.3)
15
15
  rack (~> 2.0)
16
16
  rack-test (~> 0.6.3)
17
17
  rails-dom-testing (~> 2.0)
18
18
  rails-html-sanitizer (~> 1.0, >= 1.0.2)
19
- actionview (5.0.1)
20
- activesupport (= 5.0.1)
19
+ actionview (5.0.3)
20
+ activesupport (= 5.0.3)
21
21
  builder (~> 3.1)
22
22
  erubis (~> 2.7.0)
23
23
  rails-dom-testing (~> 2.0)
24
- rails-html-sanitizer (~> 1.0, >= 1.0.2)
25
- activemodel (5.0.1)
26
- activesupport (= 5.0.1)
27
- activerecord (5.0.1)
28
- activemodel (= 5.0.1)
29
- activesupport (= 5.0.1)
24
+ rails-html-sanitizer (~> 1.0, >= 1.0.3)
25
+ activemodel (5.0.3)
26
+ activesupport (= 5.0.3)
27
+ activerecord (5.0.3)
28
+ activemodel (= 5.0.3)
29
+ activesupport (= 5.0.3)
30
30
  arel (~> 7.0)
31
- activesupport (5.0.1)
31
+ activesupport (5.0.3)
32
32
  concurrent-ruby (~> 1.0, >= 1.0.2)
33
33
  i18n (~> 0.7)
34
34
  minitest (~> 5.1)
35
35
  tzinfo (~> 1.1)
36
36
  arel (7.1.4)
37
37
  builder (3.2.3)
38
- concurrent-ruby (1.0.4)
38
+ concurrent-ruby (1.0.5)
39
39
  diff-lcs (1.3)
40
40
  erubis (2.7.0)
41
- i18n (0.8.1)
41
+ i18n (0.8.4)
42
42
  loofah (2.0.3)
43
43
  nokogiri (>= 1.5.9)
44
44
  method_source (0.8.2)
45
- mini_portile2 (2.1.0)
46
- minitest (5.10.1)
47
- nokogiri (1.7.0.1)
48
- mini_portile2 (~> 2.1.0)
49
- pg (0.19.0)
50
- rack (2.0.1)
45
+ mini_portile2 (2.2.0)
46
+ minitest (5.10.2)
47
+ nokogiri (1.8.0)
48
+ mini_portile2 (~> 2.2.0)
49
+ pg (0.21.0)
50
+ rack (2.0.3)
51
51
  rack-test (0.6.3)
52
52
  rack (>= 1.0)
53
- rails-dom-testing (2.0.2)
54
- activesupport (>= 4.2.0, < 6.0)
55
- nokogiri (~> 1.6)
53
+ rails-dom-testing (2.0.3)
54
+ activesupport (>= 4.2.0)
55
+ nokogiri (>= 1.6)
56
56
  rails-html-sanitizer (1.0.3)
57
57
  loofah (~> 2.0)
58
- railties (5.0.1)
59
- actionpack (= 5.0.1)
60
- activesupport (= 5.0.1)
58
+ railties (5.0.3)
59
+ actionpack (= 5.0.3)
60
+ activesupport (= 5.0.3)
61
61
  method_source
62
62
  rake (>= 0.8.7)
63
63
  thor (>= 0.18.1, < 2.0)
64
64
  rake (11.2.2)
65
65
  rdoc (5.0.0)
66
- responders (2.3.0)
67
- railties (>= 4.2.0, < 5.1)
66
+ responders (2.4.0)
67
+ actionpack (>= 4.2.0, < 5.3)
68
+ railties (>= 4.2.0, < 5.3)
68
69
  rspec (2.99.0)
69
70
  rspec-core (~> 2.99.0)
70
71
  rspec-expectations (~> 2.99.0)
@@ -75,7 +76,7 @@ GEM
75
76
  rspec-mocks (2.99.4)
76
77
  thor (0.19.4)
77
78
  thread_safe (0.3.6)
78
- tzinfo (1.2.2)
79
+ tzinfo (1.2.3)
79
80
  thread_safe (~> 0.1)
80
81
 
81
82
  PLATFORMS
@@ -89,4 +90,4 @@ DEPENDENCIES
89
90
  rspec (~> 2.12)
90
91
 
91
92
  BUNDLED WITH
92
- 1.14.3
93
+ 1.14.6
data/README.md CHANGED
@@ -5,7 +5,7 @@ If you need to tranfer data between a PostgreSQL database and CSV files, the Pos
5
5
  will give you a greater performance than using the ruby CSV+INSERT commands.
6
6
  I have not found time to make accurate benchmarks, but in the use scenario where I have developed the gem
7
7
  I have had a four-fold performance gain.
8
- This gem was written having the Rails framework in mind, I think it could work only with active-record,
8
+ This gem was written having the Rails framework in mind, I think it could work only with active-record,
9
9
  but I will assume in this README that you are using Rails.
10
10
 
11
11
  ## Install
@@ -34,8 +34,9 @@ end
34
34
 
35
35
  This will add the aditiontal class methods to your model:
36
36
 
37
- * copy_to
37
+ * copy_to
38
38
  * copy_to_string
39
+ * copy_to_enumerator
39
40
  * copy_from
40
41
 
41
42
  ### Using copy_to and copy_to_string
@@ -66,13 +67,23 @@ File.open('/tmp/users.csv', 'w') do |f|
66
67
  end
67
68
  ```
68
69
 
70
+ Instead of yielding each line, you could return an enumerator with all users:
71
+ ```ruby
72
+ enumerator = User.copy_to_enumerator
73
+ ```
74
+
75
+ And for better performance when rendering the result of the enumerator, you can return an enumerator with blocks of 100 lines joined:
76
+ ```ruby
77
+ enumerator = User.copy_to_enumerator(:buffer_lines => 100)
78
+ ```
79
+
69
80
  Or, if you have enough memory, you can read all table contents to a string using .copy_to_string
70
81
 
71
82
  ```ruby
72
83
  puts User.copy_to_string
73
84
  ```
74
85
 
75
- Another insteresting feature of copy_to is that it uses the scoped relation, it means that you can use ARel
86
+ Another insteresting feature of copy_to is that it uses the scoped relation, it means that you can use ARel
76
87
  operations to generate different CSV files according to your needs.
77
88
  Assuming we want to generate a file only with the names of users 1, 2 and 3:
78
89
 
data/lib/postgres-copy/acts_as_copy_target.rb CHANGED
@@ -30,6 +30,32 @@ module PostgresCopy
30
30
  return self
31
31
  end
32
32
 
33
+ # Create an enumerator with each line from the CSV.
34
+ # Note that using this directly in a controller response
35
+ # will perform very poorly as each line will get put
36
+ # into its own chunk. Joining every (eg) 100 rows together
37
+ # is much, much faster.
38
+ def copy_to_enumerator(options={})
39
+ buffer_lines = options.delete(:buffer_lines)
40
+ # Somehow, self loses its scope once inside the Enumerator
41
+ scope = self.current_scope || self
42
+ result = Enumerator.new do |y|
43
+ scope.copy_to(nil, options) do |line|
44
+ y << line
45
+ end
46
+ end
47
+
48
+ if buffer_lines.to_i > 0
49
+ Enumerator.new do |y|
50
+ result.each_slice(buffer_lines.to_i) do |slice|
51
+ y << slice.join
52
+ end
53
+ end
54
+ else
55
+ result
56
+ end
57
+ end
58
+
33
59
  # Copy all data to a single string
34
60
  def copy_to_string options = {}
35
61
  data = ''
data/postgres-copy.gemspec CHANGED
@@ -5,7 +5,7 @@ $:.unshift lib unless $:.include?(lib)
5
5
 
6
6
  Gem::Specification.new do |s|
7
7
  s.name = "postgres-copy"
8
- s.version = "1.1.2"
8
+ s.version = "1.2.0"
9
9
  s.platform = Gem::Platform::RUBY
10
10
  s.required_ruby_version = ">= 1.9.3"
11
11
  s.authors = ["Diogo Biazus"]
@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
21
21
  s.summary = "Put COPY command functionality in ActiveRecord's model class"
22
22
 
23
23
  s.add_dependency "pg", ">= 0.17"
24
- s.add_dependency "activerecord", '>= 4.0'
24
+ s.add_dependency "activerecord", '>= 4.0', '< 5.1'
25
25
  s.add_dependency "responders"
26
26
  s.add_development_dependency "bundler"
27
27
  s.add_development_dependency "rdoc"
data/spec/copy_to_spec.rb CHANGED
@@ -1,7 +1,7 @@
1
1
  require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
2
2
 
3
3
  describe "COPY TO" do
4
- before(:all) do
4
+ before(:each) do
5
5
  ActiveRecord::Base.connection.execute %{
6
6
  TRUNCATE TABLE test_models;
7
7
  SELECT setval('test_models_id_seq', 1, false);
@@ -21,6 +21,40 @@ describe "COPY TO" do
21
21
  end
22
22
  end
23
23
 
24
+ describe ".copy_to_enumerator" do
25
+ before(:each) do
26
+ TestModel.create :data => 'test data 2'
27
+ TestModel.create :data => 'test data 3'
28
+ TestModel.create :data => 'test data 4'
29
+ end
30
+
31
+ context "with no options" do
32
+ subject{ TestModel.copy_to_enumerator.to_a }
33
+ it{ should == File.open('spec/fixtures/comma_with_header_multi.csv', 'r').read.lines }
34
+ end
35
+
36
+ context "with tab as delimiter" do
37
+ subject{ TestModel.copy_to_enumerator(:delimiter => "\t").to_a }
38
+ it{ should == File.open('spec/fixtures/tab_with_header_multi.csv', 'r').read.lines }
39
+ end
40
+
41
+ context "with many records" do
42
+ context "enumerating in batches" do
43
+ subject{ TestModel.copy_to_enumerator(:buffer_lines => 2).to_a }
44
+ it do
45
+ expected = []
46
+ File.open('spec/fixtures/comma_with_header_multi.csv', 'r').read.lines.each_slice(2){|s| expected << s.join }
47
+ should == expected
48
+ end
49
+ end
50
+
51
+ context "excluding some records via a scope" do
52
+ subject{ TestModel.where("data not like '%3'").copy_to_enumerator.to_a }
53
+ it{ should == File.open('spec/fixtures/comma_with_header_and_scope.csv', 'r').read.lines }
54
+ end
55
+ end
56
+ end
57
+
24
58
  describe ".copy_to" do
25
59
  it "should copy and pass data to block if block is given and no path is passed" do
26
60
  File.open('spec/fixtures/comma_with_header.csv', 'r') do |f|
data/spec/fixtures/comma_with_header_and_scope.csv ADDED
@@ -0,0 +1,4 @@
1
+ id,data
2
+ 1,test data 1
3
+ 2,test data 2
4
+ 4,test data 4
data/spec/fixtures/comma_with_header_multi.csv ADDED
@@ -0,0 +1,5 @@
1
+ id,data
2
+ 1,test data 1
3
+ 2,test data 2
4
+ 3,test data 3
5
+ 4,test data 4
data/spec/fixtures/tab_with_header_multi.csv ADDED
@@ -0,0 +1,5 @@
1
+ id data
2
+ 1 test data 1
3
+ 2 test data 2
4
+ 3 test data 3
5
+ 4 test data 4
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: postgres-copy
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.2
4
+ version: 1.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Diogo Biazus
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-03-09 00:00:00.000000000 Z
11
+ date: 2017-06-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: pg
@@ -31,6 +31,9 @@ dependencies:
31
31
  - - ">="
32
32
  - !ruby/object:Gem::Version
33
33
  version: '4.0'
34
+ - - "<"
35
+ - !ruby/object:Gem::Version
36
+ version: '5.1'
34
37
  type: :runtime
35
38
  prerelease: false
36
39
  version_requirements: !ruby/object:Gem::Requirement
@@ -38,6 +41,9 @@ dependencies:
38
41
  - - ">="
39
42
  - !ruby/object:Gem::Version
40
43
  version: '4.0'
44
+ - - "<"
45
+ - !ruby/object:Gem::Version
46
+ version: '5.1'
41
47
  - !ruby/object:Gem::Dependency
42
48
  name: responders
43
49
  requirement: !ruby/object:Gem::Requirement
@@ -136,7 +142,9 @@ files:
136
142
  - spec/fixtures/2_col_binary_data.dat
137
143
  - spec/fixtures/comma_inside_field.csv
138
144
  - spec/fixtures/comma_with_header.csv
145
+ - spec/fixtures/comma_with_header_and_scope.csv
139
146
  - spec/fixtures/comma_with_header_empty_values_at_the_end.csv
147
+ - spec/fixtures/comma_with_header_multi.csv
140
148
  - spec/fixtures/comma_without_header.csv
141
149
  - spec/fixtures/extra_field.rb
142
150
  - spec/fixtures/reserved_word_model.rb
@@ -150,6 +158,7 @@ files:
150
158
  - spec/fixtures/tab_with_error.csv
151
159
  - spec/fixtures/tab_with_extra_line.csv
152
160
  - spec/fixtures/tab_with_header.csv
161
+ - spec/fixtures/tab_with_header_multi.csv
153
162
  - spec/fixtures/tab_with_two_lines.csv
154
163
  - spec/fixtures/test_extended_model.rb
155
164
  - spec/fixtures/test_model.rb
@@ -174,7 +183,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
174
183
  version: '0'
175
184
  requirements: []
176
185
  rubyforge_project:
177
- rubygems_version: 2.5.1
186
+ rubygems_version: 2.6.8
178
187
  signing_key:
179
188
  specification_version: 4
180
189
  summary: Put COPY command functionality in ActiveRecord's model class
@@ -186,7 +195,9 @@ test_files:
186
195
  - spec/fixtures/2_col_binary_data.dat
187
196
  - spec/fixtures/comma_inside_field.csv
188
197
  - spec/fixtures/comma_with_header.csv
198
+ - spec/fixtures/comma_with_header_and_scope.csv
189
199
  - spec/fixtures/comma_with_header_empty_values_at_the_end.csv
200
+ - spec/fixtures/comma_with_header_multi.csv
190
201
  - spec/fixtures/comma_without_header.csv
191
202
  - spec/fixtures/extra_field.rb
192
203
  - spec/fixtures/reserved_word_model.rb
@@ -200,6 +211,7 @@ test_files:
200
211
  - spec/fixtures/tab_with_error.csv
201
212
  - spec/fixtures/tab_with_extra_line.csv
202
213
  - spec/fixtures/tab_with_header.csv
214
+ - spec/fixtures/tab_with_header_multi.csv
203
215
  - spec/fixtures/tab_with_two_lines.csv
204
216
  - spec/fixtures/test_extended_model.rb
205
217
  - spec/fixtures/test_model.rb