postgres-copy 1.4.0 → 1.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +6 -0
- data/Gemfile.lock +42 -39
- data/README.md +8 -2
- data/lib/postgres-copy/acts_as_copy_target.rb +25 -8
- data/postgres-copy.gemspec +1 -1
- data/spec/copy_from_spec.rb +14 -4
- data/spec/fixtures/comma_with_carriage_returns.csv +3 -0
- data/spec/fixtures/comma_with_empty_string.csv +2 -0
- metadata +7 -4
- data/VERSION +0 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6477dfff9506950fc1f3a728138a48ce5f669a0c
|
4
|
+
data.tar.gz: 6a048abf7e41bbec754e4603b6f0fce48c6c3aa7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 29f73c848994f32475f6652fe07788180efa8ab3faede9e98bc4d13d1f5afd9a95d165c2c8eb9c077fa9c8780aab876c167b1aa0fef49875b4a27c554785ee61
|
7
|
+
data.tar.gz: dba63f2014d5caad8e5f3623e85abf684f97ed8fd6ccee207f10033a4b090d2fa469d401b61173e1d0967988bc2e370434dee13b33525a863c35f864c0bdb987
|
data/.travis.yml
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
postgres-copy (1.4.
|
4
|
+
postgres-copy (1.4.1)
|
5
5
|
activerecord (>= 5.1)
|
6
6
|
pg (>= 0.17)
|
7
7
|
responders
|
@@ -9,63 +9,66 @@ PATH
|
|
9
9
|
GEM
|
10
10
|
remote: https://rubygems.org/
|
11
11
|
specs:
|
12
|
-
actionpack (5.
|
13
|
-
actionview (= 5.
|
14
|
-
activesupport (= 5.
|
12
|
+
actionpack (5.2.2)
|
13
|
+
actionview (= 5.2.2)
|
14
|
+
activesupport (= 5.2.2)
|
15
15
|
rack (~> 2.0)
|
16
16
|
rack-test (>= 0.6.3)
|
17
17
|
rails-dom-testing (~> 2.0)
|
18
18
|
rails-html-sanitizer (~> 1.0, >= 1.0.2)
|
19
|
-
actionview (5.
|
20
|
-
activesupport (= 5.
|
19
|
+
actionview (5.2.2)
|
20
|
+
activesupport (= 5.2.2)
|
21
21
|
builder (~> 3.1)
|
22
22
|
erubi (~> 1.4)
|
23
23
|
rails-dom-testing (~> 2.0)
|
24
24
|
rails-html-sanitizer (~> 1.0, >= 1.0.3)
|
25
|
-
activemodel (5.
|
26
|
-
activesupport (= 5.
|
27
|
-
activerecord (5.
|
28
|
-
activemodel (= 5.
|
29
|
-
activesupport (= 5.
|
30
|
-
arel (
|
31
|
-
activesupport (5.
|
25
|
+
activemodel (5.2.2)
|
26
|
+
activesupport (= 5.2.2)
|
27
|
+
activerecord (5.2.2)
|
28
|
+
activemodel (= 5.2.2)
|
29
|
+
activesupport (= 5.2.2)
|
30
|
+
arel (>= 9.0)
|
31
|
+
activesupport (5.2.2)
|
32
32
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
33
|
-
i18n (
|
33
|
+
i18n (>= 0.7, < 2)
|
34
34
|
minitest (~> 5.1)
|
35
35
|
tzinfo (~> 1.1)
|
36
|
-
arel (
|
36
|
+
arel (9.0.0)
|
37
37
|
builder (3.2.3)
|
38
|
-
concurrent-ruby (1.
|
38
|
+
concurrent-ruby (1.1.4)
|
39
|
+
crass (1.0.4)
|
39
40
|
diff-lcs (1.3)
|
40
|
-
erubi (1.
|
41
|
-
i18n (
|
42
|
-
|
41
|
+
erubi (1.8.0)
|
42
|
+
i18n (1.5.3)
|
43
|
+
concurrent-ruby (~> 1.0)
|
44
|
+
loofah (2.2.3)
|
45
|
+
crass (~> 1.0.2)
|
43
46
|
nokogiri (>= 1.5.9)
|
44
|
-
method_source (0.
|
45
|
-
mini_portile2 (2.
|
46
|
-
minitest (5.
|
47
|
-
nokogiri (1.
|
48
|
-
mini_portile2 (~> 2.
|
49
|
-
pg (
|
50
|
-
rack (2.0.
|
51
|
-
rack-test (
|
47
|
+
method_source (0.9.2)
|
48
|
+
mini_portile2 (2.4.0)
|
49
|
+
minitest (5.11.3)
|
50
|
+
nokogiri (1.10.1)
|
51
|
+
mini_portile2 (~> 2.4.0)
|
52
|
+
pg (1.1.4)
|
53
|
+
rack (2.0.6)
|
54
|
+
rack-test (1.1.0)
|
52
55
|
rack (>= 1.0, < 3)
|
53
56
|
rails-dom-testing (2.0.3)
|
54
57
|
activesupport (>= 4.2.0)
|
55
58
|
nokogiri (>= 1.6)
|
56
|
-
rails-html-sanitizer (1.0.
|
57
|
-
loofah (~> 2.
|
58
|
-
railties (5.
|
59
|
-
actionpack (= 5.
|
60
|
-
activesupport (= 5.
|
59
|
+
rails-html-sanitizer (1.0.4)
|
60
|
+
loofah (~> 2.2, >= 2.2.2)
|
61
|
+
railties (5.2.2)
|
62
|
+
actionpack (= 5.2.2)
|
63
|
+
activesupport (= 5.2.2)
|
61
64
|
method_source
|
62
65
|
rake (>= 0.8.7)
|
63
|
-
thor (>= 0.
|
66
|
+
thor (>= 0.19.0, < 2.0)
|
64
67
|
rake (11.2.2)
|
65
68
|
rdoc (5.0.0)
|
66
|
-
responders (2.4.
|
67
|
-
actionpack (>= 4.2.0, <
|
68
|
-
railties (>= 4.2.0, <
|
69
|
+
responders (2.4.1)
|
70
|
+
actionpack (>= 4.2.0, < 6.0)
|
71
|
+
railties (>= 4.2.0, < 6.0)
|
69
72
|
rspec (2.99.0)
|
70
73
|
rspec-core (~> 2.99.0)
|
71
74
|
rspec-expectations (~> 2.99.0)
|
@@ -74,9 +77,9 @@ GEM
|
|
74
77
|
rspec-expectations (2.99.2)
|
75
78
|
diff-lcs (>= 1.1.3, < 2.0)
|
76
79
|
rspec-mocks (2.99.4)
|
77
|
-
thor (0.20.
|
80
|
+
thor (0.20.3)
|
78
81
|
thread_safe (0.3.6)
|
79
|
-
tzinfo (1.2.
|
82
|
+
tzinfo (1.2.5)
|
80
83
|
thread_safe (~> 0.1)
|
81
84
|
|
82
85
|
PLATFORMS
|
@@ -90,4 +93,4 @@ DEPENDENCIES
|
|
90
93
|
rspec (~> 2.12)
|
91
94
|
|
92
95
|
BUNDLED WITH
|
93
|
-
1.
|
96
|
+
1.16.3
|
data/README.md
CHANGED
@@ -55,8 +55,8 @@ This will execute in the database the command:
|
|
55
55
|
COPY (SELECT "users".* FROM "users" ) TO '/tmp/users.csv' WITH DELIMITER ',' CSV HEADER
|
56
56
|
```
|
57
57
|
|
58
|
-
Remark that the file will be created in the database server disk.
|
59
|
-
But what if you want to write the lines in a file on the server that is running Rails, instead of the database?
|
58
|
+
Remark that the file will be created in the database server disk.
|
59
|
+
But what if you want to write the lines in a file on the server that is running Rails, instead of the database?
|
60
60
|
In this case you can pass a block and retrieve the generated lines and then write them to a file:
|
61
61
|
|
62
62
|
```ruby
|
@@ -166,6 +166,12 @@ To specify NULL value you can pass the null option parameter.
|
|
166
166
|
User.copy_from "/tmp/users.csv", :null => 'null'
|
167
167
|
```
|
168
168
|
|
169
|
+
Match the specified columns' values against the null string, even if it has been quoted, and if a match is found set the value to NULL (Postgres 9.4+ only).
|
170
|
+
|
171
|
+
```ruby
|
172
|
+
User.copy_from "/tmp/users.csv", :null => '', :force_null => [:name, :city]
|
173
|
+
```
|
174
|
+
|
169
175
|
|
170
176
|
To copy a binary formatted data file or IO object you can specify the format as binary
|
171
177
|
|
@@ -77,8 +77,9 @@ module PostgresCopy
|
|
77
77
|
"BINARY"
|
78
78
|
else
|
79
79
|
quote = options[:quote] == "'" ? "''" : options[:quote]
|
80
|
-
null = options.key?(:null) ? "NULL '#{options[:null]}'" :
|
81
|
-
"
|
80
|
+
null = options.key?(:null) ? "NULL '#{options[:null]}'" : nil
|
81
|
+
force_null = options.key?(:force_null) ? "FORCE_NULL(#{options[:force_null].join(',')})" : nil
|
82
|
+
"WITH (" + ["DELIMITER '#{options[:delimiter]}'", "QUOTE '#{quote}'", null, force_null, "FORMAT CSV"].compact.join(', ') + ")"
|
82
83
|
end
|
83
84
|
io = path_or_io.instance_of?(String) ? File.open(path_or_io, 'r') : path_or_io
|
84
85
|
|
@@ -110,15 +111,31 @@ module PostgresCopy
|
|
110
111
|
rescue EOFError
|
111
112
|
end
|
112
113
|
else
|
114
|
+
line_buffer = ''
|
115
|
+
|
113
116
|
while line = io.gets do
|
114
117
|
next if line.strip.size == 0
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
118
|
+
|
119
|
+
line_buffer += line
|
120
|
+
|
121
|
+
# If line is incomplete, get the next line until it terminates
|
122
|
+
if line_buffer =~ /\n$/ || line_buffer =~ /\Z/
|
123
|
+
if block_given?
|
124
|
+
begin
|
125
|
+
row = CSV.parse_line(line_buffer.strip, {:col_sep => options[:delimiter]})
|
126
|
+
yield(row)
|
127
|
+
next if row.all?{|f| f.nil? }
|
128
|
+
line_buffer = CSV.generate_line(row, {:col_sep => options[:delimiter]})
|
129
|
+
rescue CSV::MalformedCSVError
|
130
|
+
next
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
connection.raw_connection.put_copy_data(line_buffer)
|
135
|
+
|
136
|
+
# Clear the buffer
|
137
|
+
line_buffer = ''
|
120
138
|
end
|
121
|
-
connection.raw_connection.put_copy_data line
|
122
139
|
end
|
123
140
|
end
|
124
141
|
end
|
data/postgres-copy.gemspec
CHANGED
data/spec/copy_from_spec.rb
CHANGED
@@ -100,20 +100,20 @@ describe "COPY FROM" do
|
|
100
100
|
ReservedWordModel.copy_from File.expand_path('spec/fixtures/reserved_words.csv'), :delimiter => "\t"
|
101
101
|
ReservedWordModel.order(:id).map{|r| r.attributes}.should == [{"group"=>"group name", "id"=>1, "select"=>"test select"}]
|
102
102
|
end
|
103
|
-
|
103
|
+
|
104
104
|
it "should import even last columns have empty values" do
|
105
105
|
TestExtendedModel.copy_from File.expand_path('spec/fixtures/comma_with_header_empty_values_at_the_end.csv')
|
106
|
-
TestExtendedModel.order(:id).map{|r| r.attributes}.should ==
|
106
|
+
TestExtendedModel.order(:id).map{|r| r.attributes}.should ==
|
107
107
|
[{"id"=>1, "data"=>"test data 1", "more_data"=>nil, "other_data"=>nil, "final_data"=>nil},
|
108
108
|
{"id"=>2, "data"=>"test data 2", "more_data"=>"9", "other_data"=>nil, "final_data"=>nil},
|
109
109
|
{"id"=>3, "data"=>"test data 2", "more_data"=>"9", "other_data"=>nil, "final_data"=>"0"}]
|
110
110
|
end
|
111
|
-
|
111
|
+
|
112
112
|
it "should import even last columns have empty values with block" do
|
113
113
|
TestExtendedModel.copy_from File.expand_path('spec/fixtures/comma_with_header_empty_values_at_the_end.csv') do |row|
|
114
114
|
row[4]="666"
|
115
115
|
end
|
116
|
-
TestExtendedModel.order(:id).map{|r| r.attributes}.should ==
|
116
|
+
TestExtendedModel.order(:id).map{|r| r.attributes}.should ==
|
117
117
|
[{"id"=>1, "data"=>"test data 1", "more_data"=>nil, "other_data"=>nil, "final_data"=>"666"},
|
118
118
|
{"id"=>2, "data"=>"test data 2", "more_data"=>"9", "other_data"=>nil, "final_data"=>"666"},
|
119
119
|
{"id"=>3, "data"=>"test data 2", "more_data"=>"9", "other_data"=>nil, "final_data"=>"666"}]
|
@@ -150,4 +150,14 @@ describe "COPY FROM" do
|
|
150
150
|
TestModel.copy_from File.open(File.expand_path('spec/fixtures/special_null_with_header.csv'), 'r'), :null => 'NULL'
|
151
151
|
TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => nil}]
|
152
152
|
end
|
153
|
+
|
154
|
+
it "should import with a carriage return in the value" do
|
155
|
+
TestModel.copy_from File.expand_path('spec/fixtures/comma_with_carriage_returns.csv')
|
156
|
+
TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => "test\ndata 1"}]
|
157
|
+
end
|
158
|
+
|
159
|
+
it "should import custom force null expressions from path" do
|
160
|
+
TestModel.copy_from File.expand_path('spec/fixtures/comma_with_empty_string.csv'), :null => '', :force_null => [:data]
|
161
|
+
TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => nil}]
|
162
|
+
end
|
153
163
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: postgres-copy
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.4.
|
4
|
+
version: 1.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Diogo Biazus
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-01-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: pg
|
@@ -124,7 +124,6 @@ files:
|
|
124
124
|
- LICENSE
|
125
125
|
- README.md
|
126
126
|
- Rakefile
|
127
|
-
- VERSION
|
128
127
|
- lib/postgres-copy.rb
|
129
128
|
- lib/postgres-copy/acts_as_copy_target.rb
|
130
129
|
- lib/postgres-copy/csv_responder.rb
|
@@ -135,6 +134,8 @@ files:
|
|
135
134
|
- spec/copy_to_spec.rb
|
136
135
|
- spec/fixtures/2_col_binary_data.dat
|
137
136
|
- spec/fixtures/comma_inside_field.csv
|
137
|
+
- spec/fixtures/comma_with_carriage_returns.csv
|
138
|
+
- spec/fixtures/comma_with_empty_string.csv
|
138
139
|
- spec/fixtures/comma_with_header.csv
|
139
140
|
- spec/fixtures/comma_with_header_and_scope.csv
|
140
141
|
- spec/fixtures/comma_with_header_empty_values_at_the_end.csv
|
@@ -177,7 +178,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
177
178
|
version: '0'
|
178
179
|
requirements: []
|
179
180
|
rubyforge_project:
|
180
|
-
rubygems_version: 2.6.
|
181
|
+
rubygems_version: 2.6.13
|
181
182
|
signing_key:
|
182
183
|
specification_version: 4
|
183
184
|
summary: Put COPY command functionality in ActiveRecord's model class
|
@@ -188,6 +189,8 @@ test_files:
|
|
188
189
|
- spec/copy_to_spec.rb
|
189
190
|
- spec/fixtures/2_col_binary_data.dat
|
190
191
|
- spec/fixtures/comma_inside_field.csv
|
192
|
+
- spec/fixtures/comma_with_carriage_returns.csv
|
193
|
+
- spec/fixtures/comma_with_empty_string.csv
|
191
194
|
- spec/fixtures/comma_with_header.csv
|
192
195
|
- spec/fixtures/comma_with_header_and_scope.csv
|
193
196
|
- spec/fixtures/comma_with_header_empty_values_at_the_end.csv
|
data/VERSION
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
0.5.6
|