postgres-copy 1.4.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +6 -0
- data/Gemfile.lock +42 -39
- data/README.md +8 -2
- data/lib/postgres-copy/acts_as_copy_target.rb +25 -8
- data/postgres-copy.gemspec +1 -1
- data/spec/copy_from_spec.rb +14 -4
- data/spec/fixtures/comma_with_carriage_returns.csv +3 -0
- data/spec/fixtures/comma_with_empty_string.csv +2 -0
- metadata +7 -4
- data/VERSION +0 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6477dfff9506950fc1f3a728138a48ce5f669a0c
|
4
|
+
data.tar.gz: 6a048abf7e41bbec754e4603b6f0fce48c6c3aa7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 29f73c848994f32475f6652fe07788180efa8ab3faede9e98bc4d13d1f5afd9a95d165c2c8eb9c077fa9c8780aab876c167b1aa0fef49875b4a27c554785ee61
|
7
|
+
data.tar.gz: dba63f2014d5caad8e5f3623e85abf684f97ed8fd6ccee207f10033a4b090d2fa469d401b61173e1d0967988bc2e370434dee13b33525a863c35f864c0bdb987
|
data/.travis.yml
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
postgres-copy (1.4.
|
4
|
+
postgres-copy (1.4.1)
|
5
5
|
activerecord (>= 5.1)
|
6
6
|
pg (>= 0.17)
|
7
7
|
responders
|
@@ -9,63 +9,66 @@ PATH
|
|
9
9
|
GEM
|
10
10
|
remote: https://rubygems.org/
|
11
11
|
specs:
|
12
|
-
actionpack (5.
|
13
|
-
actionview (= 5.
|
14
|
-
activesupport (= 5.
|
12
|
+
actionpack (5.2.2)
|
13
|
+
actionview (= 5.2.2)
|
14
|
+
activesupport (= 5.2.2)
|
15
15
|
rack (~> 2.0)
|
16
16
|
rack-test (>= 0.6.3)
|
17
17
|
rails-dom-testing (~> 2.0)
|
18
18
|
rails-html-sanitizer (~> 1.0, >= 1.0.2)
|
19
|
-
actionview (5.
|
20
|
-
activesupport (= 5.
|
19
|
+
actionview (5.2.2)
|
20
|
+
activesupport (= 5.2.2)
|
21
21
|
builder (~> 3.1)
|
22
22
|
erubi (~> 1.4)
|
23
23
|
rails-dom-testing (~> 2.0)
|
24
24
|
rails-html-sanitizer (~> 1.0, >= 1.0.3)
|
25
|
-
activemodel (5.
|
26
|
-
activesupport (= 5.
|
27
|
-
activerecord (5.
|
28
|
-
activemodel (= 5.
|
29
|
-
activesupport (= 5.
|
30
|
-
arel (
|
31
|
-
activesupport (5.
|
25
|
+
activemodel (5.2.2)
|
26
|
+
activesupport (= 5.2.2)
|
27
|
+
activerecord (5.2.2)
|
28
|
+
activemodel (= 5.2.2)
|
29
|
+
activesupport (= 5.2.2)
|
30
|
+
arel (>= 9.0)
|
31
|
+
activesupport (5.2.2)
|
32
32
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
33
|
-
i18n (
|
33
|
+
i18n (>= 0.7, < 2)
|
34
34
|
minitest (~> 5.1)
|
35
35
|
tzinfo (~> 1.1)
|
36
|
-
arel (
|
36
|
+
arel (9.0.0)
|
37
37
|
builder (3.2.3)
|
38
|
-
concurrent-ruby (1.
|
38
|
+
concurrent-ruby (1.1.4)
|
39
|
+
crass (1.0.4)
|
39
40
|
diff-lcs (1.3)
|
40
|
-
erubi (1.
|
41
|
-
i18n (
|
42
|
-
|
41
|
+
erubi (1.8.0)
|
42
|
+
i18n (1.5.3)
|
43
|
+
concurrent-ruby (~> 1.0)
|
44
|
+
loofah (2.2.3)
|
45
|
+
crass (~> 1.0.2)
|
43
46
|
nokogiri (>= 1.5.9)
|
44
|
-
method_source (0.
|
45
|
-
mini_portile2 (2.
|
46
|
-
minitest (5.
|
47
|
-
nokogiri (1.
|
48
|
-
mini_portile2 (~> 2.
|
49
|
-
pg (
|
50
|
-
rack (2.0.
|
51
|
-
rack-test (
|
47
|
+
method_source (0.9.2)
|
48
|
+
mini_portile2 (2.4.0)
|
49
|
+
minitest (5.11.3)
|
50
|
+
nokogiri (1.10.1)
|
51
|
+
mini_portile2 (~> 2.4.0)
|
52
|
+
pg (1.1.4)
|
53
|
+
rack (2.0.6)
|
54
|
+
rack-test (1.1.0)
|
52
55
|
rack (>= 1.0, < 3)
|
53
56
|
rails-dom-testing (2.0.3)
|
54
57
|
activesupport (>= 4.2.0)
|
55
58
|
nokogiri (>= 1.6)
|
56
|
-
rails-html-sanitizer (1.0.
|
57
|
-
loofah (~> 2.
|
58
|
-
railties (5.
|
59
|
-
actionpack (= 5.
|
60
|
-
activesupport (= 5.
|
59
|
+
rails-html-sanitizer (1.0.4)
|
60
|
+
loofah (~> 2.2, >= 2.2.2)
|
61
|
+
railties (5.2.2)
|
62
|
+
actionpack (= 5.2.2)
|
63
|
+
activesupport (= 5.2.2)
|
61
64
|
method_source
|
62
65
|
rake (>= 0.8.7)
|
63
|
-
thor (>= 0.
|
66
|
+
thor (>= 0.19.0, < 2.0)
|
64
67
|
rake (11.2.2)
|
65
68
|
rdoc (5.0.0)
|
66
|
-
responders (2.4.
|
67
|
-
actionpack (>= 4.2.0, <
|
68
|
-
railties (>= 4.2.0, <
|
69
|
+
responders (2.4.1)
|
70
|
+
actionpack (>= 4.2.0, < 6.0)
|
71
|
+
railties (>= 4.2.0, < 6.0)
|
69
72
|
rspec (2.99.0)
|
70
73
|
rspec-core (~> 2.99.0)
|
71
74
|
rspec-expectations (~> 2.99.0)
|
@@ -74,9 +77,9 @@ GEM
|
|
74
77
|
rspec-expectations (2.99.2)
|
75
78
|
diff-lcs (>= 1.1.3, < 2.0)
|
76
79
|
rspec-mocks (2.99.4)
|
77
|
-
thor (0.20.
|
80
|
+
thor (0.20.3)
|
78
81
|
thread_safe (0.3.6)
|
79
|
-
tzinfo (1.2.
|
82
|
+
tzinfo (1.2.5)
|
80
83
|
thread_safe (~> 0.1)
|
81
84
|
|
82
85
|
PLATFORMS
|
@@ -90,4 +93,4 @@ DEPENDENCIES
|
|
90
93
|
rspec (~> 2.12)
|
91
94
|
|
92
95
|
BUNDLED WITH
|
93
|
-
1.
|
96
|
+
1.16.3
|
data/README.md
CHANGED
@@ -55,8 +55,8 @@ This will execute in the database the command:
|
|
55
55
|
COPY (SELECT "users".* FROM "users" ) TO '/tmp/users.csv' WITH DELIMITER ',' CSV HEADER
|
56
56
|
```
|
57
57
|
|
58
|
-
Remark that the file will be created in the database server disk.
|
59
|
-
But what if you want to write the lines in a file on the server that is running Rails, instead of the database?
|
58
|
+
Remark that the file will be created in the database server disk.
|
59
|
+
But what if you want to write the lines in a file on the server that is running Rails, instead of the database?
|
60
60
|
In this case you can pass a block and retrieve the generated lines and then write them to a file:
|
61
61
|
|
62
62
|
```ruby
|
@@ -166,6 +166,12 @@ To specify NULL value you can pass the null option parameter.
|
|
166
166
|
User.copy_from "/tmp/users.csv", :null => 'null'
|
167
167
|
```
|
168
168
|
|
169
|
+
Match the specified columns' values against the null string, even if it has been quoted, and if a match is found set the value to NULL (Postgres 9.4+ only).
|
170
|
+
|
171
|
+
```ruby
|
172
|
+
User.copy_from "/tmp/users.csv", :null => '', :force_null => [:name, :city]
|
173
|
+
```
|
174
|
+
|
169
175
|
|
170
176
|
To copy a binary formatted data file or IO object you can specify the format as binary
|
171
177
|
|
@@ -77,8 +77,9 @@ module PostgresCopy
|
|
77
77
|
"BINARY"
|
78
78
|
else
|
79
79
|
quote = options[:quote] == "'" ? "''" : options[:quote]
|
80
|
-
null = options.key?(:null) ? "NULL '#{options[:null]}'" :
|
81
|
-
"
|
80
|
+
null = options.key?(:null) ? "NULL '#{options[:null]}'" : nil
|
81
|
+
force_null = options.key?(:force_null) ? "FORCE_NULL(#{options[:force_null].join(',')})" : nil
|
82
|
+
"WITH (" + ["DELIMITER '#{options[:delimiter]}'", "QUOTE '#{quote}'", null, force_null, "FORMAT CSV"].compact.join(', ') + ")"
|
82
83
|
end
|
83
84
|
io = path_or_io.instance_of?(String) ? File.open(path_or_io, 'r') : path_or_io
|
84
85
|
|
@@ -110,15 +111,31 @@ module PostgresCopy
|
|
110
111
|
rescue EOFError
|
111
112
|
end
|
112
113
|
else
|
114
|
+
line_buffer = ''
|
115
|
+
|
113
116
|
while line = io.gets do
|
114
117
|
next if line.strip.size == 0
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
118
|
+
|
119
|
+
line_buffer += line
|
120
|
+
|
121
|
+
# If line is incomplete, get the next line until it terminates
|
122
|
+
if line_buffer =~ /\n$/ || line_buffer =~ /\Z/
|
123
|
+
if block_given?
|
124
|
+
begin
|
125
|
+
row = CSV.parse_line(line_buffer.strip, {:col_sep => options[:delimiter]})
|
126
|
+
yield(row)
|
127
|
+
next if row.all?{|f| f.nil? }
|
128
|
+
line_buffer = CSV.generate_line(row, {:col_sep => options[:delimiter]})
|
129
|
+
rescue CSV::MalformedCSVError
|
130
|
+
next
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
connection.raw_connection.put_copy_data(line_buffer)
|
135
|
+
|
136
|
+
# Clear the buffer
|
137
|
+
line_buffer = ''
|
120
138
|
end
|
121
|
-
connection.raw_connection.put_copy_data line
|
122
139
|
end
|
123
140
|
end
|
124
141
|
end
|
data/postgres-copy.gemspec
CHANGED
data/spec/copy_from_spec.rb
CHANGED
@@ -100,20 +100,20 @@ describe "COPY FROM" do
|
|
100
100
|
ReservedWordModel.copy_from File.expand_path('spec/fixtures/reserved_words.csv'), :delimiter => "\t"
|
101
101
|
ReservedWordModel.order(:id).map{|r| r.attributes}.should == [{"group"=>"group name", "id"=>1, "select"=>"test select"}]
|
102
102
|
end
|
103
|
-
|
103
|
+
|
104
104
|
it "should import even last columns have empty values" do
|
105
105
|
TestExtendedModel.copy_from File.expand_path('spec/fixtures/comma_with_header_empty_values_at_the_end.csv')
|
106
|
-
TestExtendedModel.order(:id).map{|r| r.attributes}.should ==
|
106
|
+
TestExtendedModel.order(:id).map{|r| r.attributes}.should ==
|
107
107
|
[{"id"=>1, "data"=>"test data 1", "more_data"=>nil, "other_data"=>nil, "final_data"=>nil},
|
108
108
|
{"id"=>2, "data"=>"test data 2", "more_data"=>"9", "other_data"=>nil, "final_data"=>nil},
|
109
109
|
{"id"=>3, "data"=>"test data 2", "more_data"=>"9", "other_data"=>nil, "final_data"=>"0"}]
|
110
110
|
end
|
111
|
-
|
111
|
+
|
112
112
|
it "should import even last columns have empty values with block" do
|
113
113
|
TestExtendedModel.copy_from File.expand_path('spec/fixtures/comma_with_header_empty_values_at_the_end.csv') do |row|
|
114
114
|
row[4]="666"
|
115
115
|
end
|
116
|
-
TestExtendedModel.order(:id).map{|r| r.attributes}.should ==
|
116
|
+
TestExtendedModel.order(:id).map{|r| r.attributes}.should ==
|
117
117
|
[{"id"=>1, "data"=>"test data 1", "more_data"=>nil, "other_data"=>nil, "final_data"=>"666"},
|
118
118
|
{"id"=>2, "data"=>"test data 2", "more_data"=>"9", "other_data"=>nil, "final_data"=>"666"},
|
119
119
|
{"id"=>3, "data"=>"test data 2", "more_data"=>"9", "other_data"=>nil, "final_data"=>"666"}]
|
@@ -150,4 +150,14 @@ describe "COPY FROM" do
|
|
150
150
|
TestModel.copy_from File.open(File.expand_path('spec/fixtures/special_null_with_header.csv'), 'r'), :null => 'NULL'
|
151
151
|
TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => nil}]
|
152
152
|
end
|
153
|
+
|
154
|
+
it "should import with a carriage return in the value" do
|
155
|
+
TestModel.copy_from File.expand_path('spec/fixtures/comma_with_carriage_returns.csv')
|
156
|
+
TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => "test\ndata 1"}]
|
157
|
+
end
|
158
|
+
|
159
|
+
it "should import custom force null expressions from path" do
|
160
|
+
TestModel.copy_from File.expand_path('spec/fixtures/comma_with_empty_string.csv'), :null => '', :force_null => [:data]
|
161
|
+
TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => nil}]
|
162
|
+
end
|
153
163
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: postgres-copy
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.4.
|
4
|
+
version: 1.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Diogo Biazus
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-01-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: pg
|
@@ -124,7 +124,6 @@ files:
|
|
124
124
|
- LICENSE
|
125
125
|
- README.md
|
126
126
|
- Rakefile
|
127
|
-
- VERSION
|
128
127
|
- lib/postgres-copy.rb
|
129
128
|
- lib/postgres-copy/acts_as_copy_target.rb
|
130
129
|
- lib/postgres-copy/csv_responder.rb
|
@@ -135,6 +134,8 @@ files:
|
|
135
134
|
- spec/copy_to_spec.rb
|
136
135
|
- spec/fixtures/2_col_binary_data.dat
|
137
136
|
- spec/fixtures/comma_inside_field.csv
|
137
|
+
- spec/fixtures/comma_with_carriage_returns.csv
|
138
|
+
- spec/fixtures/comma_with_empty_string.csv
|
138
139
|
- spec/fixtures/comma_with_header.csv
|
139
140
|
- spec/fixtures/comma_with_header_and_scope.csv
|
140
141
|
- spec/fixtures/comma_with_header_empty_values_at_the_end.csv
|
@@ -177,7 +178,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
177
178
|
version: '0'
|
178
179
|
requirements: []
|
179
180
|
rubyforge_project:
|
180
|
-
rubygems_version: 2.6.
|
181
|
+
rubygems_version: 2.6.13
|
181
182
|
signing_key:
|
182
183
|
specification_version: 4
|
183
184
|
summary: Put COPY command functionality in ActiveRecord's model class
|
@@ -188,6 +189,8 @@ test_files:
|
|
188
189
|
- spec/copy_to_spec.rb
|
189
190
|
- spec/fixtures/2_col_binary_data.dat
|
190
191
|
- spec/fixtures/comma_inside_field.csv
|
192
|
+
- spec/fixtures/comma_with_carriage_returns.csv
|
193
|
+
- spec/fixtures/comma_with_empty_string.csv
|
191
194
|
- spec/fixtures/comma_with_header.csv
|
192
195
|
- spec/fixtures/comma_with_header_and_scope.csv
|
193
196
|
- spec/fixtures/comma_with_header_empty_values_at_the_end.csv
|
data/VERSION
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
0.5.6
|