postgres-copy 1.4.1 → 1.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/workflows/ruby.yml +48 -0
- data/Gemfile.lock +42 -42
- data/README.md +8 -1
- data/lib/postgres-copy/acts_as_copy_target.rb +3 -1
- data/postgres-copy.gemspec +1 -1
- data/spec/copy_from_spec.rb +13 -0
- data/spec/copy_to_binary_spec.rb +0 -11
- data/spec/copy_to_spec.rb +4 -12
- data/spec/fixtures/tab_with_header.tsv +2 -0
- data/spec/fixtures/tab_with_two_lines.tsv +3 -0
- metadata +7 -37
- data/.travis.yml +0 -13
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: ad654ff2a933f1e41f23e6347f0b20daa3e16455caea9a6107ce4d50c39598f8
|
4
|
+
data.tar.gz: e972e684f928c0afdd7fdd096ade3d28d293604ef20f42a959097234f37f7317
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e57d3d4d0bd7a0bef175dad625c77013f2616222ef9ef086c449cdcbd8f58e4bbde77625e6f70ab55149683becac6e15d8c2cbf84c09c707f1a39cbef0417b05
|
7
|
+
data.tar.gz: b8768a3cf765aa7d6c01eb5138439e20c1140f6b2e772e96d169635832fe14a2780e438732503f719eba7f0b9bcf0e6cf6da4c7241d3d42e734d4433f65799e6
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# This workflow uses actions that are not certified by GitHub.
|
2
|
+
# They are provided by a third-party and are governed by
|
3
|
+
# separate terms of service, privacy policy, and support
|
4
|
+
# documentation.
|
5
|
+
# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
|
6
|
+
# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
|
7
|
+
|
8
|
+
name: Ruby
|
9
|
+
|
10
|
+
on:
|
11
|
+
push:
|
12
|
+
branches: [ master ]
|
13
|
+
pull_request:
|
14
|
+
branches: [ master ]
|
15
|
+
|
16
|
+
jobs:
|
17
|
+
test:
|
18
|
+
services:
|
19
|
+
# Label used to access the service container
|
20
|
+
postgres:
|
21
|
+
# Docker Hub image
|
22
|
+
image: postgres
|
23
|
+
# Provide the password for postgres
|
24
|
+
env:
|
25
|
+
POSTGRES_PASSWORD: postgres
|
26
|
+
ports:
|
27
|
+
- 5432:5432
|
28
|
+
# Set health checks to wait until postgres has started
|
29
|
+
options: >-
|
30
|
+
--health-cmd pg_isready
|
31
|
+
--health-interval 10s
|
32
|
+
--health-timeout 5s
|
33
|
+
--health-retries 5
|
34
|
+
|
35
|
+
runs-on: ubuntu-latest
|
36
|
+
|
37
|
+
steps:
|
38
|
+
- uses: actions/checkout@v2
|
39
|
+
- name: Set up Ruby
|
40
|
+
# To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
|
41
|
+
# change this to (see https://github.com/ruby/setup-ruby#versioning):
|
42
|
+
uses: ruby/setup-ruby@v1
|
43
|
+
with:
|
44
|
+
ruby-version: 2.7
|
45
|
+
- name: Install dependencies
|
46
|
+
run: bundle install
|
47
|
+
- name: Run tests
|
48
|
+
run: bundle exec rake
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
postgres-copy (1.
|
4
|
+
postgres-copy (1.5.0)
|
5
5
|
activerecord (>= 5.1)
|
6
6
|
pg (>= 0.17)
|
7
7
|
responders
|
@@ -9,66 +9,65 @@ PATH
|
|
9
9
|
GEM
|
10
10
|
remote: https://rubygems.org/
|
11
11
|
specs:
|
12
|
-
actionpack (
|
13
|
-
actionview (=
|
14
|
-
activesupport (=
|
15
|
-
rack (~> 2.0)
|
12
|
+
actionpack (6.0.3.2)
|
13
|
+
actionview (= 6.0.3.2)
|
14
|
+
activesupport (= 6.0.3.2)
|
15
|
+
rack (~> 2.0, >= 2.0.8)
|
16
16
|
rack-test (>= 0.6.3)
|
17
17
|
rails-dom-testing (~> 2.0)
|
18
|
-
rails-html-sanitizer (~> 1.0, >= 1.0
|
19
|
-
actionview (
|
20
|
-
activesupport (=
|
18
|
+
rails-html-sanitizer (~> 1.0, >= 1.2.0)
|
19
|
+
actionview (6.0.3.2)
|
20
|
+
activesupport (= 6.0.3.2)
|
21
21
|
builder (~> 3.1)
|
22
22
|
erubi (~> 1.4)
|
23
23
|
rails-dom-testing (~> 2.0)
|
24
|
-
rails-html-sanitizer (~> 1.
|
25
|
-
activemodel (
|
26
|
-
activesupport (=
|
27
|
-
activerecord (
|
28
|
-
activemodel (=
|
29
|
-
activesupport (=
|
30
|
-
|
31
|
-
activesupport (5.2.2)
|
24
|
+
rails-html-sanitizer (~> 1.1, >= 1.2.0)
|
25
|
+
activemodel (6.0.3.2)
|
26
|
+
activesupport (= 6.0.3.2)
|
27
|
+
activerecord (6.0.3.2)
|
28
|
+
activemodel (= 6.0.3.2)
|
29
|
+
activesupport (= 6.0.3.2)
|
30
|
+
activesupport (6.0.3.2)
|
32
31
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
33
32
|
i18n (>= 0.7, < 2)
|
34
33
|
minitest (~> 5.1)
|
35
34
|
tzinfo (~> 1.1)
|
36
|
-
|
37
|
-
builder (3.2.
|
38
|
-
concurrent-ruby (1.1.
|
39
|
-
crass (1.0.
|
40
|
-
diff-lcs (1.
|
41
|
-
erubi (1.
|
42
|
-
i18n (1.
|
35
|
+
zeitwerk (~> 2.2, >= 2.2.2)
|
36
|
+
builder (3.2.4)
|
37
|
+
concurrent-ruby (1.1.6)
|
38
|
+
crass (1.0.6)
|
39
|
+
diff-lcs (1.4.4)
|
40
|
+
erubi (1.9.0)
|
41
|
+
i18n (1.8.3)
|
43
42
|
concurrent-ruby (~> 1.0)
|
44
|
-
loofah (2.
|
43
|
+
loofah (2.6.0)
|
45
44
|
crass (~> 1.0.2)
|
46
45
|
nokogiri (>= 1.5.9)
|
47
|
-
method_source (0.
|
46
|
+
method_source (1.0.0)
|
48
47
|
mini_portile2 (2.4.0)
|
49
|
-
minitest (5.
|
50
|
-
nokogiri (1.10.
|
48
|
+
minitest (5.14.1)
|
49
|
+
nokogiri (1.10.10)
|
51
50
|
mini_portile2 (~> 2.4.0)
|
52
|
-
pg (1.
|
53
|
-
rack (2.
|
51
|
+
pg (1.2.3)
|
52
|
+
rack (2.2.3)
|
54
53
|
rack-test (1.1.0)
|
55
54
|
rack (>= 1.0, < 3)
|
56
55
|
rails-dom-testing (2.0.3)
|
57
56
|
activesupport (>= 4.2.0)
|
58
57
|
nokogiri (>= 1.6)
|
59
|
-
rails-html-sanitizer (1.0
|
60
|
-
loofah (~> 2.
|
61
|
-
railties (
|
62
|
-
actionpack (=
|
63
|
-
activesupport (=
|
58
|
+
rails-html-sanitizer (1.3.0)
|
59
|
+
loofah (~> 2.3)
|
60
|
+
railties (6.0.3.2)
|
61
|
+
actionpack (= 6.0.3.2)
|
62
|
+
activesupport (= 6.0.3.2)
|
64
63
|
method_source
|
65
64
|
rake (>= 0.8.7)
|
66
|
-
thor (>= 0.
|
65
|
+
thor (>= 0.20.3, < 2.0)
|
67
66
|
rake (11.2.2)
|
68
|
-
rdoc (
|
69
|
-
responders (
|
70
|
-
actionpack (>=
|
71
|
-
railties (>=
|
67
|
+
rdoc (6.2.1)
|
68
|
+
responders (3.0.1)
|
69
|
+
actionpack (>= 5.0)
|
70
|
+
railties (>= 5.0)
|
72
71
|
rspec (2.99.0)
|
73
72
|
rspec-core (~> 2.99.0)
|
74
73
|
rspec-expectations (~> 2.99.0)
|
@@ -77,10 +76,11 @@ GEM
|
|
77
76
|
rspec-expectations (2.99.2)
|
78
77
|
diff-lcs (>= 1.1.3, < 2.0)
|
79
78
|
rspec-mocks (2.99.4)
|
80
|
-
thor (0.
|
79
|
+
thor (1.0.1)
|
81
80
|
thread_safe (0.3.6)
|
82
|
-
tzinfo (1.2.
|
81
|
+
tzinfo (1.2.7)
|
83
82
|
thread_safe (~> 0.1)
|
83
|
+
zeitwerk (2.4.0)
|
84
84
|
|
85
85
|
PLATFORMS
|
86
86
|
ruby
|
@@ -93,4 +93,4 @@ DEPENDENCIES
|
|
93
93
|
rspec (~> 2.12)
|
94
94
|
|
95
95
|
BUNDLED WITH
|
96
|
-
1.
|
96
|
+
2.1.2
|
data/README.md
CHANGED
@@ -1,4 +1,6 @@
|
|
1
|
-
# postgres-copy
|
1
|
+
# postgres-copy
|
2
|
+
|
3
|
+
![Ruby](https://github.com/diogob/postgres-copy/workflows/Ruby/badge.svg)
|
2
4
|
|
3
5
|
This Gem will enable your AR models to use the PostgreSQL COPY command to import/export data in CSV format.
|
4
6
|
If you need to tranfer data between a PostgreSQL database and CSV files, the PostgreSQL native CSV parser
|
@@ -172,6 +174,11 @@ Match the specified columns' values against the null string, even if it has been
|
|
172
174
|
User.copy_from "/tmp/users.csv", :null => '', :force_null => [:name, :city]
|
173
175
|
```
|
174
176
|
|
177
|
+
To copy from tsv file , you can set format `:tsv`
|
178
|
+
|
179
|
+
```ruby
|
180
|
+
User.copy_from "/tmp/users.tsv", :format => :tsv
|
181
|
+
```
|
175
182
|
|
176
183
|
To copy a binary formatted data file or IO object you can specify the format as binary
|
177
184
|
|
@@ -73,13 +73,15 @@ module PostgresCopy
|
|
73
73
|
# * For further details on usage take a look at the README.md
|
74
74
|
def copy_from path_or_io, options = {}
|
75
75
|
options = {:delimiter => ",", :format => :csv, :header => true, :quote => '"'}.merge(options)
|
76
|
+
options[:delimiter] = "\t" if options[:format] == :tsv
|
76
77
|
options_string = if options[:format] == :binary
|
77
78
|
"BINARY"
|
78
79
|
else
|
79
80
|
quote = options[:quote] == "'" ? "''" : options[:quote]
|
80
81
|
null = options.key?(:null) ? "NULL '#{options[:null]}'" : nil
|
81
82
|
force_null = options.key?(:force_null) ? "FORCE_NULL(#{options[:force_null].join(',')})" : nil
|
82
|
-
|
83
|
+
delimiter = options[:format] == :tsv ? "E'\t'" : "'#{options[:delimiter]}'"
|
84
|
+
"WITH (" + ["DELIMITER #{delimiter}", "QUOTE '#{quote}'", null, force_null, "FORMAT CSV"].compact.join(', ') + ")"
|
83
85
|
end
|
84
86
|
io = path_or_io.instance_of?(String) ? File.open(path_or_io, 'r') : path_or_io
|
85
87
|
|
data/postgres-copy.gemspec
CHANGED
data/spec/copy_from_spec.rb
CHANGED
@@ -160,4 +160,17 @@ describe "COPY FROM" do
|
|
160
160
|
TestModel.copy_from File.expand_path('spec/fixtures/comma_with_empty_string.csv'), :null => '', :force_null => [:data]
|
161
161
|
TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => nil}]
|
162
162
|
end
|
163
|
+
|
164
|
+
it "should import tsv from path" do
|
165
|
+
TestModel.copy_from File.expand_path('spec/fixtures/tab_with_header.tsv'), :format => :tsv
|
166
|
+
TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'test data 1'}]
|
167
|
+
end
|
168
|
+
|
169
|
+
it "should import 2 lines from tsv and allow changes in block" do
|
170
|
+
TestModel.copy_from(File.open(File.expand_path('spec/fixtures/tab_with_two_lines.tsv'), 'r'), :format => :tsv) do |row|
|
171
|
+
row[1] = 'changed this data'
|
172
|
+
end
|
173
|
+
TestModel.order(:id).first.attributes.should == {'id' => 1, 'data' => 'changed this data'}
|
174
|
+
TestModel.count.should == 2
|
175
|
+
end
|
163
176
|
end
|
data/spec/copy_to_binary_spec.rb
CHANGED
@@ -19,15 +19,4 @@ describe "COPY TO BINARY" do
|
|
19
19
|
it{ should == File.open('spec/fixtures/2_col_binary_data.dat', 'r:ASCII-8BIT').read }
|
20
20
|
end
|
21
21
|
end
|
22
|
-
|
23
|
-
describe "should allow binary output to file" do
|
24
|
-
it "should copy to disk if block is not given and a path is passed" do
|
25
|
-
TestModel.copy_to '/tmp/export.dat', :format => :binary
|
26
|
-
str = File.open('/tmp/export.dat', 'r:ASCII-8BIT').read
|
27
|
-
|
28
|
-
str.should == File.open('spec/fixtures/2_col_binary_data.dat', 'r:ASCII-8BIT').read
|
29
|
-
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
22
|
end
|
data/spec/copy_to_spec.rb
CHANGED
@@ -64,15 +64,6 @@ describe "COPY TO" do
|
|
64
64
|
end
|
65
65
|
end
|
66
66
|
|
67
|
-
it "should copy to disk if block is not given and a path is passed" do
|
68
|
-
TestModel.copy_to '/tmp/export.csv'
|
69
|
-
File.open('spec/fixtures/comma_with_header.csv', 'r') do |fixture|
|
70
|
-
File.open('/tmp/export.csv', 'r') do |result|
|
71
|
-
result.read.should == fixture.read
|
72
|
-
end
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
67
|
it "should raise exception if I pass a path and a block simultaneously" do
|
77
68
|
lambda do
|
78
69
|
TestModel.copy_to('/tmp/bogus_path') do |row|
|
@@ -81,9 +72,10 @@ describe "COPY TO" do
|
|
81
72
|
end
|
82
73
|
|
83
74
|
it "accepts custom sql query to run instead on the current relation" do
|
84
|
-
TestModel.copy_to
|
85
|
-
|
86
|
-
|
75
|
+
TestModel.copy_to(nil, query: 'SELECT count(*) as "Total" FROM test_models') do |row|
|
76
|
+
expect(row).to eq("Total\n")
|
77
|
+
break
|
78
|
+
end
|
87
79
|
end
|
88
80
|
end
|
89
81
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: postgres-copy
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Diogo Biazus
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-07-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: pg
|
@@ -116,9 +116,9 @@ extensions: []
|
|
116
116
|
extra_rdoc_files: []
|
117
117
|
files:
|
118
118
|
- ".document"
|
119
|
+
- ".github/workflows/ruby.yml"
|
119
120
|
- ".gitignore"
|
120
121
|
- ".rspec"
|
121
|
-
- ".travis.yml"
|
122
122
|
- Gemfile
|
123
123
|
- Gemfile.lock
|
124
124
|
- LICENSE
|
@@ -153,8 +153,10 @@ files:
|
|
153
153
|
- spec/fixtures/tab_with_error.csv
|
154
154
|
- spec/fixtures/tab_with_extra_line.csv
|
155
155
|
- spec/fixtures/tab_with_header.csv
|
156
|
+
- spec/fixtures/tab_with_header.tsv
|
156
157
|
- spec/fixtures/tab_with_header_multi.csv
|
157
158
|
- spec/fixtures/tab_with_two_lines.csv
|
159
|
+
- spec/fixtures/tab_with_two_lines.tsv
|
158
160
|
- spec/fixtures/test_extended_model.rb
|
159
161
|
- spec/fixtures/test_model.rb
|
160
162
|
- spec/spec.opts
|
@@ -177,40 +179,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
177
179
|
- !ruby/object:Gem::Version
|
178
180
|
version: '0'
|
179
181
|
requirements: []
|
180
|
-
|
181
|
-
rubygems_version: 2.6.13
|
182
|
+
rubygems_version: 3.1.2
|
182
183
|
signing_key:
|
183
184
|
specification_version: 4
|
184
185
|
summary: Put COPY command functionality in ActiveRecord's model class
|
185
|
-
test_files:
|
186
|
-
- spec/copy_from_binary_spec.rb
|
187
|
-
- spec/copy_from_spec.rb
|
188
|
-
- spec/copy_to_binary_spec.rb
|
189
|
-
- spec/copy_to_spec.rb
|
190
|
-
- spec/fixtures/2_col_binary_data.dat
|
191
|
-
- spec/fixtures/comma_inside_field.csv
|
192
|
-
- spec/fixtures/comma_with_carriage_returns.csv
|
193
|
-
- spec/fixtures/comma_with_empty_string.csv
|
194
|
-
- spec/fixtures/comma_with_header.csv
|
195
|
-
- spec/fixtures/comma_with_header_and_scope.csv
|
196
|
-
- spec/fixtures/comma_with_header_empty_values_at_the_end.csv
|
197
|
-
- spec/fixtures/comma_with_header_multi.csv
|
198
|
-
- spec/fixtures/comma_without_header.csv
|
199
|
-
- spec/fixtures/extra_field.rb
|
200
|
-
- spec/fixtures/reserved_word_model.rb
|
201
|
-
- spec/fixtures/reserved_words.csv
|
202
|
-
- spec/fixtures/semicolon_with_different_header.csv
|
203
|
-
- spec/fixtures/semicolon_with_header.csv
|
204
|
-
- spec/fixtures/semicolon_with_quote.csv
|
205
|
-
- spec/fixtures/special_null_with_header.csv
|
206
|
-
- spec/fixtures/tab_only_data.csv
|
207
|
-
- spec/fixtures/tab_with_different_header.csv
|
208
|
-
- spec/fixtures/tab_with_error.csv
|
209
|
-
- spec/fixtures/tab_with_extra_line.csv
|
210
|
-
- spec/fixtures/tab_with_header.csv
|
211
|
-
- spec/fixtures/tab_with_header_multi.csv
|
212
|
-
- spec/fixtures/tab_with_two_lines.csv
|
213
|
-
- spec/fixtures/test_extended_model.rb
|
214
|
-
- spec/fixtures/test_model.rb
|
215
|
-
- spec/spec.opts
|
216
|
-
- spec/spec_helper.rb
|
186
|
+
test_files: []
|