postgres-copy 1.1.2 → 1.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/workflows/ruby.yml +48 -0
- data/.gitignore +4 -0
- data/Gemfile.lock +55 -51
- data/README.md +45 -9
- data/lib/postgres-copy/acts_as_copy_target.rb +56 -10
- data/postgres-copy.gemspec +2 -2
- data/spec/copy_from_spec.rb +27 -4
- data/spec/copy_to_binary_spec.rb +0 -11
- data/spec/copy_to_spec.rb +43 -11
- data/spec/fixtures/comma_with_carriage_returns.csv +3 -0
- data/spec/fixtures/comma_with_empty_string.csv +2 -0
- data/spec/fixtures/comma_with_header_and_scope.csv +4 -0
- data/spec/fixtures/comma_with_header_multi.csv +5 -0
- data/spec/fixtures/tab_with_header.tsv +2 -0
- data/spec/fixtures/tab_with_header_multi.csv +5 -0
- data/spec/fixtures/tab_with_two_lines.tsv +3 -0
- metadata +14 -35
- data/.travis.yml +0 -7
- data/VERSION +0 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: ad654ff2a933f1e41f23e6347f0b20daa3e16455caea9a6107ce4d50c39598f8
|
4
|
+
data.tar.gz: e972e684f928c0afdd7fdd096ade3d28d293604ef20f42a959097234f37f7317
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e57d3d4d0bd7a0bef175dad625c77013f2616222ef9ef086c449cdcbd8f58e4bbde77625e6f70ab55149683becac6e15d8c2cbf84c09c707f1a39cbef0417b05
|
7
|
+
data.tar.gz: b8768a3cf765aa7d6c01eb5138439e20c1140f6b2e772e96d169635832fe14a2780e438732503f719eba7f0b9bcf0e6cf6da4c7241d3d42e734d4433f65799e6
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# This workflow uses actions that are not certified by GitHub.
|
2
|
+
# They are provided by a third-party and are governed by
|
3
|
+
# separate terms of service, privacy policy, and support
|
4
|
+
# documentation.
|
5
|
+
# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
|
6
|
+
# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
|
7
|
+
|
8
|
+
name: Ruby
|
9
|
+
|
10
|
+
on:
|
11
|
+
push:
|
12
|
+
branches: [ master ]
|
13
|
+
pull_request:
|
14
|
+
branches: [ master ]
|
15
|
+
|
16
|
+
jobs:
|
17
|
+
test:
|
18
|
+
services:
|
19
|
+
# Label used to access the service container
|
20
|
+
postgres:
|
21
|
+
# Docker Hub image
|
22
|
+
image: postgres
|
23
|
+
# Provide the password for postgres
|
24
|
+
env:
|
25
|
+
POSTGRES_PASSWORD: postgres
|
26
|
+
ports:
|
27
|
+
- 5432:5432
|
28
|
+
# Set health checks to wait until postgres has started
|
29
|
+
options: >-
|
30
|
+
--health-cmd pg_isready
|
31
|
+
--health-interval 10s
|
32
|
+
--health-timeout 5s
|
33
|
+
--health-retries 5
|
34
|
+
|
35
|
+
runs-on: ubuntu-latest
|
36
|
+
|
37
|
+
steps:
|
38
|
+
- uses: actions/checkout@v2
|
39
|
+
- name: Set up Ruby
|
40
|
+
# To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
|
41
|
+
# change this to (see https://github.com/ruby/setup-ruby#versioning):
|
42
|
+
uses: ruby/setup-ruby@v1
|
43
|
+
with:
|
44
|
+
ruby-version: 2.7
|
45
|
+
- name: Install dependencies
|
46
|
+
run: bundle install
|
47
|
+
- name: Run tests
|
48
|
+
run: bundle exec rake
|
data/.gitignore
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,70 +1,73 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
postgres-copy (1.
|
5
|
-
activerecord (>=
|
4
|
+
postgres-copy (1.5.0)
|
5
|
+
activerecord (>= 5.1)
|
6
6
|
pg (>= 0.17)
|
7
7
|
responders
|
8
8
|
|
9
9
|
GEM
|
10
10
|
remote: https://rubygems.org/
|
11
11
|
specs:
|
12
|
-
actionpack (
|
13
|
-
actionview (=
|
14
|
-
activesupport (=
|
15
|
-
rack (~> 2.0)
|
16
|
-
rack-test (
|
12
|
+
actionpack (6.0.3.2)
|
13
|
+
actionview (= 6.0.3.2)
|
14
|
+
activesupport (= 6.0.3.2)
|
15
|
+
rack (~> 2.0, >= 2.0.8)
|
16
|
+
rack-test (>= 0.6.3)
|
17
17
|
rails-dom-testing (~> 2.0)
|
18
|
-
rails-html-sanitizer (~> 1.0, >= 1.0
|
19
|
-
actionview (
|
20
|
-
activesupport (=
|
18
|
+
rails-html-sanitizer (~> 1.0, >= 1.2.0)
|
19
|
+
actionview (6.0.3.2)
|
20
|
+
activesupport (= 6.0.3.2)
|
21
21
|
builder (~> 3.1)
|
22
|
-
|
22
|
+
erubi (~> 1.4)
|
23
23
|
rails-dom-testing (~> 2.0)
|
24
|
-
rails-html-sanitizer (~> 1.
|
25
|
-
activemodel (
|
26
|
-
activesupport (=
|
27
|
-
activerecord (
|
28
|
-
activemodel (=
|
29
|
-
activesupport (=
|
30
|
-
|
31
|
-
activesupport (5.0.1)
|
24
|
+
rails-html-sanitizer (~> 1.1, >= 1.2.0)
|
25
|
+
activemodel (6.0.3.2)
|
26
|
+
activesupport (= 6.0.3.2)
|
27
|
+
activerecord (6.0.3.2)
|
28
|
+
activemodel (= 6.0.3.2)
|
29
|
+
activesupport (= 6.0.3.2)
|
30
|
+
activesupport (6.0.3.2)
|
32
31
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
33
|
-
i18n (
|
32
|
+
i18n (>= 0.7, < 2)
|
34
33
|
minitest (~> 5.1)
|
35
34
|
tzinfo (~> 1.1)
|
36
|
-
|
37
|
-
builder (3.2.
|
38
|
-
concurrent-ruby (1.
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
35
|
+
zeitwerk (~> 2.2, >= 2.2.2)
|
36
|
+
builder (3.2.4)
|
37
|
+
concurrent-ruby (1.1.6)
|
38
|
+
crass (1.0.6)
|
39
|
+
diff-lcs (1.4.4)
|
40
|
+
erubi (1.9.0)
|
41
|
+
i18n (1.8.3)
|
42
|
+
concurrent-ruby (~> 1.0)
|
43
|
+
loofah (2.6.0)
|
44
|
+
crass (~> 1.0.2)
|
43
45
|
nokogiri (>= 1.5.9)
|
44
|
-
method_source (0.
|
45
|
-
mini_portile2 (2.
|
46
|
-
minitest (5.
|
47
|
-
nokogiri (1.
|
48
|
-
mini_portile2 (~> 2.
|
49
|
-
pg (
|
50
|
-
rack (2.
|
51
|
-
rack-test (
|
52
|
-
rack (>= 1.0)
|
53
|
-
rails-dom-testing (2.0.
|
54
|
-
activesupport (>= 4.2.0
|
55
|
-
nokogiri (
|
56
|
-
rails-html-sanitizer (1.0
|
57
|
-
loofah (~> 2.
|
58
|
-
railties (
|
59
|
-
actionpack (=
|
60
|
-
activesupport (=
|
46
|
+
method_source (1.0.0)
|
47
|
+
mini_portile2 (2.4.0)
|
48
|
+
minitest (5.14.1)
|
49
|
+
nokogiri (1.10.10)
|
50
|
+
mini_portile2 (~> 2.4.0)
|
51
|
+
pg (1.2.3)
|
52
|
+
rack (2.2.3)
|
53
|
+
rack-test (1.1.0)
|
54
|
+
rack (>= 1.0, < 3)
|
55
|
+
rails-dom-testing (2.0.3)
|
56
|
+
activesupport (>= 4.2.0)
|
57
|
+
nokogiri (>= 1.6)
|
58
|
+
rails-html-sanitizer (1.3.0)
|
59
|
+
loofah (~> 2.3)
|
60
|
+
railties (6.0.3.2)
|
61
|
+
actionpack (= 6.0.3.2)
|
62
|
+
activesupport (= 6.0.3.2)
|
61
63
|
method_source
|
62
64
|
rake (>= 0.8.7)
|
63
|
-
thor (>= 0.
|
65
|
+
thor (>= 0.20.3, < 2.0)
|
64
66
|
rake (11.2.2)
|
65
|
-
rdoc (
|
66
|
-
responders (
|
67
|
-
|
67
|
+
rdoc (6.2.1)
|
68
|
+
responders (3.0.1)
|
69
|
+
actionpack (>= 5.0)
|
70
|
+
railties (>= 5.0)
|
68
71
|
rspec (2.99.0)
|
69
72
|
rspec-core (~> 2.99.0)
|
70
73
|
rspec-expectations (~> 2.99.0)
|
@@ -73,10 +76,11 @@ GEM
|
|
73
76
|
rspec-expectations (2.99.2)
|
74
77
|
diff-lcs (>= 1.1.3, < 2.0)
|
75
78
|
rspec-mocks (2.99.4)
|
76
|
-
thor (0.
|
79
|
+
thor (1.0.1)
|
77
80
|
thread_safe (0.3.6)
|
78
|
-
tzinfo (1.2.
|
81
|
+
tzinfo (1.2.7)
|
79
82
|
thread_safe (~> 0.1)
|
83
|
+
zeitwerk (2.4.0)
|
80
84
|
|
81
85
|
PLATFORMS
|
82
86
|
ruby
|
@@ -89,4 +93,4 @@ DEPENDENCIES
|
|
89
93
|
rspec (~> 2.12)
|
90
94
|
|
91
95
|
BUNDLED WITH
|
92
|
-
1.
|
96
|
+
2.1.2
|
data/README.md
CHANGED
@@ -1,11 +1,13 @@
|
|
1
|
-
# postgres-copy
|
1
|
+
# postgres-copy
|
2
|
+
|
3
|
+
![Ruby](https://github.com/diogob/postgres-copy/workflows/Ruby/badge.svg)
|
2
4
|
|
3
5
|
This Gem will enable your AR models to use the PostgreSQL COPY command to import/export data in CSV format.
|
4
6
|
If you need to tranfer data between a PostgreSQL database and CSV files, the PostgreSQL native CSV parser
|
5
7
|
will give you a greater performance than using the ruby CSV+INSERT commands.
|
6
8
|
I have not found time to make accurate benchmarks, but in the use scenario where I have developed the gem
|
7
9
|
I have had a four-fold performance gain.
|
8
|
-
This gem was written having the Rails framework in mind, I think it could work only with active-record,
|
10
|
+
This gem was written having the Rails framework in mind, I think it could work only with active-record,
|
9
11
|
but I will assume in this README that you are using Rails.
|
10
12
|
|
11
13
|
## Install
|
@@ -32,16 +34,17 @@ class User < ActiveRecord::Base
|
|
32
34
|
end
|
33
35
|
```
|
34
36
|
|
35
|
-
This will add the
|
37
|
+
This will add the additional class methods to your model:
|
36
38
|
|
37
|
-
* copy_to
|
39
|
+
* copy_to
|
38
40
|
* copy_to_string
|
41
|
+
* copy_to_enumerator
|
39
42
|
* copy_from
|
40
43
|
|
41
44
|
### Using copy_to and copy_to_string
|
42
45
|
|
43
46
|
You can go to the rails console and try some cool things first.
|
44
|
-
The first and most basic use case, let's copy the
|
47
|
+
The first and most basic use case, let's copy the entire content of a database table to a CSV file on the database server disk.
|
45
48
|
Assuming we have a users table and a User AR model:
|
46
49
|
|
47
50
|
```ruby
|
@@ -54,8 +57,8 @@ This will execute in the database the command:
|
|
54
57
|
COPY (SELECT "users".* FROM "users" ) TO '/tmp/users.csv' WITH DELIMITER ',' CSV HEADER
|
55
58
|
```
|
56
59
|
|
57
|
-
Remark that the file will be created in the database server disk.
|
58
|
-
But what if you want to write the lines in a file on the server that is running Rails, instead of the database?
|
60
|
+
Remark that the file will be created in the database server disk.
|
61
|
+
But what if you want to write the lines in a file on the server that is running Rails, instead of the database?
|
59
62
|
In this case you can pass a block and retrieve the generated lines and then write them to a file:
|
60
63
|
|
61
64
|
```ruby
|
@@ -66,13 +69,23 @@ File.open('/tmp/users.csv', 'w') do |f|
|
|
66
69
|
end
|
67
70
|
```
|
68
71
|
|
72
|
+
Instead of yielding each line, you could return an enumerator with all users:
|
73
|
+
```ruby
|
74
|
+
enumerator = User.copy_to_enumerator
|
75
|
+
```
|
76
|
+
|
77
|
+
And for better performance when rendering the result of the enumerator, you can return an enumerator with blocks of 100 lines joined:
|
78
|
+
```ruby
|
79
|
+
enumerator = User.copy_to_enumerator(:buffer_lines => 100)
|
80
|
+
```
|
81
|
+
|
69
82
|
Or, if you have enough memory, you can read all table contents to a string using .copy_to_string
|
70
83
|
|
71
84
|
```ruby
|
72
85
|
puts User.copy_to_string
|
73
86
|
```
|
74
87
|
|
75
|
-
Another
|
88
|
+
Another interesting feature of copy_to is that it uses the scoped relation, it means that you can use ARel
|
76
89
|
operations to generate different CSV files according to your needs.
|
77
90
|
Assuming we want to generate a file only with the names of users 1, 2 and 3:
|
78
91
|
|
@@ -86,6 +99,18 @@ Which will generate the following SQL command:
|
|
86
99
|
COPY (SELECT name FROM "users" WHERE "users"."id" IN (1, 2, 3)) TO '/tmp/users.csv' WITH DELIMITER ',' CSV HEADER
|
87
100
|
```
|
88
101
|
|
102
|
+
Alternatively, you can supply customized raw SQL query to copy_to instead of scoped relation:
|
103
|
+
|
104
|
+
```ruby
|
105
|
+
User.copy_to("/tmp/users.csv", query: 'SELECT count(*) as Total FROM users')
|
106
|
+
```
|
107
|
+
|
108
|
+
Which will generate the following SQL command:
|
109
|
+
|
110
|
+
```sql
|
111
|
+
COPY (SELECT count(*) as Total FROM users) TO '/tmp/users.csv' WITH DELIMITER ',' CSV HEADER
|
112
|
+
```
|
113
|
+
|
89
114
|
The COPY command also supports exporting the data in binary format.
|
90
115
|
|
91
116
|
```ruby
|
@@ -133,7 +158,7 @@ User.copy_from "/tmp/users.csv" do |row|
|
|
133
158
|
end
|
134
159
|
```
|
135
160
|
|
136
|
-
The above
|
161
|
+
The above example will always change the value of the first column to "fixed string" before storing it into the database.
|
137
162
|
For each iteration of the block row receives an array with the same order as the columns in the CSV file.
|
138
163
|
|
139
164
|
|
@@ -143,6 +168,17 @@ To specify NULL value you can pass the null option parameter.
|
|
143
168
|
User.copy_from "/tmp/users.csv", :null => 'null'
|
144
169
|
```
|
145
170
|
|
171
|
+
Match the specified columns' values against the null string, even if it has been quoted, and if a match is found set the value to NULL (Postgres 9.4+ only).
|
172
|
+
|
173
|
+
```ruby
|
174
|
+
User.copy_from "/tmp/users.csv", :null => '', :force_null => [:name, :city]
|
175
|
+
```
|
176
|
+
|
177
|
+
To copy from tsv file , you can set format `:tsv`
|
178
|
+
|
179
|
+
```ruby
|
180
|
+
User.copy_from "/tmp/users.tsv", :format => :tsv
|
181
|
+
```
|
146
182
|
|
147
183
|
To copy a binary formatted data file or IO object you can specify the format as binary
|
148
184
|
|
@@ -16,12 +16,13 @@ module PostgresCopy
|
|
16
16
|
else
|
17
17
|
"DELIMITER '#{options[:delimiter]}' CSV #{options[:header] ? 'HEADER' : ''}"
|
18
18
|
end
|
19
|
+
options_query = options.delete(:query) || self.all.to_sql
|
19
20
|
|
20
21
|
if path
|
21
22
|
raise "You have to choose between exporting to a file or receiving the lines inside a block" if block_given?
|
22
|
-
connection.execute "COPY (#{
|
23
|
+
connection.execute "COPY (#{options_query}) TO '#{sanitize_sql(path)}' WITH #{options_string}"
|
23
24
|
else
|
24
|
-
connection.raw_connection.copy_data "COPY (#{
|
25
|
+
connection.raw_connection.copy_data "COPY (#{options_query}) TO STDOUT WITH #{options_string}" do
|
25
26
|
while line = connection.raw_connection.get_copy_data do
|
26
27
|
yield(line) if block_given?
|
27
28
|
end
|
@@ -30,6 +31,32 @@ module PostgresCopy
|
|
30
31
|
return self
|
31
32
|
end
|
32
33
|
|
34
|
+
# Create an enumerator with each line from the CSV.
|
35
|
+
# Note that using this directly in a controller response
|
36
|
+
# will perform very poorly as each line will get put
|
37
|
+
# into its own chunk. Joining every (eg) 100 rows together
|
38
|
+
# is much, much faster.
|
39
|
+
def copy_to_enumerator(options={})
|
40
|
+
buffer_lines = options.delete(:buffer_lines)
|
41
|
+
# Somehow, self loses its scope once inside the Enumerator
|
42
|
+
scope = self.current_scope || self
|
43
|
+
result = Enumerator.new do |y|
|
44
|
+
scope.copy_to(nil, options) do |line|
|
45
|
+
y << line
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
if buffer_lines.to_i > 0
|
50
|
+
Enumerator.new do |y|
|
51
|
+
result.each_slice(buffer_lines.to_i) do |slice|
|
52
|
+
y << slice.join
|
53
|
+
end
|
54
|
+
end
|
55
|
+
else
|
56
|
+
result
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
33
60
|
# Copy all data to a single string
|
34
61
|
def copy_to_string options = {}
|
35
62
|
data = ''
|
@@ -46,12 +73,15 @@ module PostgresCopy
|
|
46
73
|
# * For further details on usage take a look at the README.md
|
47
74
|
def copy_from path_or_io, options = {}
|
48
75
|
options = {:delimiter => ",", :format => :csv, :header => true, :quote => '"'}.merge(options)
|
76
|
+
options[:delimiter] = "\t" if options[:format] == :tsv
|
49
77
|
options_string = if options[:format] == :binary
|
50
78
|
"BINARY"
|
51
79
|
else
|
52
80
|
quote = options[:quote] == "'" ? "''" : options[:quote]
|
53
|
-
null = options.key?(:null) ? "NULL '#{options[:null]}'" :
|
54
|
-
"
|
81
|
+
null = options.key?(:null) ? "NULL '#{options[:null]}'" : nil
|
82
|
+
force_null = options.key?(:force_null) ? "FORCE_NULL(#{options[:force_null].join(',')})" : nil
|
83
|
+
delimiter = options[:format] == :tsv ? "E'\t'" : "'#{options[:delimiter]}'"
|
84
|
+
"WITH (" + ["DELIMITER #{delimiter}", "QUOTE '#{quote}'", null, force_null, "FORMAT CSV"].compact.join(', ') + ")"
|
55
85
|
end
|
56
86
|
io = path_or_io.instance_of?(String) ? File.open(path_or_io, 'r') : path_or_io
|
57
87
|
|
@@ -83,15 +113,31 @@ module PostgresCopy
|
|
83
113
|
rescue EOFError
|
84
114
|
end
|
85
115
|
else
|
116
|
+
line_buffer = ''
|
117
|
+
|
86
118
|
while line = io.gets do
|
87
119
|
next if line.strip.size == 0
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
120
|
+
|
121
|
+
line_buffer += line
|
122
|
+
|
123
|
+
# If line is incomplete, get the next line until it terminates
|
124
|
+
if line_buffer =~ /\n$/ || line_buffer =~ /\Z/
|
125
|
+
if block_given?
|
126
|
+
begin
|
127
|
+
row = CSV.parse_line(line_buffer.strip, {:col_sep => options[:delimiter]})
|
128
|
+
yield(row)
|
129
|
+
next if row.all?{|f| f.nil? }
|
130
|
+
line_buffer = CSV.generate_line(row, {:col_sep => options[:delimiter]})
|
131
|
+
rescue CSV::MalformedCSVError
|
132
|
+
next
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
connection.raw_connection.put_copy_data(line_buffer)
|
137
|
+
|
138
|
+
# Clear the buffer
|
139
|
+
line_buffer = ''
|
93
140
|
end
|
94
|
-
connection.raw_connection.put_copy_data line
|
95
141
|
end
|
96
142
|
end
|
97
143
|
end
|
data/postgres-copy.gemspec
CHANGED
@@ -5,7 +5,7 @@ $:.unshift lib unless $:.include?(lib)
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "postgres-copy"
|
8
|
-
s.version = "1.
|
8
|
+
s.version = "1.5.0"
|
9
9
|
s.platform = Gem::Platform::RUBY
|
10
10
|
s.required_ruby_version = ">= 1.9.3"
|
11
11
|
s.authors = ["Diogo Biazus"]
|
@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
|
|
21
21
|
s.summary = "Put COPY command functionality in ActiveRecord's model class"
|
22
22
|
|
23
23
|
s.add_dependency "pg", ">= 0.17"
|
24
|
-
s.add_dependency "activerecord", '>=
|
24
|
+
s.add_dependency "activerecord", '>= 5.1'
|
25
25
|
s.add_dependency "responders"
|
26
26
|
s.add_development_dependency "bundler"
|
27
27
|
s.add_development_dependency "rdoc"
|
data/spec/copy_from_spec.rb
CHANGED
@@ -100,20 +100,20 @@ describe "COPY FROM" do
|
|
100
100
|
ReservedWordModel.copy_from File.expand_path('spec/fixtures/reserved_words.csv'), :delimiter => "\t"
|
101
101
|
ReservedWordModel.order(:id).map{|r| r.attributes}.should == [{"group"=>"group name", "id"=>1, "select"=>"test select"}]
|
102
102
|
end
|
103
|
-
|
103
|
+
|
104
104
|
it "should import even last columns have empty values" do
|
105
105
|
TestExtendedModel.copy_from File.expand_path('spec/fixtures/comma_with_header_empty_values_at_the_end.csv')
|
106
|
-
TestExtendedModel.order(:id).map{|r| r.attributes}.should ==
|
106
|
+
TestExtendedModel.order(:id).map{|r| r.attributes}.should ==
|
107
107
|
[{"id"=>1, "data"=>"test data 1", "more_data"=>nil, "other_data"=>nil, "final_data"=>nil},
|
108
108
|
{"id"=>2, "data"=>"test data 2", "more_data"=>"9", "other_data"=>nil, "final_data"=>nil},
|
109
109
|
{"id"=>3, "data"=>"test data 2", "more_data"=>"9", "other_data"=>nil, "final_data"=>"0"}]
|
110
110
|
end
|
111
|
-
|
111
|
+
|
112
112
|
it "should import even last columns have empty values with block" do
|
113
113
|
TestExtendedModel.copy_from File.expand_path('spec/fixtures/comma_with_header_empty_values_at_the_end.csv') do |row|
|
114
114
|
row[4]="666"
|
115
115
|
end
|
116
|
-
TestExtendedModel.order(:id).map{|r| r.attributes}.should ==
|
116
|
+
TestExtendedModel.order(:id).map{|r| r.attributes}.should ==
|
117
117
|
[{"id"=>1, "data"=>"test data 1", "more_data"=>nil, "other_data"=>nil, "final_data"=>"666"},
|
118
118
|
{"id"=>2, "data"=>"test data 2", "more_data"=>"9", "other_data"=>nil, "final_data"=>"666"},
|
119
119
|
{"id"=>3, "data"=>"test data 2", "more_data"=>"9", "other_data"=>nil, "final_data"=>"666"}]
|
@@ -150,4 +150,27 @@ describe "COPY FROM" do
|
|
150
150
|
TestModel.copy_from File.open(File.expand_path('spec/fixtures/special_null_with_header.csv'), 'r'), :null => 'NULL'
|
151
151
|
TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => nil}]
|
152
152
|
end
|
153
|
+
|
154
|
+
it "should import with a carriage return in the value" do
|
155
|
+
TestModel.copy_from File.expand_path('spec/fixtures/comma_with_carriage_returns.csv')
|
156
|
+
TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => "test\ndata 1"}]
|
157
|
+
end
|
158
|
+
|
159
|
+
it "should import custom force null expressions from path" do
|
160
|
+
TestModel.copy_from File.expand_path('spec/fixtures/comma_with_empty_string.csv'), :null => '', :force_null => [:data]
|
161
|
+
TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => nil}]
|
162
|
+
end
|
163
|
+
|
164
|
+
it "should import tsv from path" do
|
165
|
+
TestModel.copy_from File.expand_path('spec/fixtures/tab_with_header.tsv'), :format => :tsv
|
166
|
+
TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'test data 1'}]
|
167
|
+
end
|
168
|
+
|
169
|
+
it "should import 2 lines from tsv and allow changes in block" do
|
170
|
+
TestModel.copy_from(File.open(File.expand_path('spec/fixtures/tab_with_two_lines.tsv'), 'r'), :format => :tsv) do |row|
|
171
|
+
row[1] = 'changed this data'
|
172
|
+
end
|
173
|
+
TestModel.order(:id).first.attributes.should == {'id' => 1, 'data' => 'changed this data'}
|
174
|
+
TestModel.count.should == 2
|
175
|
+
end
|
153
176
|
end
|
data/spec/copy_to_binary_spec.rb
CHANGED
@@ -19,15 +19,4 @@ describe "COPY TO BINARY" do
|
|
19
19
|
it{ should == File.open('spec/fixtures/2_col_binary_data.dat', 'r:ASCII-8BIT').read }
|
20
20
|
end
|
21
21
|
end
|
22
|
-
|
23
|
-
describe "should allow binary output to file" do
|
24
|
-
it "should copy to disk if block is not given and a path is passed" do
|
25
|
-
TestModel.copy_to '/tmp/export.dat', :format => :binary
|
26
|
-
str = File.open('/tmp/export.dat', 'r:ASCII-8BIT').read
|
27
|
-
|
28
|
-
str.should == File.open('spec/fixtures/2_col_binary_data.dat', 'r:ASCII-8BIT').read
|
29
|
-
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
22
|
end
|
data/spec/copy_to_spec.rb
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
2
2
|
|
3
3
|
describe "COPY TO" do
|
4
|
-
before(:
|
4
|
+
before(:each) do
|
5
5
|
ActiveRecord::Base.connection.execute %{
|
6
6
|
TRUNCATE TABLE test_models;
|
7
7
|
SELECT setval('test_models_id_seq', 1, false);
|
8
|
-
}
|
8
|
+
}
|
9
9
|
TestModel.create :data => 'test data 1'
|
10
10
|
end
|
11
11
|
|
@@ -21,6 +21,40 @@ describe "COPY TO" do
|
|
21
21
|
end
|
22
22
|
end
|
23
23
|
|
24
|
+
describe ".copy_to_enumerator" do
|
25
|
+
before(:each) do
|
26
|
+
TestModel.create :data => 'test data 2'
|
27
|
+
TestModel.create :data => 'test data 3'
|
28
|
+
TestModel.create :data => 'test data 4'
|
29
|
+
end
|
30
|
+
|
31
|
+
context "with no options" do
|
32
|
+
subject{ TestModel.copy_to_enumerator.to_a }
|
33
|
+
it{ should == File.open('spec/fixtures/comma_with_header_multi.csv', 'r').read.lines }
|
34
|
+
end
|
35
|
+
|
36
|
+
context "with tab as delimiter" do
|
37
|
+
subject{ TestModel.copy_to_enumerator(:delimiter => "\t").to_a }
|
38
|
+
it{ should == File.open('spec/fixtures/tab_with_header_multi.csv', 'r').read.lines }
|
39
|
+
end
|
40
|
+
|
41
|
+
context "with many records" do
|
42
|
+
context "enumerating in batches" do
|
43
|
+
subject{ TestModel.copy_to_enumerator(:buffer_lines => 2).to_a }
|
44
|
+
it do
|
45
|
+
expected = []
|
46
|
+
File.open('spec/fixtures/comma_with_header_multi.csv', 'r').read.lines.each_slice(2){|s| expected << s.join }
|
47
|
+
should == expected
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
context "excluding some records via a scope" do
|
52
|
+
subject{ TestModel.where("data not like '%3'").copy_to_enumerator.to_a }
|
53
|
+
it{ should == File.open('spec/fixtures/comma_with_header_and_scope.csv', 'r').read.lines }
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
24
58
|
describe ".copy_to" do
|
25
59
|
it "should copy and pass data to block if block is given and no path is passed" do
|
26
60
|
File.open('spec/fixtures/comma_with_header.csv', 'r') do |f|
|
@@ -30,20 +64,18 @@ describe "COPY TO" do
|
|
30
64
|
end
|
31
65
|
end
|
32
66
|
|
33
|
-
it "should copy to disk if block is not given and a path is passed" do
|
34
|
-
TestModel.copy_to '/tmp/export.csv'
|
35
|
-
File.open('spec/fixtures/comma_with_header.csv', 'r') do |fixture|
|
36
|
-
File.open('/tmp/export.csv', 'r') do |result|
|
37
|
-
result.read.should == fixture.read
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
67
|
it "should raise exception if I pass a path and a block simultaneously" do
|
43
68
|
lambda do
|
44
69
|
TestModel.copy_to('/tmp/bogus_path') do |row|
|
45
70
|
end
|
46
71
|
end.should raise_error
|
47
72
|
end
|
73
|
+
|
74
|
+
it "accepts custom sql query to run instead on the current relation" do
|
75
|
+
TestModel.copy_to(nil, query: 'SELECT count(*) as "Total" FROM test_models') do |row|
|
76
|
+
expect(row).to eq("Total\n")
|
77
|
+
break
|
78
|
+
end
|
79
|
+
end
|
48
80
|
end
|
49
81
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: postgres-copy
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Diogo Biazus
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-07-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: pg
|
@@ -30,14 +30,14 @@ dependencies:
|
|
30
30
|
requirements:
|
31
31
|
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
33
|
+
version: '5.1'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
40
|
+
version: '5.1'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: responders
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -116,15 +116,14 @@ extensions: []
|
|
116
116
|
extra_rdoc_files: []
|
117
117
|
files:
|
118
118
|
- ".document"
|
119
|
+
- ".github/workflows/ruby.yml"
|
119
120
|
- ".gitignore"
|
120
121
|
- ".rspec"
|
121
|
-
- ".travis.yml"
|
122
122
|
- Gemfile
|
123
123
|
- Gemfile.lock
|
124
124
|
- LICENSE
|
125
125
|
- README.md
|
126
126
|
- Rakefile
|
127
|
-
- VERSION
|
128
127
|
- lib/postgres-copy.rb
|
129
128
|
- lib/postgres-copy/acts_as_copy_target.rb
|
130
129
|
- lib/postgres-copy/csv_responder.rb
|
@@ -135,8 +134,12 @@ files:
|
|
135
134
|
- spec/copy_to_spec.rb
|
136
135
|
- spec/fixtures/2_col_binary_data.dat
|
137
136
|
- spec/fixtures/comma_inside_field.csv
|
137
|
+
- spec/fixtures/comma_with_carriage_returns.csv
|
138
|
+
- spec/fixtures/comma_with_empty_string.csv
|
138
139
|
- spec/fixtures/comma_with_header.csv
|
140
|
+
- spec/fixtures/comma_with_header_and_scope.csv
|
139
141
|
- spec/fixtures/comma_with_header_empty_values_at_the_end.csv
|
142
|
+
- spec/fixtures/comma_with_header_multi.csv
|
140
143
|
- spec/fixtures/comma_without_header.csv
|
141
144
|
- spec/fixtures/extra_field.rb
|
142
145
|
- spec/fixtures/reserved_word_model.rb
|
@@ -150,7 +153,10 @@ files:
|
|
150
153
|
- spec/fixtures/tab_with_error.csv
|
151
154
|
- spec/fixtures/tab_with_extra_line.csv
|
152
155
|
- spec/fixtures/tab_with_header.csv
|
156
|
+
- spec/fixtures/tab_with_header.tsv
|
157
|
+
- spec/fixtures/tab_with_header_multi.csv
|
153
158
|
- spec/fixtures/tab_with_two_lines.csv
|
159
|
+
- spec/fixtures/tab_with_two_lines.tsv
|
154
160
|
- spec/fixtures/test_extended_model.rb
|
155
161
|
- spec/fixtures/test_model.rb
|
156
162
|
- spec/spec.opts
|
@@ -173,35 +179,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
173
179
|
- !ruby/object:Gem::Version
|
174
180
|
version: '0'
|
175
181
|
requirements: []
|
176
|
-
|
177
|
-
rubygems_version: 2.5.1
|
182
|
+
rubygems_version: 3.1.2
|
178
183
|
signing_key:
|
179
184
|
specification_version: 4
|
180
185
|
summary: Put COPY command functionality in ActiveRecord's model class
|
181
|
-
test_files:
|
182
|
-
- spec/copy_from_binary_spec.rb
|
183
|
-
- spec/copy_from_spec.rb
|
184
|
-
- spec/copy_to_binary_spec.rb
|
185
|
-
- spec/copy_to_spec.rb
|
186
|
-
- spec/fixtures/2_col_binary_data.dat
|
187
|
-
- spec/fixtures/comma_inside_field.csv
|
188
|
-
- spec/fixtures/comma_with_header.csv
|
189
|
-
- spec/fixtures/comma_with_header_empty_values_at_the_end.csv
|
190
|
-
- spec/fixtures/comma_without_header.csv
|
191
|
-
- spec/fixtures/extra_field.rb
|
192
|
-
- spec/fixtures/reserved_word_model.rb
|
193
|
-
- spec/fixtures/reserved_words.csv
|
194
|
-
- spec/fixtures/semicolon_with_different_header.csv
|
195
|
-
- spec/fixtures/semicolon_with_header.csv
|
196
|
-
- spec/fixtures/semicolon_with_quote.csv
|
197
|
-
- spec/fixtures/special_null_with_header.csv
|
198
|
-
- spec/fixtures/tab_only_data.csv
|
199
|
-
- spec/fixtures/tab_with_different_header.csv
|
200
|
-
- spec/fixtures/tab_with_error.csv
|
201
|
-
- spec/fixtures/tab_with_extra_line.csv
|
202
|
-
- spec/fixtures/tab_with_header.csv
|
203
|
-
- spec/fixtures/tab_with_two_lines.csv
|
204
|
-
- spec/fixtures/test_extended_model.rb
|
205
|
-
- spec/fixtures/test_model.rb
|
206
|
-
- spec/spec.opts
|
207
|
-
- spec/spec_helper.rb
|
186
|
+
test_files: []
|
data/.travis.yml
DELETED
data/VERSION
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
0.5.6
|