postgres-copy 1.3.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/workflows/ruby.yml +53 -0
- data/.gitignore +4 -0
- data/Gemfile.lock +20 -62
- data/README.md +35 -3
- data/lib/postgres-copy/acts_as_copy_target.rb +41 -13
- data/postgres-copy.gemspec +1 -2
- data/spec/copy_from_spec.rb +33 -4
- data/spec/copy_to_binary_spec.rb +0 -11
- data/spec/copy_to_spec.rb +8 -10
- data/spec/fixtures/comma_with_bom.csv +2 -0
- data/spec/fixtures/comma_with_carriage_returns.csv +3 -0
- data/spec/fixtures/comma_with_empty_string.csv +2 -0
- data/spec/fixtures/tab_with_header.tsv +2 -0
- data/spec/fixtures/tab_with_two_lines.tsv +3 -0
- metadata +13 -53
- data/.travis.yml +0 -7
- data/VERSION +0 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: ff5750f062c23bbbaf42c2c4b54e0d10a2a18c45eda5a66e00abd2fa0d3d9cdc
|
|
4
|
+
data.tar.gz: 10920c6056c38c5cc4a83ba0ace5164abf8a9eb6276e5cef7a4c19e39424de11
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 25c750eb0da06933151e7707f67fa52b9973e603d2afe3288241acf34dc61f7004179483339e630e245660f4371c9f1ddb086e9e8f3a80b32c8bc5d180a79937
|
|
7
|
+
data.tar.gz: 8e537afc3c20801bcac073f208225d8375cc25b14b40ebbe80f904eef79ffded4c00ed672987094a08691d84dbaf1818d71fbf0ec2aeedba8d9d2a611298b68d
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# This workflow uses actions that are not certified by GitHub.
|
|
2
|
+
# They are provided by a third-party and are governed by
|
|
3
|
+
# separate terms of service, privacy policy, and support
|
|
4
|
+
# documentation.
|
|
5
|
+
# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
|
|
6
|
+
# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
|
|
7
|
+
|
|
8
|
+
name: Ruby
|
|
9
|
+
|
|
10
|
+
on:
|
|
11
|
+
push:
|
|
12
|
+
branches: [ master ]
|
|
13
|
+
pull_request:
|
|
14
|
+
branches: [ master ]
|
|
15
|
+
|
|
16
|
+
jobs:
|
|
17
|
+
test:
|
|
18
|
+
services:
|
|
19
|
+
# Label used to access the service container
|
|
20
|
+
postgres:
|
|
21
|
+
# Docker Hub image
|
|
22
|
+
image: postgres
|
|
23
|
+
# Provide the password for postgres
|
|
24
|
+
env:
|
|
25
|
+
POSTGRES_PASSWORD: postgres
|
|
26
|
+
ports:
|
|
27
|
+
- 5432:5432
|
|
28
|
+
# Set health checks to wait until postgres has started
|
|
29
|
+
options: >-
|
|
30
|
+
--health-cmd pg_isready
|
|
31
|
+
--health-interval 10s
|
|
32
|
+
--health-timeout 5s
|
|
33
|
+
--health-retries 5
|
|
34
|
+
|
|
35
|
+
runs-on: ubuntu-latest
|
|
36
|
+
name: test (ruby v${{ matrix.ruby }})
|
|
37
|
+
strategy:
|
|
38
|
+
matrix:
|
|
39
|
+
ruby: ["2.7", "3.0", "3.1"]
|
|
40
|
+
|
|
41
|
+
steps:
|
|
42
|
+
- uses: actions/checkout@v2
|
|
43
|
+
- name: Set up Ruby
|
|
44
|
+
# To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
|
|
45
|
+
# change this to (see https://github.com/ruby/setup-ruby#versioning):
|
|
46
|
+
uses: ruby/setup-ruby@v1
|
|
47
|
+
with:
|
|
48
|
+
ruby-version: ${{ matrix.ruby }}
|
|
49
|
+
bundler-cache: true
|
|
50
|
+
- name: Install dependencies
|
|
51
|
+
run: bundle install
|
|
52
|
+
- name: Run tests
|
|
53
|
+
run: bundle exec rake
|
data/.gitignore
CHANGED
data/Gemfile.lock
CHANGED
|
@@ -1,71 +1,31 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
postgres-copy (1.
|
|
4
|
+
postgres-copy (1.6.0)
|
|
5
5
|
activerecord (>= 5.1)
|
|
6
6
|
pg (>= 0.17)
|
|
7
|
-
responders
|
|
8
7
|
|
|
9
8
|
GEM
|
|
10
9
|
remote: https://rubygems.org/
|
|
11
10
|
specs:
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
rails-html-sanitizer (~> 1.0, >= 1.0.2)
|
|
19
|
-
actionview (5.1.1)
|
|
20
|
-
activesupport (= 5.1.1)
|
|
21
|
-
builder (~> 3.1)
|
|
22
|
-
erubi (~> 1.4)
|
|
23
|
-
rails-dom-testing (~> 2.0)
|
|
24
|
-
rails-html-sanitizer (~> 1.0, >= 1.0.3)
|
|
25
|
-
activemodel (5.1.1)
|
|
26
|
-
activesupport (= 5.1.1)
|
|
27
|
-
activerecord (5.1.1)
|
|
28
|
-
activemodel (= 5.1.1)
|
|
29
|
-
activesupport (= 5.1.1)
|
|
30
|
-
arel (~> 8.0)
|
|
31
|
-
activesupport (5.1.1)
|
|
11
|
+
activemodel (7.0.1)
|
|
12
|
+
activesupport (= 7.0.1)
|
|
13
|
+
activerecord (7.0.1)
|
|
14
|
+
activemodel (= 7.0.1)
|
|
15
|
+
activesupport (= 7.0.1)
|
|
16
|
+
activesupport (7.0.1)
|
|
32
17
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
|
33
|
-
i18n (
|
|
34
|
-
minitest (
|
|
35
|
-
tzinfo (~>
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
loofah (2.0.3)
|
|
43
|
-
nokogiri (>= 1.5.9)
|
|
44
|
-
method_source (0.8.2)
|
|
45
|
-
mini_portile2 (2.2.0)
|
|
46
|
-
minitest (5.10.2)
|
|
47
|
-
nokogiri (1.8.0)
|
|
48
|
-
mini_portile2 (~> 2.2.0)
|
|
49
|
-
pg (0.21.0)
|
|
50
|
-
rack (2.0.3)
|
|
51
|
-
rack-test (0.6.3)
|
|
52
|
-
rack (>= 1.0)
|
|
53
|
-
rails-dom-testing (2.0.3)
|
|
54
|
-
activesupport (>= 4.2.0)
|
|
55
|
-
nokogiri (>= 1.6)
|
|
56
|
-
rails-html-sanitizer (1.0.3)
|
|
57
|
-
loofah (~> 2.0)
|
|
58
|
-
railties (5.1.1)
|
|
59
|
-
actionpack (= 5.1.1)
|
|
60
|
-
activesupport (= 5.1.1)
|
|
61
|
-
method_source
|
|
62
|
-
rake (>= 0.8.7)
|
|
63
|
-
thor (>= 0.18.1, < 2.0)
|
|
18
|
+
i18n (>= 1.6, < 2)
|
|
19
|
+
minitest (>= 5.1)
|
|
20
|
+
tzinfo (~> 2.0)
|
|
21
|
+
concurrent-ruby (1.1.9)
|
|
22
|
+
diff-lcs (1.4.4)
|
|
23
|
+
i18n (1.9.1)
|
|
24
|
+
concurrent-ruby (~> 1.0)
|
|
25
|
+
minitest (5.15.0)
|
|
26
|
+
pg (1.2.3)
|
|
64
27
|
rake (11.2.2)
|
|
65
|
-
rdoc (
|
|
66
|
-
responders (2.4.0)
|
|
67
|
-
actionpack (>= 4.2.0, < 5.3)
|
|
68
|
-
railties (>= 4.2.0, < 5.3)
|
|
28
|
+
rdoc (6.2.1)
|
|
69
29
|
rspec (2.99.0)
|
|
70
30
|
rspec-core (~> 2.99.0)
|
|
71
31
|
rspec-expectations (~> 2.99.0)
|
|
@@ -74,10 +34,8 @@ GEM
|
|
|
74
34
|
rspec-expectations (2.99.2)
|
|
75
35
|
diff-lcs (>= 1.1.3, < 2.0)
|
|
76
36
|
rspec-mocks (2.99.4)
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
tzinfo (1.2.3)
|
|
80
|
-
thread_safe (~> 0.1)
|
|
37
|
+
tzinfo (2.0.4)
|
|
38
|
+
concurrent-ruby (~> 1.0)
|
|
81
39
|
|
|
82
40
|
PLATFORMS
|
|
83
41
|
ruby
|
|
@@ -90,4 +48,4 @@ DEPENDENCIES
|
|
|
90
48
|
rspec (~> 2.12)
|
|
91
49
|
|
|
92
50
|
BUNDLED WITH
|
|
93
|
-
|
|
51
|
+
2.2.22
|
data/README.md
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
# postgres-copy
|
|
1
|
+
# postgres-copy
|
|
2
|
+
|
|
3
|
+

|
|
2
4
|
|
|
3
5
|
This Gem will enable your AR models to use the PostgreSQL COPY command to import/export data in CSV format.
|
|
4
6
|
If you need to tranfer data between a PostgreSQL database and CSV files, the PostgreSQL native CSV parser
|
|
@@ -55,8 +57,8 @@ This will execute in the database the command:
|
|
|
55
57
|
COPY (SELECT "users".* FROM "users" ) TO '/tmp/users.csv' WITH DELIMITER ',' CSV HEADER
|
|
56
58
|
```
|
|
57
59
|
|
|
58
|
-
Remark that the file will be created in the database server disk.
|
|
59
|
-
But what if you want to write the lines in a file on the server that is running Rails, instead of the database?
|
|
60
|
+
Remark that the file will be created in the database server disk.
|
|
61
|
+
But what if you want to write the lines in a file on the server that is running Rails, instead of the database?
|
|
60
62
|
In this case you can pass a block and retrieve the generated lines and then write them to a file:
|
|
61
63
|
|
|
62
64
|
```ruby
|
|
@@ -97,6 +99,18 @@ Which will generate the following SQL command:
|
|
|
97
99
|
COPY (SELECT name FROM "users" WHERE "users"."id" IN (1, 2, 3)) TO '/tmp/users.csv' WITH DELIMITER ',' CSV HEADER
|
|
98
100
|
```
|
|
99
101
|
|
|
102
|
+
Alternatively, you can supply customized raw SQL query to copy_to instead of scoped relation:
|
|
103
|
+
|
|
104
|
+
```ruby
|
|
105
|
+
User.copy_to("/tmp/users.csv", query: 'SELECT count(*) as Total FROM users')
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Which will generate the following SQL command:
|
|
109
|
+
|
|
110
|
+
```sql
|
|
111
|
+
COPY (SELECT count(*) as Total FROM users) TO '/tmp/users.csv' WITH DELIMITER ',' CSV HEADER
|
|
112
|
+
```
|
|
113
|
+
|
|
100
114
|
The COPY command also supports exporting the data in binary format.
|
|
101
115
|
|
|
102
116
|
```ruby
|
|
@@ -154,6 +168,17 @@ To specify NULL value you can pass the null option parameter.
|
|
|
154
168
|
User.copy_from "/tmp/users.csv", :null => 'null'
|
|
155
169
|
```
|
|
156
170
|
|
|
171
|
+
Match the specified columns' values against the null string, even if it has been quoted, and if a match is found set the value to NULL (Postgres 9.4+ only).
|
|
172
|
+
|
|
173
|
+
```ruby
|
|
174
|
+
User.copy_from "/tmp/users.csv", :null => '', :force_null => [:name, :city]
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
To copy from tsv file , you can set format `:tsv`
|
|
178
|
+
|
|
179
|
+
```ruby
|
|
180
|
+
User.copy_from "/tmp/users.tsv", :format => :tsv
|
|
181
|
+
```
|
|
157
182
|
|
|
158
183
|
To copy a binary formatted data file or IO object you can specify the format as binary
|
|
159
184
|
|
|
@@ -175,6 +200,13 @@ Which will generate the following SQL command:
|
|
|
175
200
|
COPY users (id, name) FROM '/tmp/users.dat' WITH BINARY
|
|
176
201
|
```
|
|
177
202
|
|
|
203
|
+
To specify the encoding with which to read the file, set the :encoding option.
|
|
204
|
+
This is useful for removing byte order marks when matching column headers.
|
|
205
|
+
|
|
206
|
+
```ruby
|
|
207
|
+
User.copy_from "/tmp/users_with_byte_order_mark.csv", :encoding => 'bom|utf-8'
|
|
208
|
+
```
|
|
209
|
+
|
|
178
210
|
|
|
179
211
|
### Using the CSV Responder
|
|
180
212
|
If you want to make the result of a COPY command available to download this gem provides a CSV responder that, in conjunction with [inherited_resources](https://github.com/josevalim/inherited_resources), is a very powerfull tool. BTW, do not try to use the responder without inherited_resources.
|
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
require 'csv'
|
|
2
2
|
|
|
3
|
+
def get_file_mode mode, encoding = nil
|
|
4
|
+
if encoding
|
|
5
|
+
"#{mode}:#{encoding}"
|
|
6
|
+
else
|
|
7
|
+
mode
|
|
8
|
+
end
|
|
9
|
+
end
|
|
10
|
+
|
|
3
11
|
module PostgresCopy
|
|
4
12
|
module ActsAsCopyTarget
|
|
5
13
|
extend ActiveSupport::Concern
|
|
@@ -10,18 +18,19 @@ module PostgresCopy
|
|
|
10
18
|
module CopyMethods
|
|
11
19
|
# Copy data to a file passed as a string (the file path) or to lines that are passed to a block
|
|
12
20
|
def copy_to path = nil, options = {}
|
|
13
|
-
options = {:
|
|
21
|
+
options = { delimiter: ",", format: :csv, header: true }.merge(options)
|
|
14
22
|
options_string = if options[:format] == :binary
|
|
15
23
|
"BINARY"
|
|
16
24
|
else
|
|
17
25
|
"DELIMITER '#{options[:delimiter]}' CSV #{options[:header] ? 'HEADER' : ''}"
|
|
18
26
|
end
|
|
27
|
+
options_query = options.delete(:query) || self.all.to_sql
|
|
19
28
|
|
|
20
29
|
if path
|
|
21
30
|
raise "You have to choose between exporting to a file or receiving the lines inside a block" if block_given?
|
|
22
|
-
connection.execute "COPY (#{
|
|
31
|
+
connection.execute "COPY (#{options_query}) TO '#{sanitize_sql(path)}' WITH #{options_string}"
|
|
23
32
|
else
|
|
24
|
-
connection.raw_connection.copy_data "COPY (#{
|
|
33
|
+
connection.raw_connection.copy_data "COPY (#{options_query}) TO STDOUT WITH #{options_string}" do
|
|
25
34
|
while line = connection.raw_connection.get_copy_data do
|
|
26
35
|
yield(line) if block_given?
|
|
27
36
|
end
|
|
@@ -71,15 +80,18 @@ module PostgresCopy
|
|
|
71
80
|
# * You can map fields from the file to different fields in the table using a map in the options hash
|
|
72
81
|
# * For further details on usage take a look at the README.md
|
|
73
82
|
def copy_from path_or_io, options = {}
|
|
74
|
-
options = {:
|
|
83
|
+
options = { delimiter: ",", format: :csv, header: true, quote: '"' }.merge(options)
|
|
84
|
+
options[:delimiter] = "\t" if options[:format] == :tsv
|
|
75
85
|
options_string = if options[:format] == :binary
|
|
76
86
|
"BINARY"
|
|
77
87
|
else
|
|
78
88
|
quote = options[:quote] == "'" ? "''" : options[:quote]
|
|
79
|
-
null = options.key?(:null) ? "NULL '#{options[:null]}'" :
|
|
80
|
-
"
|
|
89
|
+
null = options.key?(:null) ? "NULL '#{options[:null]}'" : nil
|
|
90
|
+
force_null = options.key?(:force_null) ? "FORCE_NULL(#{options[:force_null].join(',')})" : nil
|
|
91
|
+
delimiter = options[:format] == :tsv ? "E'\t'" : "'#{options[:delimiter]}'"
|
|
92
|
+
"WITH (" + ["DELIMITER #{delimiter}", "QUOTE '#{quote}'", null, force_null, "FORMAT CSV"].compact.join(', ') + ")"
|
|
81
93
|
end
|
|
82
|
-
io = path_or_io.instance_of?(String) ? File.open(path_or_io, 'r') : path_or_io
|
|
94
|
+
io = path_or_io.instance_of?(String) ? File.open(path_or_io, get_file_mode('r', options[:encoding])) : path_or_io
|
|
83
95
|
|
|
84
96
|
if options[:format] == :binary
|
|
85
97
|
columns_list = options[:columns] || []
|
|
@@ -109,15 +121,31 @@ module PostgresCopy
|
|
|
109
121
|
rescue EOFError
|
|
110
122
|
end
|
|
111
123
|
else
|
|
124
|
+
line_buffer = ''
|
|
125
|
+
|
|
112
126
|
while line = io.gets do
|
|
113
127
|
next if line.strip.size == 0
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
128
|
+
|
|
129
|
+
line_buffer += line
|
|
130
|
+
|
|
131
|
+
# If line is incomplete, get the next line until it terminates
|
|
132
|
+
if line_buffer =~ /\n$/ || line_buffer =~ /\Z/
|
|
133
|
+
if block_given?
|
|
134
|
+
begin
|
|
135
|
+
row = CSV.parse_line(line_buffer.strip, col_sep: options[:delimiter])
|
|
136
|
+
yield(row)
|
|
137
|
+
next if row.all?(&:nil?)
|
|
138
|
+
line_buffer = CSV.generate_line(row, col_sep: options[:delimiter])
|
|
139
|
+
rescue CSV::MalformedCSVError
|
|
140
|
+
next
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
connection.raw_connection.put_copy_data(line_buffer)
|
|
145
|
+
|
|
146
|
+
# Clear the buffer
|
|
147
|
+
line_buffer = ''
|
|
119
148
|
end
|
|
120
|
-
connection.raw_connection.put_copy_data line
|
|
121
149
|
end
|
|
122
150
|
end
|
|
123
151
|
end
|
data/postgres-copy.gemspec
CHANGED
|
@@ -5,7 +5,7 @@ $:.unshift lib unless $:.include?(lib)
|
|
|
5
5
|
|
|
6
6
|
Gem::Specification.new do |s|
|
|
7
7
|
s.name = "postgres-copy"
|
|
8
|
-
s.version = "1.
|
|
8
|
+
s.version = "1.6.0"
|
|
9
9
|
s.platform = Gem::Platform::RUBY
|
|
10
10
|
s.required_ruby_version = ">= 1.9.3"
|
|
11
11
|
s.authors = ["Diogo Biazus"]
|
|
@@ -22,7 +22,6 @@ Gem::Specification.new do |s|
|
|
|
22
22
|
|
|
23
23
|
s.add_dependency "pg", ">= 0.17"
|
|
24
24
|
s.add_dependency "activerecord", '>= 5.1'
|
|
25
|
-
s.add_dependency "responders"
|
|
26
25
|
s.add_development_dependency "bundler"
|
|
27
26
|
s.add_development_dependency "rdoc"
|
|
28
27
|
s.add_development_dependency "rspec", "~> 2.12"
|
data/spec/copy_from_spec.rb
CHANGED
|
@@ -100,20 +100,20 @@ describe "COPY FROM" do
|
|
|
100
100
|
ReservedWordModel.copy_from File.expand_path('spec/fixtures/reserved_words.csv'), :delimiter => "\t"
|
|
101
101
|
ReservedWordModel.order(:id).map{|r| r.attributes}.should == [{"group"=>"group name", "id"=>1, "select"=>"test select"}]
|
|
102
102
|
end
|
|
103
|
-
|
|
103
|
+
|
|
104
104
|
it "should import even last columns have empty values" do
|
|
105
105
|
TestExtendedModel.copy_from File.expand_path('spec/fixtures/comma_with_header_empty_values_at_the_end.csv')
|
|
106
|
-
TestExtendedModel.order(:id).map{|r| r.attributes}.should ==
|
|
106
|
+
TestExtendedModel.order(:id).map{|r| r.attributes}.should ==
|
|
107
107
|
[{"id"=>1, "data"=>"test data 1", "more_data"=>nil, "other_data"=>nil, "final_data"=>nil},
|
|
108
108
|
{"id"=>2, "data"=>"test data 2", "more_data"=>"9", "other_data"=>nil, "final_data"=>nil},
|
|
109
109
|
{"id"=>3, "data"=>"test data 2", "more_data"=>"9", "other_data"=>nil, "final_data"=>"0"}]
|
|
110
110
|
end
|
|
111
|
-
|
|
111
|
+
|
|
112
112
|
it "should import even last columns have empty values with block" do
|
|
113
113
|
TestExtendedModel.copy_from File.expand_path('spec/fixtures/comma_with_header_empty_values_at_the_end.csv') do |row|
|
|
114
114
|
row[4]="666"
|
|
115
115
|
end
|
|
116
|
-
TestExtendedModel.order(:id).map{|r| r.attributes}.should ==
|
|
116
|
+
TestExtendedModel.order(:id).map{|r| r.attributes}.should ==
|
|
117
117
|
[{"id"=>1, "data"=>"test data 1", "more_data"=>nil, "other_data"=>nil, "final_data"=>"666"},
|
|
118
118
|
{"id"=>2, "data"=>"test data 2", "more_data"=>"9", "other_data"=>nil, "final_data"=>"666"},
|
|
119
119
|
{"id"=>3, "data"=>"test data 2", "more_data"=>"9", "other_data"=>nil, "final_data"=>"666"}]
|
|
@@ -150,4 +150,33 @@ describe "COPY FROM" do
|
|
|
150
150
|
TestModel.copy_from File.open(File.expand_path('spec/fixtures/special_null_with_header.csv'), 'r'), :null => 'NULL'
|
|
151
151
|
TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => nil}]
|
|
152
152
|
end
|
|
153
|
+
|
|
154
|
+
it "should import with a carriage return in the value" do
|
|
155
|
+
TestModel.copy_from File.expand_path('spec/fixtures/comma_with_carriage_returns.csv')
|
|
156
|
+
TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => "test\ndata 1"}]
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
it "should import custom force null expressions from path" do
|
|
160
|
+
TestModel.copy_from File.expand_path('spec/fixtures/comma_with_empty_string.csv'), :null => '', :force_null => [:data]
|
|
161
|
+
TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => nil}]
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
it "should import tsv from path" do
|
|
165
|
+
TestModel.copy_from File.expand_path('spec/fixtures/tab_with_header.tsv'), :format => :tsv
|
|
166
|
+
TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'test data 1'}]
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
it "should import 2 lines from tsv and allow changes in block" do
|
|
170
|
+
TestModel.copy_from(File.open(File.expand_path('spec/fixtures/tab_with_two_lines.tsv'), 'r'), :format => :tsv) do |row|
|
|
171
|
+
row[1] = 'changed this data'
|
|
172
|
+
end
|
|
173
|
+
TestModel.order(:id).first.attributes.should == {'id' => 1, 'data' => 'changed this data'}
|
|
174
|
+
TestModel.count.should == 2
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
it "should import csv headers with BOM when provided encoding option" do
|
|
178
|
+
TestModel.copy_from File.expand_path("spec/fixtures/comma_with_bom.csv"), :encoding => "bom|utf-8"
|
|
179
|
+
TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'test data 1'}]
|
|
180
|
+
end
|
|
181
|
+
|
|
153
182
|
end
|
data/spec/copy_to_binary_spec.rb
CHANGED
|
@@ -19,15 +19,4 @@ describe "COPY TO BINARY" do
|
|
|
19
19
|
it{ should == File.open('spec/fixtures/2_col_binary_data.dat', 'r:ASCII-8BIT').read }
|
|
20
20
|
end
|
|
21
21
|
end
|
|
22
|
-
|
|
23
|
-
describe "should allow binary output to file" do
|
|
24
|
-
it "should copy to disk if block is not given and a path is passed" do
|
|
25
|
-
TestModel.copy_to '/tmp/export.dat', :format => :binary
|
|
26
|
-
str = File.open('/tmp/export.dat', 'r:ASCII-8BIT').read
|
|
27
|
-
|
|
28
|
-
str.should == File.open('spec/fixtures/2_col_binary_data.dat', 'r:ASCII-8BIT').read
|
|
29
|
-
|
|
30
|
-
end
|
|
31
|
-
end
|
|
32
|
-
|
|
33
22
|
end
|
data/spec/copy_to_spec.rb
CHANGED
|
@@ -5,7 +5,7 @@ describe "COPY TO" do
|
|
|
5
5
|
ActiveRecord::Base.connection.execute %{
|
|
6
6
|
TRUNCATE TABLE test_models;
|
|
7
7
|
SELECT setval('test_models_id_seq', 1, false);
|
|
8
|
-
}
|
|
8
|
+
}
|
|
9
9
|
TestModel.create :data => 'test data 1'
|
|
10
10
|
end
|
|
11
11
|
|
|
@@ -64,20 +64,18 @@ describe "COPY TO" do
|
|
|
64
64
|
end
|
|
65
65
|
end
|
|
66
66
|
|
|
67
|
-
it "should copy to disk if block is not given and a path is passed" do
|
|
68
|
-
TestModel.copy_to '/tmp/export.csv'
|
|
69
|
-
File.open('spec/fixtures/comma_with_header.csv', 'r') do |fixture|
|
|
70
|
-
File.open('/tmp/export.csv', 'r') do |result|
|
|
71
|
-
result.read.should == fixture.read
|
|
72
|
-
end
|
|
73
|
-
end
|
|
74
|
-
end
|
|
75
|
-
|
|
76
67
|
it "should raise exception if I pass a path and a block simultaneously" do
|
|
77
68
|
lambda do
|
|
78
69
|
TestModel.copy_to('/tmp/bogus_path') do |row|
|
|
79
70
|
end
|
|
80
71
|
end.should raise_error
|
|
81
72
|
end
|
|
73
|
+
|
|
74
|
+
it "accepts custom sql query to run instead on the current relation" do
|
|
75
|
+
TestModel.copy_to(nil, query: 'SELECT count(*) as "Total" FROM test_models') do |row|
|
|
76
|
+
expect(row).to eq("Total\n")
|
|
77
|
+
break
|
|
78
|
+
end
|
|
79
|
+
end
|
|
82
80
|
end
|
|
83
81
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: postgres-copy
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.6.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Diogo Biazus
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2022-02-07 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: pg
|
|
@@ -38,20 +38,6 @@ dependencies:
|
|
|
38
38
|
- - ">="
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
40
|
version: '5.1'
|
|
41
|
-
- !ruby/object:Gem::Dependency
|
|
42
|
-
name: responders
|
|
43
|
-
requirement: !ruby/object:Gem::Requirement
|
|
44
|
-
requirements:
|
|
45
|
-
- - ">="
|
|
46
|
-
- !ruby/object:Gem::Version
|
|
47
|
-
version: '0'
|
|
48
|
-
type: :runtime
|
|
49
|
-
prerelease: false
|
|
50
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
-
requirements:
|
|
52
|
-
- - ">="
|
|
53
|
-
- !ruby/object:Gem::Version
|
|
54
|
-
version: '0'
|
|
55
41
|
- !ruby/object:Gem::Dependency
|
|
56
42
|
name: bundler
|
|
57
43
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -116,15 +102,14 @@ extensions: []
|
|
|
116
102
|
extra_rdoc_files: []
|
|
117
103
|
files:
|
|
118
104
|
- ".document"
|
|
105
|
+
- ".github/workflows/ruby.yml"
|
|
119
106
|
- ".gitignore"
|
|
120
107
|
- ".rspec"
|
|
121
|
-
- ".travis.yml"
|
|
122
108
|
- Gemfile
|
|
123
109
|
- Gemfile.lock
|
|
124
110
|
- LICENSE
|
|
125
111
|
- README.md
|
|
126
112
|
- Rakefile
|
|
127
|
-
- VERSION
|
|
128
113
|
- lib/postgres-copy.rb
|
|
129
114
|
- lib/postgres-copy/acts_as_copy_target.rb
|
|
130
115
|
- lib/postgres-copy/csv_responder.rb
|
|
@@ -135,6 +120,9 @@ files:
|
|
|
135
120
|
- spec/copy_to_spec.rb
|
|
136
121
|
- spec/fixtures/2_col_binary_data.dat
|
|
137
122
|
- spec/fixtures/comma_inside_field.csv
|
|
123
|
+
- spec/fixtures/comma_with_bom.csv
|
|
124
|
+
- spec/fixtures/comma_with_carriage_returns.csv
|
|
125
|
+
- spec/fixtures/comma_with_empty_string.csv
|
|
138
126
|
- spec/fixtures/comma_with_header.csv
|
|
139
127
|
- spec/fixtures/comma_with_header_and_scope.csv
|
|
140
128
|
- spec/fixtures/comma_with_header_empty_values_at_the_end.csv
|
|
@@ -152,8 +140,10 @@ files:
|
|
|
152
140
|
- spec/fixtures/tab_with_error.csv
|
|
153
141
|
- spec/fixtures/tab_with_extra_line.csv
|
|
154
142
|
- spec/fixtures/tab_with_header.csv
|
|
143
|
+
- spec/fixtures/tab_with_header.tsv
|
|
155
144
|
- spec/fixtures/tab_with_header_multi.csv
|
|
156
145
|
- spec/fixtures/tab_with_two_lines.csv
|
|
146
|
+
- spec/fixtures/tab_with_two_lines.tsv
|
|
157
147
|
- spec/fixtures/test_extended_model.rb
|
|
158
148
|
- spec/fixtures/test_model.rb
|
|
159
149
|
- spec/spec.opts
|
|
@@ -161,7 +151,7 @@ files:
|
|
|
161
151
|
homepage: http://github.com/diogob/postgres-copy
|
|
162
152
|
licenses: []
|
|
163
153
|
metadata: {}
|
|
164
|
-
post_install_message:
|
|
154
|
+
post_install_message:
|
|
165
155
|
rdoc_options: []
|
|
166
156
|
require_paths:
|
|
167
157
|
- lib
|
|
@@ -176,38 +166,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
176
166
|
- !ruby/object:Gem::Version
|
|
177
167
|
version: '0'
|
|
178
168
|
requirements: []
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
signing_key:
|
|
169
|
+
rubygems_version: 3.2.3
|
|
170
|
+
signing_key:
|
|
182
171
|
specification_version: 4
|
|
183
172
|
summary: Put COPY command functionality in ActiveRecord's model class
|
|
184
|
-
test_files:
|
|
185
|
-
- spec/copy_from_binary_spec.rb
|
|
186
|
-
- spec/copy_from_spec.rb
|
|
187
|
-
- spec/copy_to_binary_spec.rb
|
|
188
|
-
- spec/copy_to_spec.rb
|
|
189
|
-
- spec/fixtures/2_col_binary_data.dat
|
|
190
|
-
- spec/fixtures/comma_inside_field.csv
|
|
191
|
-
- spec/fixtures/comma_with_header.csv
|
|
192
|
-
- spec/fixtures/comma_with_header_and_scope.csv
|
|
193
|
-
- spec/fixtures/comma_with_header_empty_values_at_the_end.csv
|
|
194
|
-
- spec/fixtures/comma_with_header_multi.csv
|
|
195
|
-
- spec/fixtures/comma_without_header.csv
|
|
196
|
-
- spec/fixtures/extra_field.rb
|
|
197
|
-
- spec/fixtures/reserved_word_model.rb
|
|
198
|
-
- spec/fixtures/reserved_words.csv
|
|
199
|
-
- spec/fixtures/semicolon_with_different_header.csv
|
|
200
|
-
- spec/fixtures/semicolon_with_header.csv
|
|
201
|
-
- spec/fixtures/semicolon_with_quote.csv
|
|
202
|
-
- spec/fixtures/special_null_with_header.csv
|
|
203
|
-
- spec/fixtures/tab_only_data.csv
|
|
204
|
-
- spec/fixtures/tab_with_different_header.csv
|
|
205
|
-
- spec/fixtures/tab_with_error.csv
|
|
206
|
-
- spec/fixtures/tab_with_extra_line.csv
|
|
207
|
-
- spec/fixtures/tab_with_header.csv
|
|
208
|
-
- spec/fixtures/tab_with_header_multi.csv
|
|
209
|
-
- spec/fixtures/tab_with_two_lines.csv
|
|
210
|
-
- spec/fixtures/test_extended_model.rb
|
|
211
|
-
- spec/fixtures/test_model.rb
|
|
212
|
-
- spec/spec.opts
|
|
213
|
-
- spec/spec_helper.rb
|
|
173
|
+
test_files: []
|
data/.travis.yml
DELETED
data/VERSION
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
0.5.6
|