postgres-copy 1.5.0 → 1.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +6 -1
- data/Gemfile.lock +16 -61
- data/README.md +7 -0
- data/lib/postgres-copy/acts_as_copy_target.rb +14 -6
- data/postgres-copy.gemspec +1 -2
- data/spec/copy_from_spec.rb +6 -0
- data/spec/fixtures/comma_with_bom.csv +2 -0
- metadata +7 -20
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ff5750f062c23bbbaf42c2c4b54e0d10a2a18c45eda5a66e00abd2fa0d3d9cdc
|
4
|
+
data.tar.gz: 10920c6056c38c5cc4a83ba0ace5164abf8a9eb6276e5cef7a4c19e39424de11
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 25c750eb0da06933151e7707f67fa52b9973e603d2afe3288241acf34dc61f7004179483339e630e245660f4371c9f1ddb086e9e8f3a80b32c8bc5d180a79937
|
7
|
+
data.tar.gz: 8e537afc3c20801bcac073f208225d8375cc25b14b40ebbe80f904eef79ffded4c00ed672987094a08691d84dbaf1818d71fbf0ec2aeedba8d9d2a611298b68d
|
data/.github/workflows/ruby.yml
CHANGED
@@ -33,6 +33,10 @@ jobs:
|
|
33
33
|
--health-retries 5
|
34
34
|
|
35
35
|
runs-on: ubuntu-latest
|
36
|
+
name: test (ruby v${{ matrix.ruby }})
|
37
|
+
strategy:
|
38
|
+
matrix:
|
39
|
+
ruby: ["2.7", "3.0", "3.1"]
|
36
40
|
|
37
41
|
steps:
|
38
42
|
- uses: actions/checkout@v2
|
@@ -41,7 +45,8 @@ jobs:
|
|
41
45
|
# change this to (see https://github.com/ruby/setup-ruby#versioning):
|
42
46
|
uses: ruby/setup-ruby@v1
|
43
47
|
with:
|
44
|
-
ruby-version:
|
48
|
+
ruby-version: ${{ matrix.ruby }}
|
49
|
+
bundler-cache: true
|
45
50
|
- name: Install dependencies
|
46
51
|
run: bundle install
|
47
52
|
- name: Run tests
|
data/Gemfile.lock
CHANGED
@@ -1,73 +1,31 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
postgres-copy (1.
|
4
|
+
postgres-copy (1.6.0)
|
5
5
|
activerecord (>= 5.1)
|
6
6
|
pg (>= 0.17)
|
7
|
-
responders
|
8
7
|
|
9
8
|
GEM
|
10
9
|
remote: https://rubygems.org/
|
11
10
|
specs:
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
rails-html-sanitizer (~> 1.0, >= 1.2.0)
|
19
|
-
actionview (6.0.3.2)
|
20
|
-
activesupport (= 6.0.3.2)
|
21
|
-
builder (~> 3.1)
|
22
|
-
erubi (~> 1.4)
|
23
|
-
rails-dom-testing (~> 2.0)
|
24
|
-
rails-html-sanitizer (~> 1.1, >= 1.2.0)
|
25
|
-
activemodel (6.0.3.2)
|
26
|
-
activesupport (= 6.0.3.2)
|
27
|
-
activerecord (6.0.3.2)
|
28
|
-
activemodel (= 6.0.3.2)
|
29
|
-
activesupport (= 6.0.3.2)
|
30
|
-
activesupport (6.0.3.2)
|
11
|
+
activemodel (7.0.1)
|
12
|
+
activesupport (= 7.0.1)
|
13
|
+
activerecord (7.0.1)
|
14
|
+
activemodel (= 7.0.1)
|
15
|
+
activesupport (= 7.0.1)
|
16
|
+
activesupport (7.0.1)
|
31
17
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
32
|
-
i18n (>=
|
33
|
-
minitest (
|
34
|
-
tzinfo (~>
|
35
|
-
|
36
|
-
builder (3.2.4)
|
37
|
-
concurrent-ruby (1.1.6)
|
38
|
-
crass (1.0.6)
|
18
|
+
i18n (>= 1.6, < 2)
|
19
|
+
minitest (>= 5.1)
|
20
|
+
tzinfo (~> 2.0)
|
21
|
+
concurrent-ruby (1.1.9)
|
39
22
|
diff-lcs (1.4.4)
|
40
|
-
|
41
|
-
i18n (1.8.3)
|
23
|
+
i18n (1.9.1)
|
42
24
|
concurrent-ruby (~> 1.0)
|
43
|
-
|
44
|
-
crass (~> 1.0.2)
|
45
|
-
nokogiri (>= 1.5.9)
|
46
|
-
method_source (1.0.0)
|
47
|
-
mini_portile2 (2.4.0)
|
48
|
-
minitest (5.14.1)
|
49
|
-
nokogiri (1.10.10)
|
50
|
-
mini_portile2 (~> 2.4.0)
|
25
|
+
minitest (5.15.0)
|
51
26
|
pg (1.2.3)
|
52
|
-
rack (2.2.3)
|
53
|
-
rack-test (1.1.0)
|
54
|
-
rack (>= 1.0, < 3)
|
55
|
-
rails-dom-testing (2.0.3)
|
56
|
-
activesupport (>= 4.2.0)
|
57
|
-
nokogiri (>= 1.6)
|
58
|
-
rails-html-sanitizer (1.3.0)
|
59
|
-
loofah (~> 2.3)
|
60
|
-
railties (6.0.3.2)
|
61
|
-
actionpack (= 6.0.3.2)
|
62
|
-
activesupport (= 6.0.3.2)
|
63
|
-
method_source
|
64
|
-
rake (>= 0.8.7)
|
65
|
-
thor (>= 0.20.3, < 2.0)
|
66
27
|
rake (11.2.2)
|
67
28
|
rdoc (6.2.1)
|
68
|
-
responders (3.0.1)
|
69
|
-
actionpack (>= 5.0)
|
70
|
-
railties (>= 5.0)
|
71
29
|
rspec (2.99.0)
|
72
30
|
rspec-core (~> 2.99.0)
|
73
31
|
rspec-expectations (~> 2.99.0)
|
@@ -76,11 +34,8 @@ GEM
|
|
76
34
|
rspec-expectations (2.99.2)
|
77
35
|
diff-lcs (>= 1.1.3, < 2.0)
|
78
36
|
rspec-mocks (2.99.4)
|
79
|
-
|
80
|
-
|
81
|
-
tzinfo (1.2.7)
|
82
|
-
thread_safe (~> 0.1)
|
83
|
-
zeitwerk (2.4.0)
|
37
|
+
tzinfo (2.0.4)
|
38
|
+
concurrent-ruby (~> 1.0)
|
84
39
|
|
85
40
|
PLATFORMS
|
86
41
|
ruby
|
@@ -93,4 +48,4 @@ DEPENDENCIES
|
|
93
48
|
rspec (~> 2.12)
|
94
49
|
|
95
50
|
BUNDLED WITH
|
96
|
-
2.
|
51
|
+
2.2.22
|
data/README.md
CHANGED
@@ -200,6 +200,13 @@ Which will generate the following SQL command:
|
|
200
200
|
COPY users (id, name) FROM '/tmp/users.dat' WITH BINARY
|
201
201
|
```
|
202
202
|
|
203
|
+
To specify the encoding with which to read the file, set the :encoding option.
|
204
|
+
This is useful for removing byte order marks when matching column headers.
|
205
|
+
|
206
|
+
```ruby
|
207
|
+
User.copy_from "/tmp/users_with_byte_order_mark.csv", :encoding => 'bom|utf-8'
|
208
|
+
```
|
209
|
+
|
203
210
|
|
204
211
|
### Using the CSV Responder
|
205
212
|
If you want to make the result of a COPY command available to download this gem provides a CSV responder that, in conjunction with [inherited_resources](https://github.com/josevalim/inherited_resources), is a very powerfull tool. BTW, do not try to use the responder without inherited_resources.
|
@@ -1,5 +1,13 @@
|
|
1
1
|
require 'csv'
|
2
2
|
|
3
|
+
def get_file_mode mode, encoding = nil
|
4
|
+
if encoding
|
5
|
+
"#{mode}:#{encoding}"
|
6
|
+
else
|
7
|
+
mode
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
3
11
|
module PostgresCopy
|
4
12
|
module ActsAsCopyTarget
|
5
13
|
extend ActiveSupport::Concern
|
@@ -10,7 +18,7 @@ module PostgresCopy
|
|
10
18
|
module CopyMethods
|
11
19
|
# Copy data to a file passed as a string (the file path) or to lines that are passed to a block
|
12
20
|
def copy_to path = nil, options = {}
|
13
|
-
options = {:
|
21
|
+
options = { delimiter: ",", format: :csv, header: true }.merge(options)
|
14
22
|
options_string = if options[:format] == :binary
|
15
23
|
"BINARY"
|
16
24
|
else
|
@@ -72,7 +80,7 @@ module PostgresCopy
|
|
72
80
|
# * You can map fields from the file to different fields in the table using a map in the options hash
|
73
81
|
# * For further details on usage take a look at the README.md
|
74
82
|
def copy_from path_or_io, options = {}
|
75
|
-
options = {:
|
83
|
+
options = { delimiter: ",", format: :csv, header: true, quote: '"' }.merge(options)
|
76
84
|
options[:delimiter] = "\t" if options[:format] == :tsv
|
77
85
|
options_string = if options[:format] == :binary
|
78
86
|
"BINARY"
|
@@ -83,7 +91,7 @@ module PostgresCopy
|
|
83
91
|
delimiter = options[:format] == :tsv ? "E'\t'" : "'#{options[:delimiter]}'"
|
84
92
|
"WITH (" + ["DELIMITER #{delimiter}", "QUOTE '#{quote}'", null, force_null, "FORMAT CSV"].compact.join(', ') + ")"
|
85
93
|
end
|
86
|
-
io = path_or_io.instance_of?(String) ? File.open(path_or_io, 'r') : path_or_io
|
94
|
+
io = path_or_io.instance_of?(String) ? File.open(path_or_io, get_file_mode('r', options[:encoding])) : path_or_io
|
87
95
|
|
88
96
|
if options[:format] == :binary
|
89
97
|
columns_list = options[:columns] || []
|
@@ -124,10 +132,10 @@ module PostgresCopy
|
|
124
132
|
if line_buffer =~ /\n$/ || line_buffer =~ /\Z/
|
125
133
|
if block_given?
|
126
134
|
begin
|
127
|
-
row = CSV.parse_line(line_buffer.strip,
|
135
|
+
row = CSV.parse_line(line_buffer.strip, col_sep: options[:delimiter])
|
128
136
|
yield(row)
|
129
|
-
next if row.all?
|
130
|
-
line_buffer = CSV.generate_line(row,
|
137
|
+
next if row.all?(&:nil?)
|
138
|
+
line_buffer = CSV.generate_line(row, col_sep: options[:delimiter])
|
131
139
|
rescue CSV::MalformedCSVError
|
132
140
|
next
|
133
141
|
end
|
data/postgres-copy.gemspec
CHANGED
@@ -5,7 +5,7 @@ $:.unshift lib unless $:.include?(lib)
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "postgres-copy"
|
8
|
-
s.version = "1.
|
8
|
+
s.version = "1.6.0"
|
9
9
|
s.platform = Gem::Platform::RUBY
|
10
10
|
s.required_ruby_version = ">= 1.9.3"
|
11
11
|
s.authors = ["Diogo Biazus"]
|
@@ -22,7 +22,6 @@ Gem::Specification.new do |s|
|
|
22
22
|
|
23
23
|
s.add_dependency "pg", ">= 0.17"
|
24
24
|
s.add_dependency "activerecord", '>= 5.1'
|
25
|
-
s.add_dependency "responders"
|
26
25
|
s.add_development_dependency "bundler"
|
27
26
|
s.add_development_dependency "rdoc"
|
28
27
|
s.add_development_dependency "rspec", "~> 2.12"
|
data/spec/copy_from_spec.rb
CHANGED
@@ -173,4 +173,10 @@ describe "COPY FROM" do
|
|
173
173
|
TestModel.order(:id).first.attributes.should == {'id' => 1, 'data' => 'changed this data'}
|
174
174
|
TestModel.count.should == 2
|
175
175
|
end
|
176
|
+
|
177
|
+
it "should import csv headers with BOM when provided encoding option" do
|
178
|
+
TestModel.copy_from File.expand_path("spec/fixtures/comma_with_bom.csv"), :encoding => "bom|utf-8"
|
179
|
+
TestModel.order(:id).map{|r| r.attributes}.should == [{'id' => 1, 'data' => 'test data 1'}]
|
180
|
+
end
|
181
|
+
|
176
182
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: postgres-copy
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Diogo Biazus
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-02-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: pg
|
@@ -38,20 +38,6 @@ dependencies:
|
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '5.1'
|
41
|
-
- !ruby/object:Gem::Dependency
|
42
|
-
name: responders
|
43
|
-
requirement: !ruby/object:Gem::Requirement
|
44
|
-
requirements:
|
45
|
-
- - ">="
|
46
|
-
- !ruby/object:Gem::Version
|
47
|
-
version: '0'
|
48
|
-
type: :runtime
|
49
|
-
prerelease: false
|
50
|
-
version_requirements: !ruby/object:Gem::Requirement
|
51
|
-
requirements:
|
52
|
-
- - ">="
|
53
|
-
- !ruby/object:Gem::Version
|
54
|
-
version: '0'
|
55
41
|
- !ruby/object:Gem::Dependency
|
56
42
|
name: bundler
|
57
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -134,6 +120,7 @@ files:
|
|
134
120
|
- spec/copy_to_spec.rb
|
135
121
|
- spec/fixtures/2_col_binary_data.dat
|
136
122
|
- spec/fixtures/comma_inside_field.csv
|
123
|
+
- spec/fixtures/comma_with_bom.csv
|
137
124
|
- spec/fixtures/comma_with_carriage_returns.csv
|
138
125
|
- spec/fixtures/comma_with_empty_string.csv
|
139
126
|
- spec/fixtures/comma_with_header.csv
|
@@ -164,7 +151,7 @@ files:
|
|
164
151
|
homepage: http://github.com/diogob/postgres-copy
|
165
152
|
licenses: []
|
166
153
|
metadata: {}
|
167
|
-
post_install_message:
|
154
|
+
post_install_message:
|
168
155
|
rdoc_options: []
|
169
156
|
require_paths:
|
170
157
|
- lib
|
@@ -179,8 +166,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
179
166
|
- !ruby/object:Gem::Version
|
180
167
|
version: '0'
|
181
168
|
requirements: []
|
182
|
-
rubygems_version: 3.
|
183
|
-
signing_key:
|
169
|
+
rubygems_version: 3.2.3
|
170
|
+
signing_key:
|
184
171
|
specification_version: 4
|
185
172
|
summary: Put COPY command functionality in ActiveRecord's model class
|
186
173
|
test_files: []
|