pg_csv 0.2 → 0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/README.md +14 -14
- data/lib/pg_csv/version.rb +1 -1
- data/pg_csv.gemspec +3 -2
- data/spec/pg_csv_spec.rb +69 -66
- data/spec/spec_support.rb +10 -5
- metadata +26 -19
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 88e1d29a5270587b750a05a8c3b620907988d0b4e8942ddc0dec4f8f14423d7e
|
4
|
+
data.tar.gz: 66d1d47569da691d87fba379cf223a10a8c405d20663375a75dc6fdb9068ffd2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 575ef7200517a797d4b625c7dddc7e704ffa8719d1c7e6b4f19bbf0807204ea789317892bdd46ec07a6e698e86afb4282c2c5cfce60d0c1117527f8e1e3c22d3
|
7
|
+
data.tar.gz: f392c817d176ddc268c82b0f9f4aad2642ea60b93437602732a143526e36477837f12f3a879c663ef60ee24d0e0953c2b974036e973069a69a4ac61e03cb3457
|
data/README.md
CHANGED
@@ -38,28 +38,28 @@ Options:
|
|
38
38
|
|
39
39
|
Examples:
|
40
40
|
``` ruby
|
41
|
-
PgCsv.new(:
|
42
|
-
PgCsv.new(:
|
43
|
-
PgCsv.new(:
|
44
|
-
PgCsv.new(:
|
45
|
-
File.open("a4.csv", 'a'){|f| PgCsv.new(:
|
46
|
-
export(f, :
|
47
|
-
PgCsv.new(:
|
48
|
-
PgCsv.new(:
|
49
|
-
PgCsv.new(:
|
50
|
-
PgCsv.new(:
|
51
|
-
export('a8.gz', :
|
41
|
+
PgCsv.new(sql: sql).export('a1.csv')
|
42
|
+
PgCsv.new(sql: sql).export('a2.gz', type: :gzip)
|
43
|
+
PgCsv.new(sql: sql).export('a3.csv', temp_file: true)
|
44
|
+
PgCsv.new(sql: sql, type: :plain).export
|
45
|
+
File.open("a4.csv", 'a'){ |f| PgCsv.new(sql: "select * from users").\
|
46
|
+
export(f, type: :stream) }
|
47
|
+
PgCsv.new(sql: sql).export('a5.csv', delimiter: "\t")
|
48
|
+
PgCsv.new(sql: sql).export('a6.csv', header: true)
|
49
|
+
PgCsv.new(sql: sql).export('a7.csv', columns: %w(id a b c))
|
50
|
+
PgCsv.new(sql: sql, connection: SomeDb.connection, columns: %w(id a b c), delimiter: "|").\
|
51
|
+
export('a8.gz', type: :gzip, temp_file: true)
|
52
52
|
|
53
53
|
# example collect from shards
|
54
54
|
Zlib::GzipWriter.open('some.gz') do |stream|
|
55
|
-
e = PgCsv.new(:
|
55
|
+
e = PgCsv.new(sql: sql, type: :stream)
|
56
56
|
ConnectionPool.each_shard do |connection|
|
57
|
-
e.export(stream, :
|
57
|
+
e.export(stream, connection: connection)
|
58
58
|
end
|
59
59
|
end
|
60
60
|
|
61
61
|
# yield example
|
62
|
-
PgCsv.new(:
|
62
|
+
PgCsv.new(sql: sql, type: :yield).export do |row|
|
63
63
|
puts row
|
64
64
|
end
|
65
65
|
```
|
data/lib/pg_csv/version.rb
CHANGED
data/pg_csv.gemspec
CHANGED
@@ -19,8 +19,9 @@ Gem::Specification.new do |s|
|
|
19
19
|
s.require_paths = ["lib"]
|
20
20
|
s.license = "MIT"
|
21
21
|
|
22
|
-
s.add_dependency "pg"
|
23
|
-
s.add_development_dependency "rspec"
|
22
|
+
s.add_dependency "pg"
|
23
|
+
s.add_development_dependency "rspec"
|
24
24
|
s.add_development_dependency "rake"
|
25
25
|
s.add_development_dependency "activerecord"
|
26
|
+
s.add_development_dependency "byebug"
|
26
27
|
end
|
data/spec/pg_csv_spec.rb
CHANGED
@@ -1,12 +1,13 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
+
|
2
3
|
require File.dirname(__FILE__) + '/spec_helper'
|
3
4
|
|
4
5
|
describe PgCsv do
|
5
6
|
|
6
7
|
before :each do
|
7
8
|
Test.delete_all
|
8
|
-
Test.create :
|
9
|
-
Test.create :
|
9
|
+
Test.create a: 1, b: 2, c: 3
|
10
|
+
Test.create a: 4, b: 5, c: 6
|
10
11
|
|
11
12
|
@name = tmp_dir + "1.csv"
|
12
13
|
FileUtils.rm(@name) rescue nil
|
@@ -22,171 +23,173 @@ describe PgCsv do
|
|
22
23
|
describe "simple export" do
|
23
24
|
|
24
25
|
it "1" do
|
25
|
-
PgCsv.new(:
|
26
|
-
with_file(@name){|d| d.
|
26
|
+
PgCsv.new(sql: @sql0).export(@name)
|
27
|
+
with_file(@name){ |d| expect(d).to eq("1,2,3\n4,5,6\n")}
|
27
28
|
end
|
28
29
|
|
29
30
|
it "2" do
|
30
|
-
PgCsv.new(:
|
31
|
-
with_file(@name){|d| d.
|
31
|
+
PgCsv.new(sql: @sql).export(@name)
|
32
|
+
with_file(@name){ |d| expect(d).to eq("4,5,6\n1,2,3\n") }
|
32
33
|
end
|
33
34
|
|
34
35
|
it "delimiter" do
|
35
|
-
PgCsv.new(:
|
36
|
-
with_file(@name){|d| d.
|
36
|
+
PgCsv.new(sql: @sql).export(@name, delimiter: "|")
|
37
|
+
with_file(@name){ |d| expect(d).to eq("4|5|6\n1|2|3\n") }
|
37
38
|
end
|
38
39
|
|
39
40
|
it "encoding" do
|
40
|
-
Test.create!
|
41
|
+
Test.create!(a: 2, b: 3, c: 4, d: "абсд")
|
41
42
|
|
42
|
-
PgCsv.new(:
|
43
|
-
with_file(@name){|d| d.force_encoding('cp1251').
|
43
|
+
PgCsv.new(sql: "select d from tests where a = 2").export(@name, encoding: "WIN1251")
|
44
|
+
with_file(@name){ |d| expect(d.force_encoding('cp1251')).to eq("абсд\n".encode('cp1251')) }
|
44
45
|
end
|
45
46
|
|
46
47
|
describe "headers" do
|
47
48
|
it "header" do
|
48
|
-
PgCsv.new(:
|
49
|
-
with_file(@name){|d| d.
|
49
|
+
PgCsv.new(sql: @sql).export(@name, header: true)
|
50
|
+
with_file(@name){ |d| expect(d).to eq("a,b,c\n4,5,6\n1,2,3\n") }
|
50
51
|
end
|
51
52
|
|
52
53
|
it "columns" do
|
53
|
-
PgCsv.new(:
|
54
|
-
with_file(@name){|d| d.
|
54
|
+
PgCsv.new(sql: @sql).export(@name, columns: %w(q w e))
|
55
|
+
with_file(@name){ |d| expect(d).to eq("q,w,e\n4,5,6\n1,2,3\n") }
|
55
56
|
end
|
56
57
|
|
57
58
|
it "columns with header" do
|
58
|
-
PgCsv.new(:
|
59
|
+
PgCsv.new(sql: @sql).export(@name, header: true, columns: %w(q w e))
|
59
60
|
|
60
61
|
with_file(@name) do |d|
|
61
|
-
d.
|
62
|
+
expect(d).to eq("q,w,e\n4,5,6\n1,2,3\n")
|
62
63
|
end
|
63
64
|
end
|
64
65
|
end
|
65
66
|
|
66
67
|
describe "force_quote" do
|
67
68
|
it "force_quote" do
|
68
|
-
PgCsv.new(:
|
69
|
-
with_file(@name){|d| d.
|
69
|
+
PgCsv.new(sql: @sql).export(@name, force_quote: true)
|
70
|
+
with_file(@name){ |d| expect(d).to eq("\"4\",\"5\",\"6\"\n\"1\",\"2\",\"3\"\n") }
|
70
71
|
end
|
71
72
|
|
72
73
|
it "with headers" do
|
73
|
-
PgCsv.new(:
|
74
|
-
with_file(@name){|d| d.
|
74
|
+
PgCsv.new(sql: @sql).export(@name, header: true, force_quote: true)
|
75
|
+
with_file(@name){ |d| expect(d).to eq("a,b,c\n\"4\",\"5\",\"6\"\n\"1\",\"2\",\"3\"\n") }
|
75
76
|
end
|
76
77
|
end
|
77
|
-
|
78
78
|
end
|
79
79
|
|
80
80
|
describe "moving options no matter" do
|
81
81
|
it "1" do
|
82
|
-
PgCsv.new(:
|
83
|
-
with_file(@name){|d| d.
|
82
|
+
PgCsv.new(sql: @sql).export(@name, delimiter: "|")
|
83
|
+
with_file(@name){ |d| expect(d).to eq("4|5|6\n1|2|3\n") }
|
84
84
|
end
|
85
85
|
|
86
86
|
it "2" do
|
87
|
-
PgCsv.new(:
|
88
|
-
with_file(@name){|d| d.
|
87
|
+
PgCsv.new(delimiter: "|").export(@name, sql: @sql)
|
88
|
+
with_file(@name){ |d| expect(d).to eq("4|5|6\n1|2|3\n") }
|
89
89
|
end
|
90
90
|
end
|
91
91
|
|
92
92
|
describe "local options dont recover global" do
|
93
93
|
it "test" do
|
94
|
-
e = PgCsv.new(:
|
95
|
-
e.export(@name, :
|
96
|
-
with_file(@name){|d| d.
|
94
|
+
e = PgCsv.new(sql: @sql, delimiter: "*")
|
95
|
+
e.export(@name, delimiter: "|")
|
96
|
+
with_file(@name){ |d| expect(d).to eq("4|5|6\n1|2|3\n") }
|
97
97
|
|
98
98
|
e.export(@name)
|
99
|
-
with_file(@name){|d| d.
|
99
|
+
with_file(@name){ |d| expect(d).to eq("4*5*6\n1*2*3\n") }
|
100
100
|
end
|
101
101
|
end
|
102
102
|
|
103
103
|
describe "using temp file" do
|
104
104
|
it "at least file should return to target and set correct chmod" do
|
105
|
-
File.
|
106
|
-
PgCsv.new(:
|
107
|
-
with_file(@name){|d| d.
|
108
|
-
sprintf("%o", File.stat(@name).mode).to_i.
|
105
|
+
expect(File).not_to exist(@name)
|
106
|
+
PgCsv.new(sql: @sql, temp_file: true, temp_dir: tmp_dir).export(@name)
|
107
|
+
with_file(@name){ |d| expect(d).to eq("4,5,6\n1,2,3\n") }
|
108
|
+
expect(sprintf("%o", File.stat(@name).mode).to_i).to eq(100644)
|
109
109
|
end
|
110
110
|
|
111
111
|
it "same with gzip" do
|
112
|
-
File.
|
113
|
-
PgCsv.new(:
|
114
|
-
with_gzfile(@name){|d| d.
|
115
|
-
sprintf("%o", File.stat(@name).mode).to_i.
|
112
|
+
expect(File).not_to exist(@name)
|
113
|
+
PgCsv.new(sql: @sql, temp_file: true, temp_dir: tmp_dir, type: :gzip).export(@name)
|
114
|
+
with_gzfile(@name){ |d| expect(d).to eq("4,5,6\n1,2,3\n") }
|
115
|
+
expect(sprintf("%o", File.stat(@name).mode).to_i).to eq(100644)
|
116
116
|
end
|
117
117
|
end
|
118
118
|
|
119
119
|
describe "different types of export" do
|
120
120
|
it "gzip export" do
|
121
|
-
File.
|
122
|
-
PgCsv.new(:
|
123
|
-
with_gzfile(@name){|d| d.
|
124
|
-
sprintf("%o", File.stat(@name).mode).to_i.
|
121
|
+
expect(File).not_to exist(@name)
|
122
|
+
PgCsv.new(sql: @sql, type: :gzip).export(@name)
|
123
|
+
with_gzfile(@name){ |d| expect(d).to eq("4,5,6\n1,2,3\n") }
|
124
|
+
expect(sprintf("%o", File.stat(@name).mode).to_i).to eq(100644)
|
125
125
|
end
|
126
126
|
|
127
127
|
it "plain export" do
|
128
|
-
PgCsv.new(:
|
128
|
+
expect(PgCsv.new(sql: @sql, type: :plain).export).to eq("4,5,6\n1,2,3\n")
|
129
129
|
end
|
130
130
|
|
131
131
|
it "custom stream" do
|
132
|
-
ex = PgCsv.new(:
|
132
|
+
ex = PgCsv.new(sql: @sql, type: :stream)
|
133
133
|
File.open(@name, 'w') do |stream|
|
134
134
|
ex.export(stream)
|
135
|
-
ex.export(stream, :
|
135
|
+
ex.export(stream, sql: @sql0)
|
136
136
|
end
|
137
137
|
|
138
|
-
with_file(@name){|d| d.
|
138
|
+
with_file(@name){ |d| expect(d).to eq("4,5,6\n1,2,3\n1,2,3\n4,5,6\n") }
|
139
139
|
end
|
140
140
|
|
141
141
|
it "file as default" do
|
142
|
-
PgCsv.new(:
|
143
|
-
with_file(@name){|d| d.
|
144
|
-
sprintf("%o", File.stat(@name).mode).to_i.
|
142
|
+
PgCsv.new(sql: @sql, type: :file).export(@name)
|
143
|
+
with_file(@name){ |d| expect(d).to eq("4,5,6\n1,2,3\n") }
|
144
|
+
expect(sprintf("%o", File.stat(@name).mode).to_i).to eq(100644)
|
145
145
|
end
|
146
146
|
|
147
147
|
it "yield export" do
|
148
148
|
rows = []
|
149
|
-
|
150
|
-
rows << row
|
151
|
-
|
149
|
+
expect(
|
150
|
+
PgCsv.new(sql: @sql, type: :yield).export { |row| rows << row }
|
151
|
+
).to eq(2)
|
152
152
|
|
153
|
-
rows.
|
153
|
+
expect(rows).to eq(["4,5,6\n", "1,2,3\n"])
|
154
154
|
end
|
155
155
|
end
|
156
156
|
|
157
157
|
describe "integration specs" do
|
158
158
|
it "1" do
|
159
|
-
File.
|
160
|
-
PgCsv.new(:
|
161
|
-
|
159
|
+
expect(File).not_to exist(@name)
|
160
|
+
PgCsv.new(sql: @sql, type: :gzip).export(
|
161
|
+
@name, delimiter: "|", columns: %w(q w e), temp_file: true, temp_dir: tmp_dir
|
162
|
+
)
|
163
|
+
with_gzfile(@name){ |d| expect(d).to eq("q|w|e\n4|5|6\n1|2|3\n") }
|
162
164
|
end
|
163
165
|
|
164
166
|
it "2" do
|
165
167
|
Zlib::GzipWriter.open(@name) do |gz|
|
166
|
-
e = PgCsv.new(:
|
168
|
+
e = PgCsv.new(sql: @sql, type: :stream)
|
167
169
|
|
168
|
-
e.export(gz, :
|
169
|
-
e.export(gz, :
|
170
|
+
e.export(gz, delimiter: "|", columns: %w(q w e) )
|
171
|
+
e.export(gz, delimiter: "*", sql: @sql0)
|
170
172
|
end
|
171
173
|
|
172
|
-
with_gzfile(@name){|d| d.
|
174
|
+
with_gzfile(@name){ |d| expect(d).to eq("q|w|e\n4|5|6\n1|2|3\n1*2*3\n4*5*6\n") }
|
173
175
|
end
|
174
176
|
|
175
177
|
it "gzip with empty content" do
|
176
|
-
File.
|
177
|
-
PgCsv.new(:
|
178
|
-
|
178
|
+
expect(File).not_to exist(@name)
|
179
|
+
PgCsv.new(sql: "select a,b,c from tests where a = -1", type: :gzip).export(
|
180
|
+
@name, temp_file: true, temp_dir: tmp_dir
|
181
|
+
)
|
182
|
+
with_gzfile(@name){ |d| expect(d).to be_empty }
|
179
183
|
end
|
180
184
|
end
|
181
185
|
|
182
186
|
it "custom row proc" do
|
183
|
-
e = PgCsv.new(:
|
187
|
+
e = PgCsv.new(sql: @sql)
|
184
188
|
|
185
189
|
e.export(@name) do |row|
|
186
190
|
row.split(",").join("-|-")
|
187
191
|
end
|
188
192
|
|
189
|
-
with_file(@name){|d| d.
|
193
|
+
with_file(@name){ |d| expect(d).to eq("4-|-5-|-6\n1-|-2-|-3\n") }
|
190
194
|
end
|
191
|
-
|
192
195
|
end
|
data/spec/spec_support.rb
CHANGED
@@ -1,7 +1,11 @@
|
|
1
1
|
require 'fileutils'
|
2
2
|
require 'active_record'
|
3
3
|
|
4
|
-
conn = {
|
4
|
+
conn = {
|
5
|
+
adapter: 'postgresql',
|
6
|
+
database: 'pgcsv_test',
|
7
|
+
encoding: "unicode"
|
8
|
+
}
|
5
9
|
ActiveRecord::Base.establish_connection conn
|
6
10
|
|
7
11
|
class Test < ActiveRecord::Base
|
@@ -29,7 +33,7 @@ def tmp_dir
|
|
29
33
|
end
|
30
34
|
|
31
35
|
def with_file(name)
|
32
|
-
File.
|
36
|
+
expect(File).to exist(name)
|
33
37
|
q = 1
|
34
38
|
File.open(name) do |file|
|
35
39
|
data = file.read
|
@@ -37,16 +41,17 @@ def with_file(name)
|
|
37
41
|
q = 2
|
38
42
|
end
|
39
43
|
|
40
|
-
q.
|
44
|
+
expect(q).to eq(2)
|
41
45
|
end
|
42
46
|
|
43
47
|
def with_gzfile(name)
|
44
|
-
File.exist
|
48
|
+
expect(File).to exist(name)
|
45
49
|
q = 1
|
46
50
|
Zlib::GzipReader.open(name) do |gz|
|
47
51
|
data = gz.read
|
48
52
|
yield data
|
49
53
|
q = 2
|
50
54
|
end
|
51
|
-
|
55
|
+
|
56
|
+
expect(q).to eq(2)
|
52
57
|
end
|
metadata
CHANGED
@@ -1,43 +1,43 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pg_csv
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '0.
|
4
|
+
version: '0.3'
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Makarchev Konstantin
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-11-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: pg
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '0
|
19
|
+
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - "
|
24
|
+
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '0
|
26
|
+
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: rspec
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - "
|
31
|
+
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
33
|
+
version: '0'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- - "
|
38
|
+
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
40
|
+
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: rake
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -66,6 +66,20 @@ dependencies:
|
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: byebug
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
69
83
|
description: Fast Ruby PG csv export. Uses pg function 'copy to csv'. Effective on
|
70
84
|
millions rows.
|
71
85
|
email:
|
@@ -107,16 +121,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
107
121
|
- !ruby/object:Gem::Version
|
108
122
|
version: '0'
|
109
123
|
requirements: []
|
110
|
-
|
111
|
-
rubygems_version: 2.2.2
|
124
|
+
rubygems_version: 3.1.2
|
112
125
|
signing_key:
|
113
126
|
specification_version: 4
|
114
127
|
summary: Fast Ruby PG csv export. Uses pg function 'copy to csv'. Effective on millions
|
115
128
|
rows.
|
116
|
-
test_files:
|
117
|
-
- spec/benchmark/bench.rb
|
118
|
-
- spec/benchmark/raw_bench.rb
|
119
|
-
- spec/pg_csv_spec.rb
|
120
|
-
- spec/spec_helper.rb
|
121
|
-
- spec/spec_support.rb
|
122
|
-
- spec/tmp/.gitkeep
|
129
|
+
test_files: []
|