pg_csv 0.2 → 0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/README.md +14 -14
- data/lib/pg_csv/version.rb +1 -1
- data/pg_csv.gemspec +3 -2
- data/spec/pg_csv_spec.rb +69 -66
- data/spec/spec_support.rb +10 -5
- metadata +26 -19
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 88e1d29a5270587b750a05a8c3b620907988d0b4e8942ddc0dec4f8f14423d7e
|
4
|
+
data.tar.gz: 66d1d47569da691d87fba379cf223a10a8c405d20663375a75dc6fdb9068ffd2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 575ef7200517a797d4b625c7dddc7e704ffa8719d1c7e6b4f19bbf0807204ea789317892bdd46ec07a6e698e86afb4282c2c5cfce60d0c1117527f8e1e3c22d3
|
7
|
+
data.tar.gz: f392c817d176ddc268c82b0f9f4aad2642ea60b93437602732a143526e36477837f12f3a879c663ef60ee24d0e0953c2b974036e973069a69a4ac61e03cb3457
|
data/README.md
CHANGED
@@ -38,28 +38,28 @@ Options:
|
|
38
38
|
|
39
39
|
Examples:
|
40
40
|
``` ruby
|
41
|
-
PgCsv.new(:
|
42
|
-
PgCsv.new(:
|
43
|
-
PgCsv.new(:
|
44
|
-
PgCsv.new(:
|
45
|
-
File.open("a4.csv", 'a'){|f| PgCsv.new(:
|
46
|
-
export(f, :
|
47
|
-
PgCsv.new(:
|
48
|
-
PgCsv.new(:
|
49
|
-
PgCsv.new(:
|
50
|
-
PgCsv.new(:
|
51
|
-
export('a8.gz', :
|
41
|
+
PgCsv.new(sql: sql).export('a1.csv')
|
42
|
+
PgCsv.new(sql: sql).export('a2.gz', type: :gzip)
|
43
|
+
PgCsv.new(sql: sql).export('a3.csv', temp_file: true)
|
44
|
+
PgCsv.new(sql: sql, type: :plain).export
|
45
|
+
File.open("a4.csv", 'a'){ |f| PgCsv.new(sql: "select * from users").\
|
46
|
+
export(f, type: :stream) }
|
47
|
+
PgCsv.new(sql: sql).export('a5.csv', delimiter: "\t")
|
48
|
+
PgCsv.new(sql: sql).export('a6.csv', header: true)
|
49
|
+
PgCsv.new(sql: sql).export('a7.csv', columns: %w(id a b c))
|
50
|
+
PgCsv.new(sql: sql, connection: SomeDb.connection, columns: %w(id a b c), delimiter: "|").\
|
51
|
+
export('a8.gz', type: :gzip, temp_file: true)
|
52
52
|
|
53
53
|
# example collect from shards
|
54
54
|
Zlib::GzipWriter.open('some.gz') do |stream|
|
55
|
-
e = PgCsv.new(:
|
55
|
+
e = PgCsv.new(sql: sql, type: :stream)
|
56
56
|
ConnectionPool.each_shard do |connection|
|
57
|
-
e.export(stream, :
|
57
|
+
e.export(stream, connection: connection)
|
58
58
|
end
|
59
59
|
end
|
60
60
|
|
61
61
|
# yield example
|
62
|
-
PgCsv.new(:
|
62
|
+
PgCsv.new(sql: sql, type: :yield).export do |row|
|
63
63
|
puts row
|
64
64
|
end
|
65
65
|
```
|
data/lib/pg_csv/version.rb
CHANGED
data/pg_csv.gemspec
CHANGED
@@ -19,8 +19,9 @@ Gem::Specification.new do |s|
|
|
19
19
|
s.require_paths = ["lib"]
|
20
20
|
s.license = "MIT"
|
21
21
|
|
22
|
-
s.add_dependency "pg"
|
23
|
-
s.add_development_dependency "rspec"
|
22
|
+
s.add_dependency "pg"
|
23
|
+
s.add_development_dependency "rspec"
|
24
24
|
s.add_development_dependency "rake"
|
25
25
|
s.add_development_dependency "activerecord"
|
26
|
+
s.add_development_dependency "byebug"
|
26
27
|
end
|
data/spec/pg_csv_spec.rb
CHANGED
@@ -1,12 +1,13 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
+
|
2
3
|
require File.dirname(__FILE__) + '/spec_helper'
|
3
4
|
|
4
5
|
describe PgCsv do
|
5
6
|
|
6
7
|
before :each do
|
7
8
|
Test.delete_all
|
8
|
-
Test.create :
|
9
|
-
Test.create :
|
9
|
+
Test.create a: 1, b: 2, c: 3
|
10
|
+
Test.create a: 4, b: 5, c: 6
|
10
11
|
|
11
12
|
@name = tmp_dir + "1.csv"
|
12
13
|
FileUtils.rm(@name) rescue nil
|
@@ -22,171 +23,173 @@ describe PgCsv do
|
|
22
23
|
describe "simple export" do
|
23
24
|
|
24
25
|
it "1" do
|
25
|
-
PgCsv.new(:
|
26
|
-
with_file(@name){|d| d.
|
26
|
+
PgCsv.new(sql: @sql0).export(@name)
|
27
|
+
with_file(@name){ |d| expect(d).to eq("1,2,3\n4,5,6\n")}
|
27
28
|
end
|
28
29
|
|
29
30
|
it "2" do
|
30
|
-
PgCsv.new(:
|
31
|
-
with_file(@name){|d| d.
|
31
|
+
PgCsv.new(sql: @sql).export(@name)
|
32
|
+
with_file(@name){ |d| expect(d).to eq("4,5,6\n1,2,3\n") }
|
32
33
|
end
|
33
34
|
|
34
35
|
it "delimiter" do
|
35
|
-
PgCsv.new(:
|
36
|
-
with_file(@name){|d| d.
|
36
|
+
PgCsv.new(sql: @sql).export(@name, delimiter: "|")
|
37
|
+
with_file(@name){ |d| expect(d).to eq("4|5|6\n1|2|3\n") }
|
37
38
|
end
|
38
39
|
|
39
40
|
it "encoding" do
|
40
|
-
Test.create!
|
41
|
+
Test.create!(a: 2, b: 3, c: 4, d: "абсд")
|
41
42
|
|
42
|
-
PgCsv.new(:
|
43
|
-
with_file(@name){|d| d.force_encoding('cp1251').
|
43
|
+
PgCsv.new(sql: "select d from tests where a = 2").export(@name, encoding: "WIN1251")
|
44
|
+
with_file(@name){ |d| expect(d.force_encoding('cp1251')).to eq("абсд\n".encode('cp1251')) }
|
44
45
|
end
|
45
46
|
|
46
47
|
describe "headers" do
|
47
48
|
it "header" do
|
48
|
-
PgCsv.new(:
|
49
|
-
with_file(@name){|d| d.
|
49
|
+
PgCsv.new(sql: @sql).export(@name, header: true)
|
50
|
+
with_file(@name){ |d| expect(d).to eq("a,b,c\n4,5,6\n1,2,3\n") }
|
50
51
|
end
|
51
52
|
|
52
53
|
it "columns" do
|
53
|
-
PgCsv.new(:
|
54
|
-
with_file(@name){|d| d.
|
54
|
+
PgCsv.new(sql: @sql).export(@name, columns: %w(q w e))
|
55
|
+
with_file(@name){ |d| expect(d).to eq("q,w,e\n4,5,6\n1,2,3\n") }
|
55
56
|
end
|
56
57
|
|
57
58
|
it "columns with header" do
|
58
|
-
PgCsv.new(:
|
59
|
+
PgCsv.new(sql: @sql).export(@name, header: true, columns: %w(q w e))
|
59
60
|
|
60
61
|
with_file(@name) do |d|
|
61
|
-
d.
|
62
|
+
expect(d).to eq("q,w,e\n4,5,6\n1,2,3\n")
|
62
63
|
end
|
63
64
|
end
|
64
65
|
end
|
65
66
|
|
66
67
|
describe "force_quote" do
|
67
68
|
it "force_quote" do
|
68
|
-
PgCsv.new(:
|
69
|
-
with_file(@name){|d| d.
|
69
|
+
PgCsv.new(sql: @sql).export(@name, force_quote: true)
|
70
|
+
with_file(@name){ |d| expect(d).to eq("\"4\",\"5\",\"6\"\n\"1\",\"2\",\"3\"\n") }
|
70
71
|
end
|
71
72
|
|
72
73
|
it "with headers" do
|
73
|
-
PgCsv.new(:
|
74
|
-
with_file(@name){|d| d.
|
74
|
+
PgCsv.new(sql: @sql).export(@name, header: true, force_quote: true)
|
75
|
+
with_file(@name){ |d| expect(d).to eq("a,b,c\n\"4\",\"5\",\"6\"\n\"1\",\"2\",\"3\"\n") }
|
75
76
|
end
|
76
77
|
end
|
77
|
-
|
78
78
|
end
|
79
79
|
|
80
80
|
describe "moving options no matter" do
|
81
81
|
it "1" do
|
82
|
-
PgCsv.new(:
|
83
|
-
with_file(@name){|d| d.
|
82
|
+
PgCsv.new(sql: @sql).export(@name, delimiter: "|")
|
83
|
+
with_file(@name){ |d| expect(d).to eq("4|5|6\n1|2|3\n") }
|
84
84
|
end
|
85
85
|
|
86
86
|
it "2" do
|
87
|
-
PgCsv.new(:
|
88
|
-
with_file(@name){|d| d.
|
87
|
+
PgCsv.new(delimiter: "|").export(@name, sql: @sql)
|
88
|
+
with_file(@name){ |d| expect(d).to eq("4|5|6\n1|2|3\n") }
|
89
89
|
end
|
90
90
|
end
|
91
91
|
|
92
92
|
describe "local options dont recover global" do
|
93
93
|
it "test" do
|
94
|
-
e = PgCsv.new(:
|
95
|
-
e.export(@name, :
|
96
|
-
with_file(@name){|d| d.
|
94
|
+
e = PgCsv.new(sql: @sql, delimiter: "*")
|
95
|
+
e.export(@name, delimiter: "|")
|
96
|
+
with_file(@name){ |d| expect(d).to eq("4|5|6\n1|2|3\n") }
|
97
97
|
|
98
98
|
e.export(@name)
|
99
|
-
with_file(@name){|d| d.
|
99
|
+
with_file(@name){ |d| expect(d).to eq("4*5*6\n1*2*3\n") }
|
100
100
|
end
|
101
101
|
end
|
102
102
|
|
103
103
|
describe "using temp file" do
|
104
104
|
it "at least file should return to target and set correct chmod" do
|
105
|
-
File.
|
106
|
-
PgCsv.new(:
|
107
|
-
with_file(@name){|d| d.
|
108
|
-
sprintf("%o", File.stat(@name).mode).to_i.
|
105
|
+
expect(File).not_to exist(@name)
|
106
|
+
PgCsv.new(sql: @sql, temp_file: true, temp_dir: tmp_dir).export(@name)
|
107
|
+
with_file(@name){ |d| expect(d).to eq("4,5,6\n1,2,3\n") }
|
108
|
+
expect(sprintf("%o", File.stat(@name).mode).to_i).to eq(100644)
|
109
109
|
end
|
110
110
|
|
111
111
|
it "same with gzip" do
|
112
|
-
File.
|
113
|
-
PgCsv.new(:
|
114
|
-
with_gzfile(@name){|d| d.
|
115
|
-
sprintf("%o", File.stat(@name).mode).to_i.
|
112
|
+
expect(File).not_to exist(@name)
|
113
|
+
PgCsv.new(sql: @sql, temp_file: true, temp_dir: tmp_dir, type: :gzip).export(@name)
|
114
|
+
with_gzfile(@name){ |d| expect(d).to eq("4,5,6\n1,2,3\n") }
|
115
|
+
expect(sprintf("%o", File.stat(@name).mode).to_i).to eq(100644)
|
116
116
|
end
|
117
117
|
end
|
118
118
|
|
119
119
|
describe "different types of export" do
|
120
120
|
it "gzip export" do
|
121
|
-
File.
|
122
|
-
PgCsv.new(:
|
123
|
-
with_gzfile(@name){|d| d.
|
124
|
-
sprintf("%o", File.stat(@name).mode).to_i.
|
121
|
+
expect(File).not_to exist(@name)
|
122
|
+
PgCsv.new(sql: @sql, type: :gzip).export(@name)
|
123
|
+
with_gzfile(@name){ |d| expect(d).to eq("4,5,6\n1,2,3\n") }
|
124
|
+
expect(sprintf("%o", File.stat(@name).mode).to_i).to eq(100644)
|
125
125
|
end
|
126
126
|
|
127
127
|
it "plain export" do
|
128
|
-
PgCsv.new(:
|
128
|
+
expect(PgCsv.new(sql: @sql, type: :plain).export).to eq("4,5,6\n1,2,3\n")
|
129
129
|
end
|
130
130
|
|
131
131
|
it "custom stream" do
|
132
|
-
ex = PgCsv.new(:
|
132
|
+
ex = PgCsv.new(sql: @sql, type: :stream)
|
133
133
|
File.open(@name, 'w') do |stream|
|
134
134
|
ex.export(stream)
|
135
|
-
ex.export(stream, :
|
135
|
+
ex.export(stream, sql: @sql0)
|
136
136
|
end
|
137
137
|
|
138
|
-
with_file(@name){|d| d.
|
138
|
+
with_file(@name){ |d| expect(d).to eq("4,5,6\n1,2,3\n1,2,3\n4,5,6\n") }
|
139
139
|
end
|
140
140
|
|
141
141
|
it "file as default" do
|
142
|
-
PgCsv.new(:
|
143
|
-
with_file(@name){|d| d.
|
144
|
-
sprintf("%o", File.stat(@name).mode).to_i.
|
142
|
+
PgCsv.new(sql: @sql, type: :file).export(@name)
|
143
|
+
with_file(@name){ |d| expect(d).to eq("4,5,6\n1,2,3\n") }
|
144
|
+
expect(sprintf("%o", File.stat(@name).mode).to_i).to eq(100644)
|
145
145
|
end
|
146
146
|
|
147
147
|
it "yield export" do
|
148
148
|
rows = []
|
149
|
-
|
150
|
-
rows << row
|
151
|
-
|
149
|
+
expect(
|
150
|
+
PgCsv.new(sql: @sql, type: :yield).export { |row| rows << row }
|
151
|
+
).to eq(2)
|
152
152
|
|
153
|
-
rows.
|
153
|
+
expect(rows).to eq(["4,5,6\n", "1,2,3\n"])
|
154
154
|
end
|
155
155
|
end
|
156
156
|
|
157
157
|
describe "integration specs" do
|
158
158
|
it "1" do
|
159
|
-
File.
|
160
|
-
PgCsv.new(:
|
161
|
-
|
159
|
+
expect(File).not_to exist(@name)
|
160
|
+
PgCsv.new(sql: @sql, type: :gzip).export(
|
161
|
+
@name, delimiter: "|", columns: %w(q w e), temp_file: true, temp_dir: tmp_dir
|
162
|
+
)
|
163
|
+
with_gzfile(@name){ |d| expect(d).to eq("q|w|e\n4|5|6\n1|2|3\n") }
|
162
164
|
end
|
163
165
|
|
164
166
|
it "2" do
|
165
167
|
Zlib::GzipWriter.open(@name) do |gz|
|
166
|
-
e = PgCsv.new(:
|
168
|
+
e = PgCsv.new(sql: @sql, type: :stream)
|
167
169
|
|
168
|
-
e.export(gz, :
|
169
|
-
e.export(gz, :
|
170
|
+
e.export(gz, delimiter: "|", columns: %w(q w e) )
|
171
|
+
e.export(gz, delimiter: "*", sql: @sql0)
|
170
172
|
end
|
171
173
|
|
172
|
-
with_gzfile(@name){|d| d.
|
174
|
+
with_gzfile(@name){ |d| expect(d).to eq("q|w|e\n4|5|6\n1|2|3\n1*2*3\n4*5*6\n") }
|
173
175
|
end
|
174
176
|
|
175
177
|
it "gzip with empty content" do
|
176
|
-
File.
|
177
|
-
PgCsv.new(:
|
178
|
-
|
178
|
+
expect(File).not_to exist(@name)
|
179
|
+
PgCsv.new(sql: "select a,b,c from tests where a = -1", type: :gzip).export(
|
180
|
+
@name, temp_file: true, temp_dir: tmp_dir
|
181
|
+
)
|
182
|
+
with_gzfile(@name){ |d| expect(d).to be_empty }
|
179
183
|
end
|
180
184
|
end
|
181
185
|
|
182
186
|
it "custom row proc" do
|
183
|
-
e = PgCsv.new(:
|
187
|
+
e = PgCsv.new(sql: @sql)
|
184
188
|
|
185
189
|
e.export(@name) do |row|
|
186
190
|
row.split(",").join("-|-")
|
187
191
|
end
|
188
192
|
|
189
|
-
with_file(@name){|d| d.
|
193
|
+
with_file(@name){ |d| expect(d).to eq("4-|-5-|-6\n1-|-2-|-3\n") }
|
190
194
|
end
|
191
|
-
|
192
195
|
end
|
data/spec/spec_support.rb
CHANGED
@@ -1,7 +1,11 @@
|
|
1
1
|
require 'fileutils'
|
2
2
|
require 'active_record'
|
3
3
|
|
4
|
-
conn = {
|
4
|
+
conn = {
|
5
|
+
adapter: 'postgresql',
|
6
|
+
database: 'pgcsv_test',
|
7
|
+
encoding: "unicode"
|
8
|
+
}
|
5
9
|
ActiveRecord::Base.establish_connection conn
|
6
10
|
|
7
11
|
class Test < ActiveRecord::Base
|
@@ -29,7 +33,7 @@ def tmp_dir
|
|
29
33
|
end
|
30
34
|
|
31
35
|
def with_file(name)
|
32
|
-
File.
|
36
|
+
expect(File).to exist(name)
|
33
37
|
q = 1
|
34
38
|
File.open(name) do |file|
|
35
39
|
data = file.read
|
@@ -37,16 +41,17 @@ def with_file(name)
|
|
37
41
|
q = 2
|
38
42
|
end
|
39
43
|
|
40
|
-
q.
|
44
|
+
expect(q).to eq(2)
|
41
45
|
end
|
42
46
|
|
43
47
|
def with_gzfile(name)
|
44
|
-
File.exist
|
48
|
+
expect(File).to exist(name)
|
45
49
|
q = 1
|
46
50
|
Zlib::GzipReader.open(name) do |gz|
|
47
51
|
data = gz.read
|
48
52
|
yield data
|
49
53
|
q = 2
|
50
54
|
end
|
51
|
-
|
55
|
+
|
56
|
+
expect(q).to eq(2)
|
52
57
|
end
|
metadata
CHANGED
@@ -1,43 +1,43 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pg_csv
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '0.
|
4
|
+
version: '0.3'
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Makarchev Konstantin
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-11-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: pg
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '0
|
19
|
+
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - "
|
24
|
+
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '0
|
26
|
+
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: rspec
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - "
|
31
|
+
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
33
|
+
version: '0'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- - "
|
38
|
+
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
40
|
+
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: rake
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -66,6 +66,20 @@ dependencies:
|
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: byebug
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
69
83
|
description: Fast Ruby PG csv export. Uses pg function 'copy to csv'. Effective on
|
70
84
|
millions rows.
|
71
85
|
email:
|
@@ -107,16 +121,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
107
121
|
- !ruby/object:Gem::Version
|
108
122
|
version: '0'
|
109
123
|
requirements: []
|
110
|
-
|
111
|
-
rubygems_version: 2.2.2
|
124
|
+
rubygems_version: 3.1.2
|
112
125
|
signing_key:
|
113
126
|
specification_version: 4
|
114
127
|
summary: Fast Ruby PG csv export. Uses pg function 'copy to csv'. Effective on millions
|
115
128
|
rows.
|
116
|
-
test_files:
|
117
|
-
- spec/benchmark/bench.rb
|
118
|
-
- spec/benchmark/raw_bench.rb
|
119
|
-
- spec/pg_csv_spec.rb
|
120
|
-
- spec/spec_helper.rb
|
121
|
-
- spec/spec_support.rb
|
122
|
-
- spec/tmp/.gitkeep
|
129
|
+
test_files: []
|