pg_csv 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +2 -1
- data/lib/pg_csv.rb +171 -166
- data/lib/pg_csv_version.rb +1 -1
- data/spec/pg_csv_spec.rb +2 -2
- metadata +8 -3
- data/Gemfile.lock +0 -45
data/.gitignore
CHANGED
data/lib/pg_csv.rb
CHANGED
@@ -2,201 +2,206 @@ require 'active_record'
|
|
2
2
|
|
3
3
|
class PgCsv
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
def export(to = nil, opts = {}, &row_proc)
|
11
|
-
@row_proc = row_proc
|
12
|
-
@local_options = opts.symbolize_keys
|
5
|
+
module Base
|
6
|
+
|
7
|
+
def initialize(opts = {})
|
8
|
+
@options = opts.symbolize_keys
|
9
|
+
end
|
13
10
|
|
14
|
-
|
15
|
-
|
11
|
+
# do export :to - filename or stream
|
12
|
+
def export(to = nil, opts = {}, &row_proc)
|
13
|
+
@row_proc = row_proc
|
14
|
+
@local_options = opts.symbolize_keys
|
15
|
+
|
16
|
+
raise ":connection should be" unless connection
|
17
|
+
raise ":sql should be" unless sql
|
16
18
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
protected
|
19
|
+
with_temp_file?(to, temp_file, temp_dir) do |dest|
|
20
|
+
export_to(dest)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
protected
|
23
25
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
26
|
+
def with_temp_file?(to, use_temp_file, tmp_dir)
|
27
|
+
if use_temp_file && [:file, :gzip].include?(type)
|
28
|
+
check_str(to)
|
29
|
+
|
30
|
+
self.class.with_temp_file(to, tmp_dir) do |filename|
|
31
|
+
yield(filename)
|
32
|
+
end
|
30
33
|
|
31
|
-
|
32
|
-
|
34
|
+
info "<=== moving export to #{to}"
|
35
|
+
else
|
36
|
+
yield(to)
|
37
|
+
end
|
38
|
+
end
|
33
39
|
|
34
|
-
|
35
|
-
|
36
|
-
|
40
|
+
def export_to(to)
|
41
|
+
|
42
|
+
start = Time.now
|
43
|
+
info "===> start generate export #{to}, type: #{type}"
|
44
|
+
|
45
|
+
result = nil
|
37
46
|
|
38
|
-
|
39
|
-
|
47
|
+
case type
|
48
|
+
|
49
|
+
when :file
|
50
|
+
check_str(to)
|
51
|
+
File.open(to, 'w', &exporter)
|
52
|
+
|
53
|
+
when :gzip
|
54
|
+
check_str(to)
|
55
|
+
Zlib::GzipWriter.open(to, &exporter)
|
56
|
+
|
57
|
+
when :stream
|
58
|
+
raise "'to' should be" unless to
|
59
|
+
exporter[to]
|
60
|
+
|
61
|
+
when :plain
|
62
|
+
require 'stringio'
|
63
|
+
sio = StringIO.new
|
64
|
+
exporter[sio]
|
65
|
+
result = sio.string
|
66
|
+
|
67
|
+
when :yield
|
68
|
+
# not real saving anywhere, just yield each record
|
69
|
+
raise "row_proc should be" unless @row_proc
|
70
|
+
result = load_data{|_|}
|
40
71
|
end
|
41
72
|
|
42
|
-
info "<===
|
43
|
-
|
44
|
-
|
73
|
+
info "<=== finished write #{to} in #{Time.now - start}"
|
74
|
+
|
75
|
+
result
|
45
76
|
end
|
46
|
-
end
|
47
|
-
|
48
|
-
def export_to(to)
|
49
|
-
|
50
|
-
start = Time.now
|
51
|
-
info "===> start generate export #{to}, type: #{type}"
|
52
|
-
|
53
|
-
result = nil
|
54
|
-
|
55
|
-
case type
|
56
77
|
|
57
|
-
|
58
|
-
|
59
|
-
File.open(to, 'w', &exporter)
|
60
|
-
|
61
|
-
when :gzip
|
62
|
-
check_str(to)
|
63
|
-
Zlib::GzipWriter.open(to, &exporter)
|
64
|
-
|
65
|
-
when :stream
|
66
|
-
raise "'to' should be" unless to
|
67
|
-
exporter[to]
|
68
|
-
|
69
|
-
when :plain
|
70
|
-
require 'stringio'
|
71
|
-
sio = StringIO.new
|
72
|
-
exporter[sio]
|
73
|
-
result = sio.string
|
74
|
-
|
75
|
-
when :yield
|
76
|
-
# not real saving anywhere, just yield each record
|
77
|
-
raise "row_proc should be" unless @row_proc
|
78
|
-
result = load_data{|_|}
|
78
|
+
def check_str(to)
|
79
|
+
raise "'to' should be an string" unless to.is_a?(String)
|
79
80
|
end
|
80
81
|
|
81
|
-
|
82
|
-
|
83
|
-
result
|
84
|
-
end
|
85
|
-
|
86
|
-
def check_str(to)
|
87
|
-
raise "'to' should be an string" unless to.is_a?(String)
|
88
|
-
end
|
89
|
-
|
90
|
-
def exporter
|
91
|
-
method(:export_to_stream).to_proc
|
92
|
-
end
|
93
|
-
|
94
|
-
def export_to_stream(stream)
|
95
|
-
count = write_csv(stream)
|
96
|
-
stream.flush if stream.respond_to?(:flush)
|
97
|
-
|
98
|
-
info "<= done exporting (#{count}) records."
|
99
|
-
end
|
100
|
-
|
101
|
-
def write_csv(stream)
|
102
|
-
load_data do |row|
|
103
|
-
stream.write(row)
|
82
|
+
def exporter
|
83
|
+
method(:export_to_stream).to_proc
|
104
84
|
end
|
105
|
-
end
|
106
|
-
|
107
|
-
def load_data
|
108
|
-
info "#{query}"
|
109
|
-
raw = connection.raw_connection
|
110
85
|
|
111
|
-
|
112
|
-
|
113
|
-
|
86
|
+
def export_to_stream(stream)
|
87
|
+
count = write_csv(stream)
|
88
|
+
stream.flush if stream.respond_to?(:flush)
|
89
|
+
|
90
|
+
info "<= done exporting (#{count}) records."
|
91
|
+
end
|
114
92
|
|
115
|
-
|
116
|
-
|
117
|
-
|
93
|
+
def write_csv(stream)
|
94
|
+
load_data do |row|
|
95
|
+
stream.write(row)
|
96
|
+
end
|
118
97
|
end
|
98
|
+
|
99
|
+
def load_data
|
100
|
+
info "#{query}"
|
101
|
+
raw = connection.raw_connection
|
102
|
+
|
103
|
+
info "=> query"
|
104
|
+
q = raw.exec(query)
|
105
|
+
info "<= query"
|
119
106
|
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
yield(@row_proc[row])
|
124
|
-
count += 1
|
107
|
+
info "=> write data"
|
108
|
+
if columns_str
|
109
|
+
yield(@row_proc ? @row_proc[columns_str] : columns_str)
|
125
110
|
end
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
111
|
+
|
112
|
+
count = 0
|
113
|
+
if @row_proc
|
114
|
+
while row = raw.get_copy_data()
|
115
|
+
yield(@row_proc[row])
|
116
|
+
count += 1
|
117
|
+
end
|
118
|
+
else
|
119
|
+
while row = raw.get_copy_data()
|
120
|
+
yield(row)
|
121
|
+
count += 1
|
122
|
+
end
|
130
123
|
end
|
131
|
-
|
132
|
-
info "<= write data"
|
124
|
+
info "<= write data"
|
133
125
|
|
134
|
-
|
135
|
-
|
136
|
-
|
126
|
+
q.clear
|
127
|
+
count
|
128
|
+
end
|
137
129
|
|
138
|
-
|
139
|
-
|
140
|
-
COPY (
|
141
|
-
|
142
|
-
) TO STDOUT
|
143
|
-
WITH CSV
|
144
|
-
DELIMITER '#{delimiter}'
|
145
|
-
#{use_pg_header? ? 'HEADER' : ''}
|
146
|
-
|
147
|
-
|
130
|
+
def query
|
131
|
+
<<-SQL
|
132
|
+
COPY (
|
133
|
+
#{sql}
|
134
|
+
) TO STDOUT
|
135
|
+
WITH CSV
|
136
|
+
DELIMITER '#{delimiter}'
|
137
|
+
#{use_pg_header? ? 'HEADER' : ''}
|
138
|
+
SQL
|
139
|
+
end
|
148
140
|
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
141
|
+
def info(message)
|
142
|
+
logger.info(message) if logger
|
143
|
+
end
|
144
|
+
|
145
|
+
# ==== options/defaults =============
|
146
|
+
|
147
|
+
def o(key)
|
148
|
+
@local_options[key] || @options[key]
|
149
|
+
end
|
158
150
|
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
151
|
+
def connection
|
152
|
+
o(:connection) || (defined?(ActiveRecord::Base) ? ActiveRecord::Base.connection : nil)
|
153
|
+
end
|
154
|
+
|
155
|
+
def logger
|
156
|
+
o(:logger)
|
157
|
+
end
|
166
158
|
|
167
|
-
|
168
|
-
|
169
|
-
|
159
|
+
def type
|
160
|
+
o(:type) || :file
|
161
|
+
end
|
170
162
|
|
171
|
-
|
172
|
-
|
173
|
-
|
163
|
+
def use_pg_header?
|
164
|
+
o(:header) && !o(:columns)
|
165
|
+
end
|
174
166
|
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
167
|
+
def columns_str
|
168
|
+
if o(:columns)
|
169
|
+
col = o(:columns)
|
170
|
+
if col.is_a?(Array)
|
171
|
+
col.join(delimiter) + "\n"
|
172
|
+
else
|
173
|
+
col + "\n"
|
174
|
+
end
|
182
175
|
end
|
183
176
|
end
|
184
|
-
end
|
185
177
|
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
178
|
+
def delimiter
|
179
|
+
o(:delimiter) || ','
|
180
|
+
end
|
181
|
+
|
182
|
+
def sql
|
183
|
+
o(:sql)
|
184
|
+
end
|
185
|
+
|
186
|
+
def temp_file
|
187
|
+
o(:temp_file)
|
188
|
+
end
|
189
|
+
|
190
|
+
def temp_dir
|
191
|
+
o(:temp_dir) || '/tmp'
|
192
|
+
end
|
196
193
|
end
|
197
|
-
|
198
|
-
|
199
|
-
|
194
|
+
|
195
|
+
include Base
|
196
|
+
|
197
|
+
def self.with_temp_file(dest, tmp_dir = '/tmp', &block)
|
198
|
+
require 'fileutils'
|
199
|
+
require 'tempfile'
|
200
|
+
|
201
|
+
tempfile = Tempfile.new("pg_csv", tmp_dir)
|
202
|
+
yield(tempfile.path)
|
203
|
+
|
204
|
+
FileUtils.mv(tempfile.path, dest)
|
200
205
|
end
|
201
|
-
|
206
|
+
|
202
207
|
end
|
data/lib/pg_csv_version.rb
CHANGED
data/spec/pg_csv_spec.rb
CHANGED
@@ -97,7 +97,7 @@ describe PgCsv do
|
|
97
97
|
end
|
98
98
|
|
99
99
|
it "plain export" do
|
100
|
-
PgCsv.new(:sql => @sql, :type => :plain).export
|
100
|
+
PgCsv.new(:sql => @sql, :type => :plain).export.should == "4,5,6\n1,2,3\n"
|
101
101
|
end
|
102
102
|
|
103
103
|
it "custom stream" do
|
@@ -144,7 +144,7 @@ describe PgCsv do
|
|
144
144
|
end
|
145
145
|
end
|
146
146
|
|
147
|
-
it "custom
|
147
|
+
it "custom row proc" do
|
148
148
|
e = PgCsv.new(:sql => @sql)
|
149
149
|
|
150
150
|
e.export(@name) do |row|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pg_csv
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.5
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-08-17 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: pg
|
@@ -85,7 +85,6 @@ extra_rdoc_files: []
|
|
85
85
|
files:
|
86
86
|
- .gitignore
|
87
87
|
- Gemfile
|
88
|
-
- Gemfile.lock
|
89
88
|
- MIT-LICENSE
|
90
89
|
- README.md
|
91
90
|
- Rakefile
|
@@ -109,12 +108,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
109
108
|
- - ! '>='
|
110
109
|
- !ruby/object:Gem::Version
|
111
110
|
version: '0'
|
111
|
+
segments:
|
112
|
+
- 0
|
113
|
+
hash: 933370835
|
112
114
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
113
115
|
none: false
|
114
116
|
requirements:
|
115
117
|
- - ! '>='
|
116
118
|
- !ruby/object:Gem::Version
|
117
119
|
version: '0'
|
120
|
+
segments:
|
121
|
+
- 0
|
122
|
+
hash: 933370835
|
118
123
|
requirements: []
|
119
124
|
rubyforge_project:
|
120
125
|
rubygems_version: 1.8.24
|
data/Gemfile.lock
DELETED
@@ -1,45 +0,0 @@
|
|
1
|
-
PATH
|
2
|
-
remote: .
|
3
|
-
specs:
|
4
|
-
pg_csv (0.1.4)
|
5
|
-
activerecord
|
6
|
-
pg
|
7
|
-
|
8
|
-
GEM
|
9
|
-
remote: http://rubygems.org/
|
10
|
-
specs:
|
11
|
-
activemodel (3.2.3)
|
12
|
-
activesupport (= 3.2.3)
|
13
|
-
builder (~> 3.0.0)
|
14
|
-
activerecord (3.2.3)
|
15
|
-
activemodel (= 3.2.3)
|
16
|
-
activesupport (= 3.2.3)
|
17
|
-
arel (~> 3.0.2)
|
18
|
-
tzinfo (~> 0.3.29)
|
19
|
-
activesupport (3.2.3)
|
20
|
-
i18n (~> 0.6)
|
21
|
-
multi_json (~> 1.0)
|
22
|
-
arel (3.0.2)
|
23
|
-
builder (3.0.0)
|
24
|
-
diff-lcs (1.1.3)
|
25
|
-
i18n (0.6.0)
|
26
|
-
multi_json (1.3.6)
|
27
|
-
pg (0.13.2)
|
28
|
-
rake (0.9.2.2)
|
29
|
-
rspec (2.10.0)
|
30
|
-
rspec-core (~> 2.10.0)
|
31
|
-
rspec-expectations (~> 2.10.0)
|
32
|
-
rspec-mocks (~> 2.10.0)
|
33
|
-
rspec-core (2.10.1)
|
34
|
-
rspec-expectations (2.10.0)
|
35
|
-
diff-lcs (~> 1.1.3)
|
36
|
-
rspec-mocks (2.10.1)
|
37
|
-
tzinfo (0.3.33)
|
38
|
-
|
39
|
-
PLATFORMS
|
40
|
-
ruby
|
41
|
-
|
42
|
-
DEPENDENCIES
|
43
|
-
pg_csv!
|
44
|
-
rake
|
45
|
-
rspec
|