pg_csv 0.1.4 → 0.1.5
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +2 -1
- data/lib/pg_csv.rb +171 -166
- data/lib/pg_csv_version.rb +1 -1
- data/spec/pg_csv_spec.rb +2 -2
- metadata +8 -3
- data/Gemfile.lock +0 -45
data/.gitignore
CHANGED
data/lib/pg_csv.rb
CHANGED
@@ -2,201 +2,206 @@ require 'active_record'
|
|
2
2
|
|
3
3
|
class PgCsv
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
def export(to = nil, opts = {}, &row_proc)
|
11
|
-
@row_proc = row_proc
|
12
|
-
@local_options = opts.symbolize_keys
|
5
|
+
module Base
|
6
|
+
|
7
|
+
def initialize(opts = {})
|
8
|
+
@options = opts.symbolize_keys
|
9
|
+
end
|
13
10
|
|
14
|
-
|
15
|
-
|
11
|
+
# do export :to - filename or stream
|
12
|
+
def export(to = nil, opts = {}, &row_proc)
|
13
|
+
@row_proc = row_proc
|
14
|
+
@local_options = opts.symbolize_keys
|
15
|
+
|
16
|
+
raise ":connection should be" unless connection
|
17
|
+
raise ":sql should be" unless sql
|
16
18
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
protected
|
19
|
+
with_temp_file?(to, temp_file, temp_dir) do |dest|
|
20
|
+
export_to(dest)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
protected
|
23
25
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
26
|
+
def with_temp_file?(to, use_temp_file, tmp_dir)
|
27
|
+
if use_temp_file && [:file, :gzip].include?(type)
|
28
|
+
check_str(to)
|
29
|
+
|
30
|
+
self.class.with_temp_file(to, tmp_dir) do |filename|
|
31
|
+
yield(filename)
|
32
|
+
end
|
30
33
|
|
31
|
-
|
32
|
-
|
34
|
+
info "<=== moving export to #{to}"
|
35
|
+
else
|
36
|
+
yield(to)
|
37
|
+
end
|
38
|
+
end
|
33
39
|
|
34
|
-
|
35
|
-
|
36
|
-
|
40
|
+
def export_to(to)
|
41
|
+
|
42
|
+
start = Time.now
|
43
|
+
info "===> start generate export #{to}, type: #{type}"
|
44
|
+
|
45
|
+
result = nil
|
37
46
|
|
38
|
-
|
39
|
-
|
47
|
+
case type
|
48
|
+
|
49
|
+
when :file
|
50
|
+
check_str(to)
|
51
|
+
File.open(to, 'w', &exporter)
|
52
|
+
|
53
|
+
when :gzip
|
54
|
+
check_str(to)
|
55
|
+
Zlib::GzipWriter.open(to, &exporter)
|
56
|
+
|
57
|
+
when :stream
|
58
|
+
raise "'to' should be" unless to
|
59
|
+
exporter[to]
|
60
|
+
|
61
|
+
when :plain
|
62
|
+
require 'stringio'
|
63
|
+
sio = StringIO.new
|
64
|
+
exporter[sio]
|
65
|
+
result = sio.string
|
66
|
+
|
67
|
+
when :yield
|
68
|
+
# not real saving anywhere, just yield each record
|
69
|
+
raise "row_proc should be" unless @row_proc
|
70
|
+
result = load_data{|_|}
|
40
71
|
end
|
41
72
|
|
42
|
-
info "<===
|
43
|
-
|
44
|
-
|
73
|
+
info "<=== finished write #{to} in #{Time.now - start}"
|
74
|
+
|
75
|
+
result
|
45
76
|
end
|
46
|
-
end
|
47
|
-
|
48
|
-
def export_to(to)
|
49
|
-
|
50
|
-
start = Time.now
|
51
|
-
info "===> start generate export #{to}, type: #{type}"
|
52
|
-
|
53
|
-
result = nil
|
54
|
-
|
55
|
-
case type
|
56
77
|
|
57
|
-
|
58
|
-
|
59
|
-
File.open(to, 'w', &exporter)
|
60
|
-
|
61
|
-
when :gzip
|
62
|
-
check_str(to)
|
63
|
-
Zlib::GzipWriter.open(to, &exporter)
|
64
|
-
|
65
|
-
when :stream
|
66
|
-
raise "'to' should be" unless to
|
67
|
-
exporter[to]
|
68
|
-
|
69
|
-
when :plain
|
70
|
-
require 'stringio'
|
71
|
-
sio = StringIO.new
|
72
|
-
exporter[sio]
|
73
|
-
result = sio.string
|
74
|
-
|
75
|
-
when :yield
|
76
|
-
# not real saving anywhere, just yield each record
|
77
|
-
raise "row_proc should be" unless @row_proc
|
78
|
-
result = load_data{|_|}
|
78
|
+
def check_str(to)
|
79
|
+
raise "'to' should be an string" unless to.is_a?(String)
|
79
80
|
end
|
80
81
|
|
81
|
-
|
82
|
-
|
83
|
-
result
|
84
|
-
end
|
85
|
-
|
86
|
-
def check_str(to)
|
87
|
-
raise "'to' should be an string" unless to.is_a?(String)
|
88
|
-
end
|
89
|
-
|
90
|
-
def exporter
|
91
|
-
method(:export_to_stream).to_proc
|
92
|
-
end
|
93
|
-
|
94
|
-
def export_to_stream(stream)
|
95
|
-
count = write_csv(stream)
|
96
|
-
stream.flush if stream.respond_to?(:flush)
|
97
|
-
|
98
|
-
info "<= done exporting (#{count}) records."
|
99
|
-
end
|
100
|
-
|
101
|
-
def write_csv(stream)
|
102
|
-
load_data do |row|
|
103
|
-
stream.write(row)
|
82
|
+
def exporter
|
83
|
+
method(:export_to_stream).to_proc
|
104
84
|
end
|
105
|
-
end
|
106
|
-
|
107
|
-
def load_data
|
108
|
-
info "#{query}"
|
109
|
-
raw = connection.raw_connection
|
110
85
|
|
111
|
-
|
112
|
-
|
113
|
-
|
86
|
+
def export_to_stream(stream)
|
87
|
+
count = write_csv(stream)
|
88
|
+
stream.flush if stream.respond_to?(:flush)
|
89
|
+
|
90
|
+
info "<= done exporting (#{count}) records."
|
91
|
+
end
|
114
92
|
|
115
|
-
|
116
|
-
|
117
|
-
|
93
|
+
def write_csv(stream)
|
94
|
+
load_data do |row|
|
95
|
+
stream.write(row)
|
96
|
+
end
|
118
97
|
end
|
98
|
+
|
99
|
+
def load_data
|
100
|
+
info "#{query}"
|
101
|
+
raw = connection.raw_connection
|
102
|
+
|
103
|
+
info "=> query"
|
104
|
+
q = raw.exec(query)
|
105
|
+
info "<= query"
|
119
106
|
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
yield(@row_proc[row])
|
124
|
-
count += 1
|
107
|
+
info "=> write data"
|
108
|
+
if columns_str
|
109
|
+
yield(@row_proc ? @row_proc[columns_str] : columns_str)
|
125
110
|
end
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
111
|
+
|
112
|
+
count = 0
|
113
|
+
if @row_proc
|
114
|
+
while row = raw.get_copy_data()
|
115
|
+
yield(@row_proc[row])
|
116
|
+
count += 1
|
117
|
+
end
|
118
|
+
else
|
119
|
+
while row = raw.get_copy_data()
|
120
|
+
yield(row)
|
121
|
+
count += 1
|
122
|
+
end
|
130
123
|
end
|
131
|
-
|
132
|
-
info "<= write data"
|
124
|
+
info "<= write data"
|
133
125
|
|
134
|
-
|
135
|
-
|
136
|
-
|
126
|
+
q.clear
|
127
|
+
count
|
128
|
+
end
|
137
129
|
|
138
|
-
|
139
|
-
|
140
|
-
COPY (
|
141
|
-
|
142
|
-
) TO STDOUT
|
143
|
-
WITH CSV
|
144
|
-
DELIMITER '#{delimiter}'
|
145
|
-
#{use_pg_header? ? 'HEADER' : ''}
|
146
|
-
|
147
|
-
|
130
|
+
def query
|
131
|
+
<<-SQL
|
132
|
+
COPY (
|
133
|
+
#{sql}
|
134
|
+
) TO STDOUT
|
135
|
+
WITH CSV
|
136
|
+
DELIMITER '#{delimiter}'
|
137
|
+
#{use_pg_header? ? 'HEADER' : ''}
|
138
|
+
SQL
|
139
|
+
end
|
148
140
|
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
141
|
+
def info(message)
|
142
|
+
logger.info(message) if logger
|
143
|
+
end
|
144
|
+
|
145
|
+
# ==== options/defaults =============
|
146
|
+
|
147
|
+
def o(key)
|
148
|
+
@local_options[key] || @options[key]
|
149
|
+
end
|
158
150
|
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
151
|
+
def connection
|
152
|
+
o(:connection) || (defined?(ActiveRecord::Base) ? ActiveRecord::Base.connection : nil)
|
153
|
+
end
|
154
|
+
|
155
|
+
def logger
|
156
|
+
o(:logger)
|
157
|
+
end
|
166
158
|
|
167
|
-
|
168
|
-
|
169
|
-
|
159
|
+
def type
|
160
|
+
o(:type) || :file
|
161
|
+
end
|
170
162
|
|
171
|
-
|
172
|
-
|
173
|
-
|
163
|
+
def use_pg_header?
|
164
|
+
o(:header) && !o(:columns)
|
165
|
+
end
|
174
166
|
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
167
|
+
def columns_str
|
168
|
+
if o(:columns)
|
169
|
+
col = o(:columns)
|
170
|
+
if col.is_a?(Array)
|
171
|
+
col.join(delimiter) + "\n"
|
172
|
+
else
|
173
|
+
col + "\n"
|
174
|
+
end
|
182
175
|
end
|
183
176
|
end
|
184
|
-
end
|
185
177
|
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
178
|
+
def delimiter
|
179
|
+
o(:delimiter) || ','
|
180
|
+
end
|
181
|
+
|
182
|
+
def sql
|
183
|
+
o(:sql)
|
184
|
+
end
|
185
|
+
|
186
|
+
def temp_file
|
187
|
+
o(:temp_file)
|
188
|
+
end
|
189
|
+
|
190
|
+
def temp_dir
|
191
|
+
o(:temp_dir) || '/tmp'
|
192
|
+
end
|
196
193
|
end
|
197
|
-
|
198
|
-
|
199
|
-
|
194
|
+
|
195
|
+
include Base
|
196
|
+
|
197
|
+
def self.with_temp_file(dest, tmp_dir = '/tmp', &block)
|
198
|
+
require 'fileutils'
|
199
|
+
require 'tempfile'
|
200
|
+
|
201
|
+
tempfile = Tempfile.new("pg_csv", tmp_dir)
|
202
|
+
yield(tempfile.path)
|
203
|
+
|
204
|
+
FileUtils.mv(tempfile.path, dest)
|
200
205
|
end
|
201
|
-
|
206
|
+
|
202
207
|
end
|
data/lib/pg_csv_version.rb
CHANGED
data/spec/pg_csv_spec.rb
CHANGED
@@ -97,7 +97,7 @@ describe PgCsv do
|
|
97
97
|
end
|
98
98
|
|
99
99
|
it "plain export" do
|
100
|
-
PgCsv.new(:sql => @sql, :type => :plain).export
|
100
|
+
PgCsv.new(:sql => @sql, :type => :plain).export.should == "4,5,6\n1,2,3\n"
|
101
101
|
end
|
102
102
|
|
103
103
|
it "custom stream" do
|
@@ -144,7 +144,7 @@ describe PgCsv do
|
|
144
144
|
end
|
145
145
|
end
|
146
146
|
|
147
|
-
it "custom
|
147
|
+
it "custom row proc" do
|
148
148
|
e = PgCsv.new(:sql => @sql)
|
149
149
|
|
150
150
|
e.export(@name) do |row|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pg_csv
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.5
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-08-17 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: pg
|
@@ -85,7 +85,6 @@ extra_rdoc_files: []
|
|
85
85
|
files:
|
86
86
|
- .gitignore
|
87
87
|
- Gemfile
|
88
|
-
- Gemfile.lock
|
89
88
|
- MIT-LICENSE
|
90
89
|
- README.md
|
91
90
|
- Rakefile
|
@@ -109,12 +108,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
109
108
|
- - ! '>='
|
110
109
|
- !ruby/object:Gem::Version
|
111
110
|
version: '0'
|
111
|
+
segments:
|
112
|
+
- 0
|
113
|
+
hash: 933370835
|
112
114
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
113
115
|
none: false
|
114
116
|
requirements:
|
115
117
|
- - ! '>='
|
116
118
|
- !ruby/object:Gem::Version
|
117
119
|
version: '0'
|
120
|
+
segments:
|
121
|
+
- 0
|
122
|
+
hash: 933370835
|
118
123
|
requirements: []
|
119
124
|
rubyforge_project:
|
120
125
|
rubygems_version: 1.8.24
|
data/Gemfile.lock
DELETED
@@ -1,45 +0,0 @@
|
|
1
|
-
PATH
|
2
|
-
remote: .
|
3
|
-
specs:
|
4
|
-
pg_csv (0.1.4)
|
5
|
-
activerecord
|
6
|
-
pg
|
7
|
-
|
8
|
-
GEM
|
9
|
-
remote: http://rubygems.org/
|
10
|
-
specs:
|
11
|
-
activemodel (3.2.3)
|
12
|
-
activesupport (= 3.2.3)
|
13
|
-
builder (~> 3.0.0)
|
14
|
-
activerecord (3.2.3)
|
15
|
-
activemodel (= 3.2.3)
|
16
|
-
activesupport (= 3.2.3)
|
17
|
-
arel (~> 3.0.2)
|
18
|
-
tzinfo (~> 0.3.29)
|
19
|
-
activesupport (3.2.3)
|
20
|
-
i18n (~> 0.6)
|
21
|
-
multi_json (~> 1.0)
|
22
|
-
arel (3.0.2)
|
23
|
-
builder (3.0.0)
|
24
|
-
diff-lcs (1.1.3)
|
25
|
-
i18n (0.6.0)
|
26
|
-
multi_json (1.3.6)
|
27
|
-
pg (0.13.2)
|
28
|
-
rake (0.9.2.2)
|
29
|
-
rspec (2.10.0)
|
30
|
-
rspec-core (~> 2.10.0)
|
31
|
-
rspec-expectations (~> 2.10.0)
|
32
|
-
rspec-mocks (~> 2.10.0)
|
33
|
-
rspec-core (2.10.1)
|
34
|
-
rspec-expectations (2.10.0)
|
35
|
-
diff-lcs (~> 1.1.3)
|
36
|
-
rspec-mocks (2.10.1)
|
37
|
-
tzinfo (0.3.33)
|
38
|
-
|
39
|
-
PLATFORMS
|
40
|
-
ruby
|
41
|
-
|
42
|
-
DEPENDENCIES
|
43
|
-
pg_csv!
|
44
|
-
rake
|
45
|
-
rspec
|