pg_csv 0.1.6 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 0e7903c3f73583176d0a969c4c6f2f4b9de2db78
4
+ data.tar.gz: 7fde910cb1a7639c0b2f3594a15fdadd095546e3
5
+ SHA512:
6
+ metadata.gz: c9fe3b47ccea65c02c81cd76b5028306c385985d5c9864a263a598d61a5623b6238e68527653a3a58a0bb45bd5cfa3f4f7022a292056ee74104da0716a738e4a
7
+ data.tar.gz: 8c8c3f0e1fbdf77f34fd59e424ae31573e42d090d42141f5bf13a4f77a1f92f0d792f7d9a3561e126f760de952972d682204e1674b14b31b912cddc620cb8322
data/README.md CHANGED
@@ -23,6 +23,7 @@ Options:
23
23
  :header => boolean, use pg header for fields?
24
24
  :logger => logger
25
25
  :columns => array of column names, ignore :header option
26
+ :encoding => encoding (default is pg_default), list of encodings: http://www.postgresql.org/docs/8.4/static/multibyte.html#CHARSET-TABLE
26
27
 
27
28
  :temp_file => boolean, generate throught temp file? final file appears by mv
28
29
  :temp_dir => for :temp_file, ex: '/tmp'
@@ -1,4 +1,5 @@
1
1
  require 'active_record'
2
+ require File.expand_path(File.join(File.dirname(__FILE__), 'pg_csv/version'))
2
3
 
3
4
  class PgCsv
4
5
 
@@ -7,20 +8,20 @@ class PgCsv
7
8
  def initialize(opts = {})
8
9
  @options = opts.symbolize_keys
9
10
  end
10
-
11
- # do export :to - filename or stream
11
+
12
+ # do export :to - filename or stream
12
13
  def export(to = nil, opts = {}, &row_proc)
13
14
  @row_proc = row_proc
14
15
  @local_options = opts.symbolize_keys
15
-
16
+
16
17
  raise ":connection should be" unless connection
17
18
  raise ":sql should be" unless sql
18
19
 
19
20
  with_temp_file?(to, temp_file, temp_dir) do |dest|
20
21
  export_to(dest)
21
- end
22
+ end
22
23
  end
23
-
24
+
24
25
  protected
25
26
 
26
27
  def with_temp_file?(to, use_temp_file, tmp_dir)
@@ -30,63 +31,61 @@ class PgCsv
30
31
  self.class.with_temp_file(to, tmp_dir) do |filename|
31
32
  yield(filename)
32
33
  end
33
-
34
+
34
35
  info "<=== moving export to #{to}"
35
36
  else
36
37
  yield(to)
37
38
  end
38
- end
39
+ end
39
40
 
40
41
  def export_to(to)
41
-
42
+
42
43
  start = Time.now
43
44
  info "===> start generate export #{to}, type: #{type}"
44
-
45
+
45
46
  result = nil
47
+ exporter = method(:export_to_stream).to_proc
46
48
 
47
49
  case type
48
-
50
+
49
51
  when :file
50
52
  check_str(to)
51
53
  File.open(to, 'w', &exporter)
52
-
54
+
53
55
  when :gzip
54
56
  check_str(to)
55
- Zlib::GzipWriter.open(to, &exporter)
56
-
57
+ require 'zlib'
58
+ ::Zlib::GzipWriter.open(to, &exporter)
59
+
57
60
  when :stream
58
61
  raise "'to' should be" unless to
59
62
  exporter[to]
60
-
63
+
61
64
  when :plain
62
65
  require 'stringio'
63
66
  sio = StringIO.new
64
67
  exporter[sio]
65
68
  result = sio.string
66
-
69
+
67
70
  when :yield
68
71
  # not real saving anywhere, just yield each record
69
72
  raise "row_proc should be" unless @row_proc
70
73
  result = load_data{|_|}
71
74
  end
72
-
75
+
73
76
  info "<=== finished write #{to} in #{Time.now - start}"
74
-
77
+
75
78
  result
76
79
  end
77
-
80
+
78
81
  def check_str(to)
79
82
  raise "'to' should be an string" unless to.is_a?(String)
80
83
  end
81
-
82
- def exporter
83
- method(:export_to_stream).to_proc
84
- end
85
-
84
+
86
85
  def export_to_stream(stream)
87
86
  count = write_csv(stream)
88
87
  stream.flush if stream.respond_to?(:flush) && count > 0
89
-
88
+
90
89
  info "<= done exporting (#{count}) records."
91
90
  end
92
91
 
@@ -95,11 +94,11 @@ class PgCsv
95
94
  stream.write(row)
96
95
  end
97
96
  end
98
-
97
+
99
98
  def load_data
100
99
  info "#{query}"
101
100
  raw = connection.raw_connection
102
-
101
+
103
102
  info "=> query"
104
103
  q = raw.exec(query)
105
104
  info "<= query"
@@ -109,7 +108,7 @@ class PgCsv
109
108
  yield(@row_proc ? @row_proc[columns_str] : columns_str)
110
109
  end
111
110
 
112
- count = 0
111
+ count = 0
113
112
  if @row_proc
114
113
  while row = raw.get_copy_data()
115
114
  yield(@row_proc[row])
@@ -134,16 +133,16 @@ class PgCsv
134
133
  ) TO STDOUT
135
134
  WITH CSV
136
135
  DELIMITER '#{delimiter}'
137
- #{use_pg_header? ? 'HEADER' : ''}
136
+ #{use_pg_header? ? 'HEADER' : ''} #{encoding ? "ENCODING '#{encoding}'" : ''}
138
137
  SQL
139
138
  end
140
139
 
141
140
  def info(message)
142
141
  logger.info(message) if logger
143
142
  end
144
-
143
+
145
144
  # ==== options/defaults =============
146
-
145
+
147
146
  def o(key)
148
147
  @local_options[key] || @options[key]
149
148
  end
@@ -151,7 +150,7 @@ class PgCsv
151
150
  def connection
152
151
  o(:connection) || (defined?(ActiveRecord::Base) ? ActiveRecord::Base.connection : nil)
153
152
  end
154
-
153
+
155
154
  def logger
156
155
  o(:logger)
157
156
  end
@@ -178,18 +177,22 @@ class PgCsv
178
177
  def delimiter
179
178
  o(:delimiter) || ','
180
179
  end
181
-
180
+
182
181
  def sql
183
182
  o(:sql)
184
183
  end
185
-
184
+
186
185
  def temp_file
187
186
  o(:temp_file)
188
187
  end
189
-
188
+
190
189
  def temp_dir
191
190
  o(:temp_dir) || '/tmp'
192
191
  end
192
+
193
+ def encoding
194
+ o(:encoding)
195
+ end
193
196
  end
194
197
 
195
198
  include Base
@@ -0,0 +1,3 @@
1
+ class PgCsv
2
+ VERSION = "0.1.7"
3
+ end
@@ -1,7 +1,7 @@
1
1
  $:.push File.expand_path("../lib", __FILE__)
2
2
 
3
3
  # Maintain your gem's version:
4
- require "pg_csv_version"
4
+ require "pg_csv/version"
5
5
 
6
6
  # Describe your gem and declare its dependencies:
7
7
  Gem::Specification.new do |s|
@@ -17,6 +17,7 @@ Gem::Specification.new do |s|
17
17
  s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
18
18
  s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
19
19
  s.require_paths = ["lib"]
20
+ s.license = "MIT"
20
21
 
21
22
  s.add_dependency "pg"
22
23
  s.add_dependency "activerecord"
@@ -12,11 +12,11 @@ class PgCsv
12
12
  def sql
13
13
  ""
14
14
  end
15
-
15
+
16
16
  def connection
17
17
  1
18
18
  end
19
-
19
+
20
20
  def load_data
21
21
  n = o(:times).to_i
22
22
  c = 0
@@ -0,0 +1,86 @@
1
+ require 'rubygems'
2
+ require "bundler"
3
+ Bundler.setup
4
+
5
+ $:.unshift(File.dirname(__FILE__) + '/../../lib')
6
+ require 'pg_csv'
7
+ require 'benchmark'
8
+ require 'fileutils'
9
+
10
+ N = 50000
11
+ T = 10
12
+
13
+ class Raw
14
+ def initialize
15
+ @c = 0
16
+ end
17
+
18
+ def exec(x)
19
+ q = ""
20
+ def q.clear
21
+ end
22
+ q
23
+ end
24
+
25
+ def get_copy_data()
26
+ @c += 1
27
+ if @c < N
28
+ "#{@c},#{@c*2},#{@c * 249},#{rand(100)},#{rand(@c)},blablabla,hahah,ahah,ahaha,ahahah,ah,1.55234143\n"
29
+ end
30
+ end
31
+ end
32
+
33
+ class Con
34
+ def raw_connection
35
+ @raw ||= Raw.new
36
+ end
37
+ end
38
+
39
+
40
+ $con = Con.new
41
+
42
+ class PgCsv
43
+
44
+ module FixConnection
45
+ def connection
46
+ @con ||= Con.new
47
+ end
48
+
49
+ def sql
50
+ ""
51
+ end
52
+ end
53
+
54
+ include FixConnection
55
+
56
+ end
57
+
58
+ class Stre
59
+ def write(str)
60
+ end
61
+ end
62
+
63
+ $stream = Stre.new
64
+
65
+
66
+ tm = Benchmark.realtime{ T.times{ PgCsv.new(:type => :plain).export } }
67
+ puts "export plain #{tm}"
68
+
69
+ tm = Benchmark.realtime{ T.times{ PgCsv.new(:type => :stream).export($stream) }}
70
+ puts "export stream #{tm}"
71
+
72
+ tm = Benchmark.realtime{ T.times{ PgCsv.new(:type => :yield).export{|row| row } }}
73
+ puts "export yield #{tm}"
74
+
75
+ =begin
76
+ ree:
77
+ export plain 5.67214202880859
78
+ export stream 5.46862411499023
79
+ export yield 5.83969807624817
80
+
81
+ 1.9.3
82
+ export plain 6.976197355
83
+ export stream 5.685256024
84
+ export yield 5.960436236
85
+
86
+ =end
@@ -1,3 +1,4 @@
1
+ # encoding: utf-8
1
2
  require File.dirname(__FILE__) + '/spec_helper'
2
3
 
3
4
  describe PgCsv do
@@ -6,36 +7,42 @@ describe PgCsv do
6
7
  Test.delete_all
7
8
  Test.create :a => 1, :b => 2, :c => 3
8
9
  Test.create :a => 4, :b => 5, :c => 6
9
-
10
+
10
11
  @name = tmp_dir + "1.csv"
11
12
  FileUtils.rm(@name) rescue nil
12
13
 
13
- @sql0 = "select a,b,c from tests order by a asc"
14
+ @sql0 = "select a,b,c from tests order by a asc"
14
15
  @sql = "select a,b,c from tests order by a desc"
15
16
  end
16
-
17
+
17
18
  after :each do
18
19
  FileUtils.rm(@name) rescue nil
19
20
  end
20
-
21
+
21
22
  describe "simple export" do
22
-
23
+
23
24
  it "1" do
24
25
  PgCsv.new(:sql => @sql0).export(@name)
25
26
  with_file(@name){|d| d.should == "1,2,3\n4,5,6\n" }
26
27
  end
27
-
28
+
28
29
  it "2" do
29
30
  PgCsv.new(:sql => @sql).export(@name)
30
31
  with_file(@name){|d| d.should == "4,5,6\n1,2,3\n" }
31
32
  end
32
-
33
+
33
34
  it "delimiter" do
34
35
  PgCsv.new(:sql => @sql).export(@name, :delimiter => "|")
35
36
  with_file(@name){|d| d.should == "4|5|6\n1|2|3\n" }
36
37
  end
37
-
38
-
38
+
39
+ it "encoding" do
40
+ Test.create! :a => 2, :b => 3, :c => 4, :d => "абсд"
41
+
42
+ PgCsv.new(:sql => "select d from tests where a = 2").export(@name, :encoding => "WIN1251")
43
+ with_file(@name){|d| d.force_encoding('cp1251').should == "абсд\n".encode('cp1251') }
44
+ end
45
+
39
46
  describe "headers" do
40
47
  it "header" do
41
48
  PgCsv.new(:sql => @sql).export(@name, :header => true)
@@ -46,41 +53,41 @@ describe PgCsv do
46
53
  PgCsv.new(:sql => @sql).export(@name, :columns => %w(q w e))
47
54
  with_file(@name){|d| d.should == "q,w,e\n4,5,6\n1,2,3\n" }
48
55
  end
49
-
56
+
50
57
  it "columns with header" do
51
58
  PgCsv.new(:sql => @sql).export(@name, :header => true, :columns => %w(q w e))
52
-
59
+
53
60
  with_file(@name) do |d|
54
61
  d.should == "q,w,e\n4,5,6\n1,2,3\n"
55
62
  end
56
63
  end
57
64
  end
58
-
65
+
59
66
  end
60
-
67
+
61
68
  describe "moving options no matter" do
62
69
  it "1" do
63
70
  PgCsv.new(:sql => @sql).export(@name, :delimiter => "|")
64
71
  with_file(@name){|d| d.should == "4|5|6\n1|2|3\n" }
65
72
  end
66
-
73
+
67
74
  it "2" do
68
75
  PgCsv.new(:delimiter => "|").export(@name, :sql => @sql)
69
76
  with_file(@name){|d| d.should == "4|5|6\n1|2|3\n"}
70
77
  end
71
78
  end
72
-
79
+
73
80
  describe "local options dont recover global" do
74
81
  it "test" do
75
82
  e = PgCsv.new(:sql => @sql, :delimiter => "*")
76
83
  e.export(@name, :delimiter => "|")
77
84
  with_file(@name){|d| d.should == "4|5|6\n1|2|3\n" }
78
-
85
+
79
86
  e.export(@name)
80
87
  with_file(@name){|d| d.should == "4*5*6\n1*2*3\n" }
81
88
  end
82
89
  end
83
-
90
+
84
91
  describe "using temp file" do
85
92
  it "at least file should return to target and set correct chmod" do
86
93
  File.exists?(@name).should be_false
@@ -104,33 +111,33 @@ describe PgCsv do
104
111
  with_gzfile(@name){|d| d.should == "4,5,6\n1,2,3\n" }
105
112
  sprintf("%o", File.stat(@name).mode).to_i.should >= 100660
106
113
  end
107
-
114
+
108
115
  it "plain export" do
109
116
  PgCsv.new(:sql => @sql, :type => :plain).export.should == "4,5,6\n1,2,3\n"
110
117
  end
111
-
118
+
112
119
  it "custom stream" do
113
120
  ex = PgCsv.new(:sql => @sql, :type => :stream)
114
121
  File.open(@name, 'w') do |stream|
115
122
  ex.export(stream)
116
123
  ex.export(stream, :sql => @sql0)
117
124
  end
118
-
125
+
119
126
  with_file(@name){|d| d.should == "4,5,6\n1,2,3\n1,2,3\n4,5,6\n" }
120
127
  end
121
-
128
+
122
129
  it "file as default" do
123
130
  PgCsv.new(:sql => @sql, :type => :file).export(@name)
124
- with_file(@name){|d| d.should == "4,5,6\n1,2,3\n" }
131
+ with_file(@name){|d| d.should == "4,5,6\n1,2,3\n" }
125
132
  sprintf("%o", File.stat(@name).mode).to_i.should >= 100660
126
133
  end
127
-
134
+
128
135
  it "yield export" do
129
136
  rows = []
130
137
  PgCsv.new(:sql => @sql, :type => :yield).export do |row|
131
138
  rows << row
132
139
  end.should == 2
133
-
140
+
134
141
  rows.should == ["4,5,6\n", "1,2,3\n"]
135
142
  end
136
143
  end
@@ -141,15 +148,15 @@ describe PgCsv do
141
148
  PgCsv.new(:sql => @sql, :type => :gzip).export(@name, :delimiter => "|", :columns => %w{q w e}, :temp_file => true, :temp_dir => tmp_dir)
142
149
  with_gzfile(@name){|d| d.should == "q|w|e\n4|5|6\n1|2|3\n" }
143
150
  end
144
-
151
+
145
152
  it "2" do
146
153
  Zlib::GzipWriter.open(@name) do |gz|
147
154
  e = PgCsv.new(:sql => @sql, :type => :stream)
148
-
155
+
149
156
  e.export(gz, :delimiter => "|", :columns => %w{q w e} )
150
157
  e.export(gz, :delimiter => "*", :sql => @sql0)
151
158
  end
152
-
159
+
153
160
  with_gzfile(@name){|d| d.should == "q|w|e\n4|5|6\n1|2|3\n1*2*3\n4*5*6\n" }
154
161
  end
155
162
 
@@ -159,15 +166,15 @@ describe PgCsv do
159
166
  with_gzfile(@name){|d| d.should == "" }
160
167
  end
161
168
  end
162
-
169
+
163
170
  it "custom row proc" do
164
171
  e = PgCsv.new(:sql => @sql)
165
-
172
+
166
173
  e.export(@name) do |row|
167
174
  row.split(",").join("-|-")
168
175
  end
169
-
176
+
170
177
  with_file(@name){|d| d.should == "4-|-5-|-6\n1-|-2-|-3\n" }
171
178
  end
172
-
179
+
173
180
  end
@@ -1,6 +1,6 @@
1
1
  require 'fileutils'
2
2
 
3
- conn = {'adapter' => 'postgresql', 'database' => 'pgcsv_test'}
3
+ conn = {'adapter' => 'postgresql', 'database' => 'pgcsv_test', :encoding => "unicode"}
4
4
  ActiveRecord::Base.establish_connection conn
5
5
 
6
6
  class Test < ActiveRecord::Base
@@ -12,6 +12,7 @@ def pg_create_schema
12
12
  t.integer :a
13
13
  t.integer :b
14
14
  t.integer :c
15
+ t.string :d
15
16
  end
16
17
  end
17
18
 
@@ -34,7 +35,7 @@ def with_file(name)
34
35
  yield data
35
36
  q = 2
36
37
  end
37
-
38
+
38
39
  q.should == 2
39
40
  end
40
41
 
metadata CHANGED
@@ -1,78 +1,69 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pg_csv
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.6
5
- prerelease:
4
+ version: 0.1.7
6
5
  platform: ruby
7
6
  authors:
8
7
  - Makarchev Konstantin
9
8
  autorequire:
10
9
  bindir: bin
11
10
  cert_chain: []
12
- date: 2012-08-29 00:00:00.000000000 Z
11
+ date: 2013-08-23 00:00:00.000000000 Z
13
12
  dependencies:
14
13
  - !ruby/object:Gem::Dependency
15
14
  name: pg
16
15
  requirement: !ruby/object:Gem::Requirement
17
- none: false
18
16
  requirements:
19
- - - ! '>='
17
+ - - '>='
20
18
  - !ruby/object:Gem::Version
21
19
  version: '0'
22
20
  type: :runtime
23
21
  prerelease: false
24
22
  version_requirements: !ruby/object:Gem::Requirement
25
- none: false
26
23
  requirements:
27
- - - ! '>='
24
+ - - '>='
28
25
  - !ruby/object:Gem::Version
29
26
  version: '0'
30
27
  - !ruby/object:Gem::Dependency
31
28
  name: activerecord
32
29
  requirement: !ruby/object:Gem::Requirement
33
- none: false
34
30
  requirements:
35
- - - ! '>='
31
+ - - '>='
36
32
  - !ruby/object:Gem::Version
37
33
  version: '0'
38
34
  type: :runtime
39
35
  prerelease: false
40
36
  version_requirements: !ruby/object:Gem::Requirement
41
- none: false
42
37
  requirements:
43
- - - ! '>='
38
+ - - '>='
44
39
  - !ruby/object:Gem::Version
45
40
  version: '0'
46
41
  - !ruby/object:Gem::Dependency
47
42
  name: rspec
48
43
  requirement: !ruby/object:Gem::Requirement
49
- none: false
50
44
  requirements:
51
- - - ! '>='
45
+ - - '>='
52
46
  - !ruby/object:Gem::Version
53
47
  version: '0'
54
48
  type: :development
55
49
  prerelease: false
56
50
  version_requirements: !ruby/object:Gem::Requirement
57
- none: false
58
51
  requirements:
59
- - - ! '>='
52
+ - - '>='
60
53
  - !ruby/object:Gem::Version
61
54
  version: '0'
62
55
  - !ruby/object:Gem::Dependency
63
56
  name: rake
64
57
  requirement: !ruby/object:Gem::Requirement
65
- none: false
66
58
  requirements:
67
- - - ! '>='
59
+ - - '>='
68
60
  - !ruby/object:Gem::Version
69
61
  version: '0'
70
62
  type: :development
71
63
  prerelease: false
72
64
  version_requirements: !ruby/object:Gem::Requirement
73
- none: false
74
65
  requirements:
75
- - - ! '>='
66
+ - - '>='
76
67
  - !ruby/object:Gem::Version
77
68
  version: '0'
78
69
  description: Fast AR/PostgreSQL csv export. Used pg function 'copy to csv'. Effective
@@ -89,42 +80,37 @@ files:
89
80
  - README.md
90
81
  - Rakefile
91
82
  - lib/pg_csv.rb
92
- - lib/pg_csv_version.rb
83
+ - lib/pg_csv/version.rb
93
84
  - pg_csv.gemspec
94
85
  - spec/benchmark/bench.rb
86
+ - spec/benchmark/raw_bench.rb
95
87
  - spec/pg_csv_spec.rb
96
88
  - spec/spec_helper.rb
97
89
  - spec/spec_support.rb
98
90
  - spec/tmp/.gitkeep
99
91
  homepage: http://github.com/kostya/pg_csv
100
- licenses: []
92
+ licenses:
93
+ - MIT
94
+ metadata: {}
101
95
  post_install_message:
102
96
  rdoc_options: []
103
97
  require_paths:
104
98
  - lib
105
99
  required_ruby_version: !ruby/object:Gem::Requirement
106
- none: false
107
100
  requirements:
108
- - - ! '>='
101
+ - - '>='
109
102
  - !ruby/object:Gem::Version
110
103
  version: '0'
111
- segments:
112
- - 0
113
- hash: -2157019647217736024
114
104
  required_rubygems_version: !ruby/object:Gem::Requirement
115
- none: false
116
105
  requirements:
117
- - - ! '>='
106
+ - - '>='
118
107
  - !ruby/object:Gem::Version
119
108
  version: '0'
120
- segments:
121
- - 0
122
- hash: -2157019647217736024
123
109
  requirements: []
124
110
  rubyforge_project:
125
- rubygems_version: 1.8.23
111
+ rubygems_version: 2.0.2
126
112
  signing_key:
127
- specification_version: 3
113
+ specification_version: 4
128
114
  summary: Fast AR/PostgreSQL csv export. Used pg function 'copy to csv'. Effective
129
115
  on millions rows.
130
116
  test_files: []
@@ -1,3 +0,0 @@
1
- class PgCsv
2
- VERSION = "0.1.6"
3
- end