pg_csv 0.1.2 → 0.1.3

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pg_csv (0.1.1)
+    pg_csv (0.1.3)
       activerecord
       pg
 

data/README.md

@@ -31,6 +31,7 @@ Options:
             => :gzip  - save file to gzip
             => :stream - save to stream
             => :file - just save to file = default
+            => :yield - return each row to block
 ```
 
 Examples:
@@ -38,7 +39,7 @@ Examples:
 PgCsv.new(:sql => User.good.to_sql).export('a1.csv')
 PgCsv.new(:sql => sql).export('a2.gz', :type => :gzip)
 PgCsv.new(:sql => sql).export('a3.csv', :temp_file => true)
-PgCsv.new(:sql => sql).export(nil, :type => :plain)
+PgCsv.new(:sql => sql, :type => :plain).export
 File.open("a4.csv", 'a'){|f| FastPgCsv.new(:sql => "select * from users").\
     export(f, :type => :stream) }
 PgCsv.new(:sql => sql).export('a5.csv', :delimiter => "\t")
@@ -54,4 +55,9 @@ Zlib::GzipWriter.open('some.gz') do |stream|
     e.export(stream, :connection => connection)
   end
 end
+
+# yield example
+PgCsv.new(:sql => sql, :type => :yield).export do |row|
+  puts row  
+end
 ```

data/benchmark/bench.rb

@@ -0,0 +1,48 @@
+require 'rubygems'
+require "bundler"
+Bundler.setup
+
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+require 'pg_csv'
+
+require 'benchmark'
+require 'fileutils'
+
+class PgCsv
+  def sql
+    ""
+  end
+  
+  def connection
+    1
+  end
+    
+  def load_data
+    n = o(:times).to_i
+    c = 0
+    n.times do
+      c += 1
+      yield(@block["#{c},#{c*2},#{c * 249},#{rand(100)},#{rand(n)},blablabla,hahah,ahah,ahaha,ahahah,ah,1.55234143\n"])
+    end
+  end
+end
+
+filename = "./blah.test.file"
+N = 500_000
+
+tm = Benchmark.realtime{ PgCsv.new(:times => N, :type => :file).export(filename) }
+puts "export file #{tm}"
+
+tm = Benchmark.realtime{ PgCsv.new(:times => N, :type => :gzip).export(filename) }
+puts "export gzip #{tm}"
+
+tm = Benchmark.realtime{ PgCsv.new(:times => N, :type => :plain).export }
+puts "export plain #{tm}"
+
+tm = Benchmark.realtime{ File.open(filename, 'w'){|f| PgCsv.new(:times => N, :type => :stream).export(f) } }
+puts "export stream #{tm}"
+
+tm = Benchmark.realtime{ PgCsv.new(:times => N, :type => :yield).export{|row| row } }
+puts "export yield #{tm}"
+
+FileUtils.rm(filename) rescue nil

data/lib/pg_csv.rb

@@ -2,28 +2,13 @@ require 'active_record'
 
 class PgCsv
 
-  # opts:
-  #   :sql        => "select u.*, p.* from users u, projects p where p.user_id = u.id order by email limit 100"
-  #   :connection => AR.connection
-  #   :delimiter  => ["\t", ",", ]
-  #   :header     => boolean, use pg header for fields?
-  #   :logger     => logger
-  #   :columns    => manual array of column names, ignore :header option
-
-  #   :temp_file  => boolean, generating throught temp file, final file appears by mv
-  #   :temp_dir   => path, ex: /tmp
-  
-  #   :type       => :plain - return full string
-  #               => :gzip  - save file to gzip
-  #               => :stream - save to stream
-  #               => :file - just save to file * default
-  
   def initialize(opts = {})
     @options = opts.symbolize_keys
   end
   
   # do export :to - filename or stream  
-  def export(to, opts = {})
+  def export(to = nil, opts = {}, &block)
+    @block = block || Proc.new{|x|x}
     @local_options = opts.symbolize_keys
     
     raise ":connection should be" unless connection
@@ -37,7 +22,7 @@ class PgCsv
 protected
 
   def with_temp_file(to, use_temp_file, tmp_dir)
-    if use_temp_file
+    if use_temp_file && [:file, :gzip].include?(type)
       check_str(to)
       
       require 'fileutils'
@@ -70,6 +55,7 @@ protected
         Zlib::GzipWriter.open(to, &exporter)
         
       when :stream
+        raise "'to' should be" unless to
         exporter[to]
         
       when :plain
@@ -78,6 +64,10 @@ protected
         exporter[sio]
         result = sio.string
         
+      when :yield
+        # not real saving anywhere, just yield each record
+        raise "block should be" unless @block
+        result = load_data{|_|}
     end
     
     info "<=== finished write #{to} in #{Time.now - start}"
@@ -94,22 +84,18 @@ protected
   end
   
   def export_to_stream(stream)
-    write_csv(stream)
+    count = write_csv(stream)
     stream.flush if stream.respond_to?(:flush)
+    
+    info "<= done exporting (#{count}) records."
   end
 
   def write_csv(stream)
-    count = 0
-    
     load_data do |row|
-      count += 1
-      stream.write prepare_row(row)
+      stream.write(row)
     end
-
-    info "<= done exporting (#{count}) records."
-    count
   end
-
+  
   def load_data
     info "#{query}"
     raw = connection.raw_connection
@@ -119,14 +105,17 @@ protected
     info "<= query"
 
     info "=> write data"
-    yield(columns_str) if columns_str
-    
+    yield(@block[columns_str]) if columns_str
+
+    count = 0    
     while row = raw.get_copy_data()
-      yield row
+      yield(@block[row])
+      count += 1
     end
     info "<= write data"
 
     q.clear
+    count
   end
 
   def query
@@ -140,10 +129,6 @@ DELIMITER '#{delimiter}'
     SQL
   end
 
-  def prepare_row(row)
-    row
-  end
-  
   def info(message)
     logger.info(message) if logger
   end

data/lib/pg_csv_version.rb

@@ -1,3 +1,3 @@
 class PgCsv
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

data/spec/pg_csv_spec.rb

@@ -114,6 +114,15 @@ describe PgCsv do
       PgCsv.new(:sql => @sql, :type => :file).export(@name)
       with_file(@name){|d| d.should == "4,5,6\n1,2,3\n" }            
     end
+    
+    it "yield export" do
+      rows = []
+      PgCsv.new(:sql => @sql, :type => :yield).export do |row|
+        rows << row
+      end.should == 2
+      
+      rows.should == ["4,5,6\n", "1,2,3\n"]
+    end
   end
 
   describe "integration specs" do
@@ -138,11 +147,10 @@ describe PgCsv do
   it "custom prepare row" do
     e = PgCsv.new(:sql => @sql)
       
-    def e.prepare_row(row)
+    e.export(@name) do |row|
       row.split(",").join("-|-")
     end
-      
-    e.export(@name)
+    
     with_file(@name){|d| d.should == "4-|-5-|-6\n1-|-2-|-3\n" }
   end
   

metadata

@@ -1,94 +1,95 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification 
 name: pg_csv
-version: !ruby/object:Gem::Version
-  version: 0.1.2
+version: !ruby/object:Gem::Version 
+  hash: 29
   prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 3
+  version: 0.1.3
 platform: ruby
-authors:
+authors: 
 - Makarchev Konstantin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-12 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
+
+date: 2012-06-13 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
   name: pg
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: activerecord
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
   type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: activerecord
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
   name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
   type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: rake
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-description: Fast AR/PostgreSQL csv export. Used pg function 'copy to csv'. Effective
-  on millions rows.
-email:
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id004
+description: Fast AR/PostgreSQL csv export. Used pg function 'copy to csv'. Effective on millions rows.
+email: 
 - kostya27@gmail.com
 executables: []
+
 extensions: []
+
 extra_rdoc_files: []
-files:
+
+files: 
 - .gitignore
 - Gemfile
 - Gemfile.lock
 - MIT-LICENSE
 - README.md
 - Rakefile
+- benchmark/bench.rb
 - lib/pg_csv.rb
 - lib/pg_csv_version.rb
 - pg_csv.gemspec
@@ -98,27 +99,36 @@ files:
 - spec/tmp/.gitkeep
 homepage: http://github.com/kostya/pg_csv
 licenses: []
+
 post_install_message: 
 rdoc_options: []
-require_paths:
+
+require_paths: 
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement
+required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
-required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
 requirements: []
+
 rubyforge_project: 
 rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
-summary: Fast AR/PostgreSQL csv export. Used pg function 'copy to csv'. Effective
-  on millions rows.
+summary: Fast AR/PostgreSQL csv export. Used pg function 'copy to csv'. Effective on millions rows.
 test_files: []
+