pg_csv 0.1.6 → 0.1.7

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0e7903c3f73583176d0a969c4c6f2f4b9de2db78
+  data.tar.gz: 7fde910cb1a7639c0b2f3594a15fdadd095546e3
+SHA512:
+  metadata.gz: c9fe3b47ccea65c02c81cd76b5028306c385985d5c9864a263a598d61a5623b6238e68527653a3a58a0bb45bd5cfa3f4f7022a292056ee74104da0716a738e4a
+  data.tar.gz: 8c8c3f0e1fbdf77f34fd59e424ae31573e42d090d42141f5bf13a4f77a1f92f0d792f7d9a3561e126f760de952972d682204e1674b14b31b912cddc620cb8322

data/README.md

@@ -23,6 +23,7 @@ Options:
 :header     => boolean, use pg header for fields?
 :logger     => logger
 :columns    => array of column names, ignore :header option
+:encoding   => encoding (default is pg_default), list of encodings: http://www.postgresql.org/docs/8.4/static/multibyte.html#CHARSET-TABLE
 
 :temp_file  => boolean, generate throught temp file? final file appears by mv
 :temp_dir   => for :temp_file, ex: '/tmp'

data/lib/pg_csv.rb

@@ -1,4 +1,5 @@
 require 'active_record'
+require File.expand_path(File.join(File.dirname(__FILE__), 'pg_csv/version'))
 
 class PgCsv
 
@@ -7,20 +8,20 @@ class PgCsv
     def initialize(opts = {})
       @options = opts.symbolize_keys
     end
-    
-    # do export :to - filename or stream  
+
+    # do export :to - filename or stream
     def export(to = nil, opts = {}, &row_proc)
       @row_proc = row_proc
       @local_options = opts.symbolize_keys
-      
+
       raise ":connection should be" unless connection
       raise ":sql should be" unless sql
 
       with_temp_file?(to, temp_file, temp_dir) do |dest|
         export_to(dest)
-      end        
+      end
     end
-    
+
   protected
 
     def with_temp_file?(to, use_temp_file, tmp_dir)
@@ -30,63 +31,61 @@ class PgCsv
         self.class.with_temp_file(to, tmp_dir) do |filename|
           yield(filename)
         end
-        
+
         info "<=== moving export to #{to}"
       else
         yield(to)
       end
-    end                                                            
+    end
 
     def export_to(to)
-    
+
       start = Time.now
       info "===> start generate export #{to}, type: #{type}"
-      
+
       result = nil
+      exporter = method(:export_to_stream).to_proc
 
       case type
-      
+
         when :file
           check_str(to)
           File.open(to, 'w', &exporter)
-          
+
         when :gzip
           check_str(to)
-          Zlib::GzipWriter.open(to, &exporter)
-          
+          require 'zlib'
+          ::Zlib::GzipWriter.open(to, &exporter)
+
         when :stream
           raise "'to' should be" unless to
           exporter[to]
-          
+
         when :plain
           require 'stringio'
           sio = StringIO.new
           exporter[sio]
           result = sio.string
-          
+
         when :yield
           # not real saving anywhere, just yield each record
           raise "row_proc should be" unless @row_proc
           result = load_data{|_|}
       end
-      
+
       info "<=== finished write #{to} in #{Time.now - start}"
-      
+
       result
     end
-    
+
     def check_str(to)
       raise "'to' should be an string" unless to.is_a?(String)
     end
-    
-    def exporter
-      method(:export_to_stream).to_proc
-    end
-    
+
     def export_to_stream(stream)
       count = write_csv(stream)
       stream.flush if stream.respond_to?(:flush) && count > 0
-      
+
       info "<= done exporting (#{count}) records."
     end
 
@@ -95,11 +94,11 @@ class PgCsv
         stream.write(row)
       end
     end
-    
+
     def load_data
       info "#{query}"
       raw = connection.raw_connection
-      
+
       info "=> query"
       q = raw.exec(query)
       info "<= query"
@@ -109,7 +108,7 @@ class PgCsv
         yield(@row_proc ? @row_proc[columns_str] : columns_str)
       end
 
-      count = 0    
+      count = 0
       if @row_proc
         while row = raw.get_copy_data()
           yield(@row_proc[row])
@@ -134,16 +133,16 @@ class PgCsv
   ) TO STDOUT
   WITH CSV
   DELIMITER '#{delimiter}'
-  #{use_pg_header? ? 'HEADER' : ''}
+  #{use_pg_header? ? 'HEADER' : ''} #{encoding ? "ENCODING '#{encoding}'" : ''}
       SQL
     end
 
     def info(message)
       logger.info(message) if logger
     end
-    
+
     # ==== options/defaults =============
-    
+
     def o(key)
       @local_options[key] || @options[key]
     end
@@ -151,7 +150,7 @@ class PgCsv
     def connection
       o(:connection) || (defined?(ActiveRecord::Base) ? ActiveRecord::Base.connection : nil)
     end
-    
+
     def logger
       o(:logger)
     end
@@ -178,18 +177,22 @@ class PgCsv
     def delimiter
       o(:delimiter) || ','
     end
-    
+
     def sql
       o(:sql)
     end
-    
+
     def temp_file
       o(:temp_file)
     end
-    
+
     def temp_dir
       o(:temp_dir) || '/tmp'
     end
+
+    def encoding
+      o(:encoding)
+    end
   end
 
   include Base

data/lib/pg_csv/version.rb

@@ -0,0 +1,3 @@
+class PgCsv
+  VERSION = "0.1.7"
+end

data/pg_csv.gemspec

@@ -1,7 +1,7 @@
 $:.push File.expand_path("../lib", __FILE__)
 
 # Maintain your gem's version:
-require "pg_csv_version"
+require "pg_csv/version"
 
 # Describe your gem and declare its dependencies:
 Gem::Specification.new do |s|
@@ -17,6 +17,7 @@ Gem::Specification.new do |s|
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
+  s.license       = "MIT"
   
   s.add_dependency "pg"
   s.add_dependency "activerecord"

data/spec/benchmark/bench.rb

@@ -12,11 +12,11 @@ class PgCsv
   def sql
     ""
   end
-  
+
   def connection
     1
   end
-    
+
   def load_data
     n = o(:times).to_i
     c = 0

data/spec/benchmark/raw_bench.rb

@@ -0,0 +1,86 @@
+require 'rubygems'
+require "bundler"
+Bundler.setup
+
+$:.unshift(File.dirname(__FILE__) + '/../../lib')
+require 'pg_csv'
+require 'benchmark'
+require 'fileutils'
+
+N = 50000
+T = 10
+
+class Raw
+  def initialize
+    @c = 0
+  end
+
+  def exec(x)
+    q = ""
+    def q.clear
+    end
+    q
+  end
+
+  def get_copy_data()
+    @c += 1
+    if @c < N
+      "#{@c},#{@c*2},#{@c * 249},#{rand(100)},#{rand(@c)},blablabla,hahah,ahah,ahaha,ahahah,ah,1.55234143\n"
+    end
+  end
+end
+
+class Con
+  def raw_connection
+    @raw ||= Raw.new
+  end
+end
+
+
+$con = Con.new
+
+class PgCsv
+
+  module FixConnection
+    def connection
+      @con ||= Con.new
+    end
+
+    def sql
+      ""
+    end
+  end
+
+  include FixConnection
+
+end
+
+class Stre
+  def write(str)
+  end
+end
+
+$stream = Stre.new
+
+
+tm = Benchmark.realtime{ T.times{ PgCsv.new(:type => :plain).export } }
+puts "export plain #{tm}"
+
+tm = Benchmark.realtime{ T.times{ PgCsv.new(:type => :stream).export($stream) }}
+puts "export stream #{tm}"
+
+tm = Benchmark.realtime{ T.times{ PgCsv.new(:type => :yield).export{|row| row } }}
+puts "export yield #{tm}"
+
+=begin
+ree:
+export plain 5.67214202880859
+export stream 5.46862411499023
+export yield 5.83969807624817
+
+1.9.3
+export plain 6.976197355
+export stream 5.685256024
+export yield 5.960436236
+
+=end

data/spec/pg_csv_spec.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require File.dirname(__FILE__) + '/spec_helper'
 
 describe PgCsv do
@@ -6,36 +7,42 @@ describe PgCsv do
     Test.delete_all
     Test.create :a => 1, :b => 2, :c => 3
     Test.create :a => 4, :b => 5, :c => 6
-    
+
     @name = tmp_dir + "1.csv"
     FileUtils.rm(@name) rescue nil
 
-    @sql0 = "select a,b,c from tests order by a asc"    
+    @sql0 = "select a,b,c from tests order by a asc"
     @sql = "select a,b,c from tests order by a desc"
   end
-  
+
   after :each do
     FileUtils.rm(@name) rescue nil
   end
-  
+
   describe "simple export" do
-  
+
     it "1" do
       PgCsv.new(:sql => @sql0).export(@name)
       with_file(@name){|d| d.should == "1,2,3\n4,5,6\n" }
     end
-    
+
     it "2" do
       PgCsv.new(:sql => @sql).export(@name)
       with_file(@name){|d| d.should == "4,5,6\n1,2,3\n" }
     end
-    
+
     it "delimiter" do
       PgCsv.new(:sql => @sql).export(@name, :delimiter => "|")
       with_file(@name){|d| d.should == "4|5|6\n1|2|3\n" }
     end
-    
-    
+
+    it "encoding" do
+      Test.create! :a => 2, :b => 3, :c => 4, :d => "абсд"
+
+      PgCsv.new(:sql => "select d from tests where a = 2").export(@name, :encoding => "WIN1251")
+      with_file(@name){|d| d.force_encoding('cp1251').should == "абсд\n".encode('cp1251') }
+    end
+
     describe "headers" do
       it "header" do
         PgCsv.new(:sql => @sql).export(@name, :header => true)
@@ -46,41 +53,41 @@ describe PgCsv do
         PgCsv.new(:sql => @sql).export(@name, :columns => %w(q w e))
         with_file(@name){|d| d.should == "q,w,e\n4,5,6\n1,2,3\n" }
       end
-      
+
       it "columns with header" do
         PgCsv.new(:sql => @sql).export(@name, :header => true, :columns => %w(q w e))
-                    
+
         with_file(@name) do |d|
           d.should == "q,w,e\n4,5,6\n1,2,3\n"
         end
       end
     end
-    
+
   end
-  
+
   describe "moving options no matter" do
     it "1" do
       PgCsv.new(:sql => @sql).export(@name, :delimiter => "|")
       with_file(@name){|d| d.should == "4|5|6\n1|2|3\n" }
     end
-    
+
     it "2" do
       PgCsv.new(:delimiter => "|").export(@name, :sql => @sql)
       with_file(@name){|d| d.should == "4|5|6\n1|2|3\n"}
     end
   end
-  
+
   describe "local options dont recover global" do
     it "test" do
       e = PgCsv.new(:sql => @sql, :delimiter => "*")
       e.export(@name, :delimiter => "|")
       with_file(@name){|d| d.should == "4|5|6\n1|2|3\n" }
-      
+
       e.export(@name)
       with_file(@name){|d| d.should == "4*5*6\n1*2*3\n" }
     end
   end
-  
+
   describe "using temp file" do
     it "at least file should return to target and set correct chmod" do
       File.exists?(@name).should be_false
@@ -104,33 +111,33 @@ describe PgCsv do
       with_gzfile(@name){|d| d.should == "4,5,6\n1,2,3\n" }
       sprintf("%o", File.stat(@name).mode).to_i.should >= 100660
     end
-    
+
     it "plain export" do
       PgCsv.new(:sql => @sql, :type => :plain).export.should == "4,5,6\n1,2,3\n"
     end
-    
+
     it "custom stream" do
       ex = PgCsv.new(:sql => @sql, :type => :stream)
       File.open(@name, 'w') do |stream|
         ex.export(stream)
         ex.export(stream, :sql => @sql0)
       end
-      
+
       with_file(@name){|d| d.should == "4,5,6\n1,2,3\n1,2,3\n4,5,6\n" }
     end
-    
+
     it "file as default" do
       PgCsv.new(:sql => @sql, :type => :file).export(@name)
-      with_file(@name){|d| d.should == "4,5,6\n1,2,3\n" }            
+      with_file(@name){|d| d.should == "4,5,6\n1,2,3\n" }
       sprintf("%o", File.stat(@name).mode).to_i.should >= 100660
     end
-    
+
     it "yield export" do
       rows = []
       PgCsv.new(:sql => @sql, :type => :yield).export do |row|
         rows << row
       end.should == 2
-      
+
       rows.should == ["4,5,6\n", "1,2,3\n"]
     end
   end
@@ -141,15 +148,15 @@ describe PgCsv do
       PgCsv.new(:sql => @sql, :type => :gzip).export(@name, :delimiter => "|", :columns => %w{q w e}, :temp_file => true, :temp_dir => tmp_dir)
       with_gzfile(@name){|d| d.should == "q|w|e\n4|5|6\n1|2|3\n" }
     end
-    
+
     it "2" do
       Zlib::GzipWriter.open(@name) do |gz|
         e = PgCsv.new(:sql => @sql, :type => :stream)
-        
+
         e.export(gz, :delimiter => "|", :columns => %w{q w e} )
         e.export(gz, :delimiter => "*", :sql => @sql0)
       end
-      
+
       with_gzfile(@name){|d| d.should == "q|w|e\n4|5|6\n1|2|3\n1*2*3\n4*5*6\n" }
     end
 
@@ -159,15 +166,15 @@ describe PgCsv do
       with_gzfile(@name){|d| d.should == "" }
     end
   end
-  
+
   it "custom row proc" do
     e = PgCsv.new(:sql => @sql)
-      
+
     e.export(@name) do |row|
       row.split(",").join("-|-")
     end
-    
+
     with_file(@name){|d| d.should == "4-|-5-|-6\n1-|-2-|-3\n" }
   end
-  
+
 end

data/spec/spec_support.rb

@@ -1,6 +1,6 @@
 require 'fileutils'
 
-conn = {'adapter' => 'postgresql', 'database' => 'pgcsv_test'}
+conn = {'adapter' => 'postgresql', 'database' => 'pgcsv_test', :encoding => "unicode"}
 ActiveRecord::Base.establish_connection conn
 
 class Test < ActiveRecord::Base
@@ -12,6 +12,7 @@ def pg_create_schema
     t.integer :a
     t.integer :b
     t.integer :c
+    t.string  :d
   end
 end
 
@@ -34,7 +35,7 @@ def with_file(name)
     yield data
     q = 2
   end
-  
+
   q.should == 2
 end
 

metadata

@@ -1,78 +1,69 @@
 --- !ruby/object:Gem::Specification
 name: pg_csv
 version: !ruby/object:Gem::Version
-  version: 0.1.6
-  prerelease: 
+  version: 0.1.7
 platform: ruby
 authors:
 - Makarchev Konstantin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-29 00:00:00.000000000 Z
+date: 2013-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: Fast AR/PostgreSQL csv export. Used pg function 'copy to csv'. Effective
@@ -89,42 +80,37 @@ files:
 - README.md
 - Rakefile
 - lib/pg_csv.rb
-- lib/pg_csv_version.rb
+- lib/pg_csv/version.rb
 - pg_csv.gemspec
 - spec/benchmark/bench.rb
+- spec/benchmark/raw_bench.rb
 - spec/pg_csv_spec.rb
 - spec/spec_helper.rb
 - spec/spec_support.rb
 - spec/tmp/.gitkeep
 homepage: http://github.com/kostya/pg_csv
-licenses: []
+licenses:
+- MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -2157019647217736024
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -2157019647217736024
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.0.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Fast AR/PostgreSQL csv export. Used pg function 'copy to csv'. Effective
   on millions rows.
 test_files: []

data/lib/pg_csv_version.rb

@@ -1,3 +0,0 @@
-class PgCsv
-  VERSION = "0.1.6"
-end