pg_csv 0.1

data/.gitignore

@@ -0,0 +1,13 @@
+.bundle/
+log/*.log
+pkg/
+test/dummy/db/*.sqlite3
+test/dummy/log/*.log
+test/dummy/tmp/
+test/dummy/.sass-cache
+*.csv
+*.gz
+*.log
+tmp/*
+!tmp/.gitkeep
+*.gem

data/Gemfile

@@ -0,0 +1,5 @@
+source "http://rubygems.org"
+
+gemspec
+
+gem 'activerecord', :require => "active_record"

data/Gemfile.lock

@@ -0,0 +1,46 @@
+PATH
+  remote: .
+  specs:
+    pg_csv (0.1)
+      activerecord
+      pg
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activemodel (3.2.3)
+      activesupport (= 3.2.3)
+      builder (~> 3.0.0)
+    activerecord (3.2.3)
+      activemodel (= 3.2.3)
+      activesupport (= 3.2.3)
+      arel (~> 3.0.2)
+      tzinfo (~> 0.3.29)
+    activesupport (3.2.3)
+      i18n (~> 0.6)
+      multi_json (~> 1.0)
+    arel (3.0.2)
+    builder (3.0.0)
+    diff-lcs (1.1.3)
+    i18n (0.6.0)
+    multi_json (1.3.6)
+    pg (0.13.2)
+    rake (0.9.2.2)
+    rspec (2.10.0)
+      rspec-core (~> 2.10.0)
+      rspec-expectations (~> 2.10.0)
+      rspec-mocks (~> 2.10.0)
+    rspec-core (2.10.1)
+    rspec-expectations (2.10.0)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.10.1)
+    tzinfo (0.3.33)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord
+  pg_csv!
+  rake
+  rspec

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2012 Makarchev K
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,57 @@
+PgCsv
+=====
+
+Fast AR/PostgreSQL csv export. Uses pg function 'copy to csv'. Effective on millions rows.
+
+Gemfile:
+``` ruby
+gem 'pg_csv'
+``` 
+
+Usage:
+``` ruby
+PgCsv.new(opts).export(to, opts)
+```
+
+'to' is a stream or filename
+
+Options:
+``` ruby
+:sql        => "select p.* from users u, projects p where p.user_id = u.id order by email limit 10"
+:connection => AR.connection
+:delimiter  => ["\t", ",", ]
+:header     => boolean, use pg header for fields?
+:logger     => logger
+:columns    => manual array of column names, ignore :header option
+
+:temp_file  => boolean, generate throught temp file, final file appears by mv
+:temp_dir   => for :temp_file, ex: '/tmp'
+
+:type       => :plain - return full string
+            => :gzip  - save file to gzip
+            => :stream - save to stream
+            => :file - just save to file = default
+```
+
+Examples:
+``` ruby
+PgCsv.new(:sql => User.good.to_sql).export('a1.csv')
+PgCsv.new(:sql => sql).export('a2.gz', :type => :gzip)
+PgCsv.new(:sql => sql).export('a3.csv', :temp_file => true)
+PgCsv.new(:sql => sql).export(nil, :type => :plain)
+File.open("a4.csv", 'a'){|f| FastPgCsv.new(:sql => "select * from users").\
+    export(f, :type => :stream) }
+PgCsv.new(:sql => sql).export('a5.csv', :delimiter => "\t")
+PgCsv.new(:sql => sql).export('a6.csv', :header => true)
+PgCsv.new(:sql => sql).export('a7.csv', :columns => %w{id a b c})
+PgCsv.new(:sql => sql, :connection => SomeDb.connection, :columns => %w{id a b c}, :delimiter => "|").\
+    export('a8.gz', :type => :gzip, :temp_file => true)
+
+# example collect from shards
+Zlib::GzipWriter.open('some.gz') do |stream|
+  e = PgCsv.new(:sql => sql, :type => :stream)
+  ConnectionPool.each_shard do |connection|
+    e.export(stream, :connection => connection)
+  end
+end
+```

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rspec/core/rake_task'
+task :default => :spec
+RSpec::Core::RakeTask.new(:spec)

data/lib/pg_csv.rb

@@ -0,0 +1,182 @@
+require 'active_record'
+
+class PgCsv
+
+  # opts:
+  #   :sql        => "select u.*, p.* from users u, projects p where p.user_id = u.id order by email limit 100"
+  #   :connection => AR.connection
+  #   :delimiter  => ["\t", ",", ]
+  #   :header     => boolean, use pg header for fields?
+  #   :logger     => logger
+  #   :columns    => manual array of column names, ignore :header option
+
+  #   :temp_file  => boolean, generating throught temp file, final file appears by mv
+  #   :temp_dir   => path, ex: /tmp
+  
+  #   :type       => :plain - return full string
+  #               => :gzip  - save file to gzip
+  #               => :stream - save to stream
+  #               => :file - just save to file * default
+  
+  def initialize(opts = {})
+    @options = opts
+  end
+  
+  # do export :to - filename or stream  
+  def export(to, opts = {})
+    @local_options = opts
+
+    with_temp_file(to, o(:temp_file), o(:temp_dir)) do |_to|
+      export_to(_to)
+    end        
+  end
+  
+protected
+
+  def with_temp_file(to, use_temp_file, tmp_dir)
+    if use_temp_file
+      check_to_str(to)
+      
+      require 'fileutils'
+      require 'tempfile'
+                    
+      tempfile = Tempfile.new("pg_csv", tmp_dir || '/tmp')
+      yield(tempfile.path)
+      FileUtils.mv(tempfile.path, to)
+      info "<=== moving export to #{to}"
+    else
+      yield(to)
+    end
+  end                                                            
+
+  def export_to(to)
+    exporter = method(:export_to_stream).to_proc
+
+    start = Time.now
+    info "===> start generate export #{to}, type: #{type}"
+    
+    result = nil
+
+    case type
+    
+      when :file
+        check_to_str(to)
+        File.open(to, 'w', &exporter)
+        
+      when :gzip
+        check_to_str(to)        
+        Zlib::GzipWriter.open(to, &exporter)
+        
+      when :stream
+        exporter[to]
+        
+      when :plain
+        require 'stringio'
+        sio = StringIO.new
+        exporter[sio]
+        result = sio.string
+        
+    end
+    
+    info "<=== finished write #{to} in #{Time.now - start}"
+    
+    result
+  end
+  
+  def check_to_str(to)
+    raise "to should be an string" unless to.is_a?(String)
+  end
+  
+  def export_to_stream(stream)
+    write_csv(stream)
+    stream.flush
+  end
+
+  def write_csv(stream)
+    count = 0
+    
+    load_data do |row|
+      count += 1
+      stream.write prepare_row(row)
+    end
+
+    info "<= done exporting (#{count}) records."
+    count
+  end
+
+  def load_data
+    info "#{query}"
+    conn = connection.raw_connection
+    
+    info "=> query"
+    q = conn.exec(query)
+    info "<= query"
+
+    info "=> write data"
+    yield(columns_str) if columns_str
+    
+    while row = conn.get_copy_data()
+      yield row
+    end
+    info "<= write data"
+
+    q.clear
+  end
+
+  def query
+    <<-SQL
+COPY (
+  #{o(:sql)}
+) TO STDOUT
+WITH CSV
+DELIMITER '#{delimiter}'
+#{use_pg_header? ? 'HEADER' : ''}
+    SQL
+  end
+
+  def prepare_row(row)
+    row
+  end
+  
+  def info(message)
+    logger.info(message) if logger
+  end
+  
+  # ==== options/defaults =============
+  
+  def o(key)
+    @local_options[key] || @options[key]
+  end
+
+  def connection
+    o(:connection) || (defined?(ActiveRecord::Base) ? ActiveRecord::Base.connection : nil)
+  end
+  
+  def logger
+    o(:logger)
+  end
+
+  def type
+    o(:type) || :file
+  end
+
+  def use_pg_header?
+    o(:header) && !o(:columns)
+  end
+
+  def columns_str
+    if o(:columns)
+      col = o(:columns)
+      if col.is_a?(Array)
+        col.join(delimiter) + "\n"
+      else
+        col + "\n"
+      end
+    end
+  end
+
+  def delimiter
+    o(:delimiter) || ','
+  end
+  
+end

data/lib/pg_csv_version.rb

@@ -0,0 +1,3 @@
+class PgCsv
+  VERSION = "0.1"
+end

data/pg_csv.gemspec

@@ -0,0 +1,26 @@
+$:.push File.expand_path("../lib", __FILE__)
+
+# Maintain your gem's version:
+require "pg_csv_version"
+
+# Describe your gem and declare its dependencies:
+Gem::Specification.new do |s|
+  s.name        = "pg_csv"
+  s.version     = PgCsv::VERSION
+  s.authors     = ["Makarchev Konstantin"]
+  s.email       = ["kostya27@gmail.com"]
+  s.homepage    = "http://github.com/kostya/pg_csv"
+  s.summary     = "Fast AR/PostgreSQL csv export. Used pg function 'copy to csv'. Effective on millions rows."
+  s.description = "Fast AR/PostgreSQL csv export. Used pg function 'copy to csv'. Effective on millions rows."
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  
+  s.add_dependency "pg"
+  s.add_dependency "activerecord"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "rake"
+  
+end

data/spec/pg_csv_spec.rb

@@ -0,0 +1,124 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe PgCsv do
+
+  before :each do
+    Test.delete_all
+    Test.create :a => 1, :b => 2, :c => 3
+    Test.create :a => 4, :b => 5, :c => 6
+    
+    @name = tmp_dir + "1.csv"
+    @gzname = tmp_dir + "1.gz"
+    
+    FileUtils.rm(@name) rescue nil
+    FileUtils.rm(@gzname) rescue nil
+
+    @sql0 = "select a,b,c from tests order by a asc"    
+    @sql = "select a,b,c from tests order by a desc"
+  end
+  
+  after :each do
+    FileUtils.rm(@name) rescue nil
+    FileUtils.rm(@gzname) rescue nil            
+  end
+  
+  
+  describe "simple export" do
+  
+    it "1" do
+      PgCsv.new(:sql => @sql0).export(@name)
+      with_file(@name){|d| d.should == "1,2,3\n4,5,6\n" }
+    end
+    
+    it "2" do
+      PgCsv.new(:sql => @sql).export(@name)
+      with_file(@name){|d| d.should == "4,5,6\n1,2,3\n" }
+    end
+    
+    it "delimiter" do
+      PgCsv.new(:sql => @sql).export(@name, :delimiter => "|")
+      with_file(@name){|d| d.should == "4|5|6\n1|2|3\n" }
+    end
+    
+    
+    describe "headers" do
+      it "header" do
+        PgCsv.new(:sql => @sql).export(@name, :header => true)
+        with_file(@name){|d| d.should == "a,b,c\n4,5,6\n1,2,3\n" }
+      end
+
+      it "columns" do
+        PgCsv.new(:sql => @sql).export(@name, :columns => %w(q w e))
+        with_file(@name){|d| d.should == "q,w,e\n4,5,6\n1,2,3\n" }
+      end
+      
+      it "columns with header" do
+        PgCsv.new(:sql => @sql).export(@name, :header => true, :columns => %w(q w e))
+                    
+        with_file(@name) do |d|
+          d.should == "q,w,e\n4,5,6\n1,2,3\n"
+        end
+      end
+    end
+    
+  end
+  
+  describe "moving options no matter" do
+    it "1" do
+      PgCsv.new(:sql => @sql).export(@name, :delimiter => "|")
+      with_file(@name){|d| d.should == "4|5|6\n1|2|3\n" }
+    end
+    
+    it "2" do
+      PgCsv.new(:delimiter => "|").export(@name, :sql => @sql)
+      with_file(@name){|d| d.should == "4|5|6\n1|2|3\n"}
+    end
+  end
+  
+  describe "local options dont recover global" do
+    it "test" do
+      e = PgCsv.new(:sql => @sql, :delimiter => "*")
+      e.export(@name, :delimiter => "|")
+      with_file(@name){|d| d.should == "4|5|6\n1|2|3\n" }
+      
+      e.export(@name)
+      with_file(@name){|d| d.should == "4*5*6\n1*2*3\n" }
+    end
+  end
+  
+  describe "using temp file" do
+    it "at least file should return to target" do
+      File.exists?(@name).should be_false
+      PgCsv.new(:sql => @sql, :temp_file => true, :temp_dir => tmp_dir).export(@name)
+      with_file(@name){|d| d.should == "4,5,6\n1,2,3\n" }
+    end
+  end
+
+  describe "different types of export" do
+    it "gzip export" do
+      File.exists?(@gzname).should be_false
+      PgCsv.new(:sql => @sql, :type => :gzip).export(@gzname)
+      with_gzfile(@gzname){|d| d.should == "4,5,6\n1,2,3\n" }
+    end
+    
+    it "plain export" do
+      PgCsv.new(:sql => @sql, :type => :plain).export(nil).should == "4,5,6\n1,2,3\n"
+    end
+    
+    it "custom stream" do
+      ex = PgCsv.new(:sql => @sql, :type => :stream)
+      File.open(@name, 'w') do |stream|
+        ex.export(stream)
+        ex.export(stream, :sql => @sql0)
+      end
+      
+      with_file(@name){|d| d.should == "4,5,6\n1,2,3\n1,2,3\n4,5,6\n" }
+    end
+    
+    it "file as default" do
+      PgCsv.new(:sql => @sql, :type => :file).export(@name)
+      with_file(@name){|d| d.should == "4,5,6\n1,2,3\n" }            
+    end
+  end
+  
+end

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+require 'rubygems'
+require "bundler"
+Bundler.setup
+ENV['RAILS_ENV'] ||= 'test'
+
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+require 'pg_csv'
+
+require File.dirname(__FILE__) + '/spec_support.rb'

data/spec/spec_support.rb

@@ -0,0 +1,50 @@
+require 'fileutils'
+
+conn = {'adapter' => 'postgresql', 'database' => 'pgcsv_test', 'encoding' => 'utf8', 'username' => 'kostya', 'password' => 'password'}
+ActiveRecord::Base.establish_connection conn
+
+class Test < ActiveRecord::Base
+  self.table_name = 'tests'
+end
+
+def pg_create_schema
+  ActiveRecord::Migration.create_table :tests do |t|
+    t.integer :a
+    t.integer :b
+    t.integer :c
+  end
+end
+
+def pg_drop_data
+  ActiveRecord::Migration.drop_table :tests
+end
+
+pg_drop_data rescue nil
+pg_create_schema
+
+def tmp_dir
+  File.dirname(__FILE__) + "/tmp/"
+end
+
+def with_file(name)
+  File.exists?(name).should be_true
+  q = 1
+  File.open(name) do |file|
+    data = file.read
+    yield data
+    q = 2
+  end
+  
+  q.should == 2
+end
+
+def with_gzfile(name)
+  File.exist?(name).should be_true
+  q = 1
+  Zlib::GzipReader.open(name) do |gz|
+    data = gz.read
+    yield data
+    q = 2
+  end
+  q.should == 2
+end

metadata

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification 
+name: pg_csv
+version: !ruby/object:Gem::Version 
+  hash: 9
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  version: "0.1"
+platform: ruby
+authors: 
+- Makarchev Konstantin
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-05-31 00:00:00 +04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: pg
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id004
+description: Fast AR/PostgreSQL csv export. Used pg function 'copy to csv'. Effective on millions rows.
+email: 
+- kostya27@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- MIT-LICENSE
+- README.md
+- Rakefile
+- lib/pg_csv.rb
+- lib/pg_csv_version.rb
+- pg_csv.gemspec
+- spec/pg_csv_spec.rb
+- spec/spec_helper.rb
+- spec/spec_support.rb
+- spec/tmp/.gitkeep
+has_rdoc: true
+homepage: http://github.com/kostya/pg_csv
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Fast AR/PostgreSQL csv export. Used pg function 'copy to csv'. Effective on millions rows.
+test_files: []
+