em-pg-client-12 0.3.4

data/Rakefile

@@ -0,0 +1,157 @@
+require 'coveralls/rake/task'
+Coveralls::RakeTask.new
+task :test_with_coveralls => ['test:all', 'coveralls:push']
+
+$:.unshift "lib"
+
+task :default => [:test]
+
+$gem_name = "em-pg-client"
+
+def windows_os?
+  RbConfig::CONFIG['host_os'] =~ /cygwin|mswin|mingw|bccwin|wince|emx/
+end
+
+desc "Run tests"
+task :test => :'test:safe'
+
+namespace :test do
+  env_common = {'PGDATABASE' => 'test'}
+  env_unix = env_common.merge('PGHOST' => ENV['PGHOST_UNIX'] || '/tmp')
+  env_inet = env_common.merge('PGHOST' => ENV['PGHOST_INET'] || 'localhost')
+
+  task :warn do
+    puts "WARNING: The tests needs to be run with an available local PostgreSQL server"
+  end
+
+  desc "Run specs only"
+  task :spec => [:spec_defer, :spec_pool, :spec_client]
+
+  task :spec_client do
+    sh({'COVNAME'=>'spec:client'},     "rspec spec/pg_em_client_*.rb")
+  end
+
+  task :spec_pool do
+    sh({'COVNAME'=>'spec:pool'},       "rspec spec/pg_em_connection_pool.rb")
+  end
+
+  task :spec_defer do
+    sh({'COVNAME'=>'spec:deferrable'}, "rspec spec/pg_em_featured_deferrable.rb")
+  end
+
+  desc "Run safe tests only"
+  task :safe  => [:warn, :spec, :async, :fiber, :on_connect, :pool]
+  task :async => [:async_inet, :async_unix]
+  task :fiber => [:fiber_inet, :fiber_unix]
+  task :on_connect => [:on_connect_inet, :on_connect_unix]
+  task :pool => [:pool_inet, :pool_unix]
+
+  task :pool_inet do
+    sh env_inet.merge('COVNAME'=>'pool:inet'), "rspec spec/em_connection_pool.rb"
+  end
+
+  task :pool_unix do
+    sh env_unix.merge('COVNAME'=>'pool:unix'), "rspec spec/em_connection_pool.rb" unless windows_os?
+  end
+
+  task :on_connect_inet do
+    sh env_inet.merge('COVNAME'=>'on_connect:inet'), "rspec spec/em_client_on_connect.rb"
+  end
+
+  task :on_connect_unix do
+    sh env_unix.merge('COVNAME'=>'on_connect:unix'), "rspec spec/em_client_on_connect.rb" unless windows_os?
+  end
+
+  task :async_inet do
+    sh env_inet.merge('COVNAME'=>'async:inet'), "rspec spec/em_client.rb"
+  end
+
+  task :async_unix do
+    sh env_unix.merge('COVNAME'=>'async:unix'), "rspec spec/em_client.rb" unless windows_os?
+  end
+
+  task :fiber_inet do
+    sh env_inet.merge('COVNAME'=>'fiber:inet'), "rspec spec/em_synchrony_client.rb"
+  end
+
+  task :fiber_unix do
+    sh env_unix.merge('COVNAME'=>'fiber:unix'), "rspec spec/em_synchrony_client.rb" unless windows_os?
+  end
+
+  task :pgdata_check do
+    unless ENV['PGDATA'] || (ENV['PG_CTL_STOP_CMD'] && ENV['PG_CTL_START_CMD'])
+      raise "Set PGDATA environment variable before running the autoreconnect tests."
+    end
+  end
+
+  desc "Run unsafe tests only"
+  task :unsafe => [:warn, :pgdata_check,
+    :async_autoreconnect_inet,
+    :async_autoreconnect_unix,
+    :fiber_autoreconnect_inet,
+    :fiber_autoreconnect_unix]
+
+  task :async_autoreconnect_inet do
+    sh env_inet.merge('COVNAME'=>'async:autoreconnect:inet'), "rspec spec/em_client_autoreconnect.rb"
+  end
+
+  task :async_autoreconnect_unix do
+    sh env_unix.merge('COVNAME'=>'async:autoreconnect:unix'), "rspec spec/em_client_autoreconnect.rb"
+  end
+
+  task :fiber_autoreconnect_inet do
+    sh env_inet.merge('COVNAME'=>'fiber:autoreconnect:inet'), "rspec spec/em_synchrony_client_autoreconnect.rb"
+  end
+
+  task :fiber_autoreconnect_unix do
+    sh env_unix.merge('COVNAME'=>'fiber:autoreconnect:unix'), "rspec spec/em_synchrony_client_autoreconnect.rb"
+  end
+
+  desc "Run safe and unsafe tests"
+  task :all => [:spec, :safe, :unsafe]
+end
+
+desc "Build the gem"
+task :gem do
+  sh "gem build #$gem_name.gemspec"
+end
+
+desc "Install the library at local machnie"
+task :install => :gem do 
+  sh "gem install #$gem_name -l"
+end
+
+desc "Uninstall the library from local machnie"
+task :uninstall do
+  sh "gem uninstall #$gem_name"
+end
+
+desc "Clean"
+task :clean do
+  sh "rm #$gem_name*.gem" 
+end
+
+desc "Documentation"
+task :doc do
+  sh "yardoc"
+end
+
+desc "Benchmark"
+task :benchmark do
+  require "./benchmarks/em_pg.rb"
+  [10, 100, 1000].each do |i|
+    puts "Repeat: #{i}"
+    benchmark(i)
+  end
+end
+
+desc "Console"
+task :console do
+  require 'irb'
+  require 'irb/completion'
+  require 'em-synchrony'
+  require 'em-pg-client'
+  require 'pg/em/connection_pool'
+  ARGV.clear
+  IRB.start
+end

data/benchmarks/em_pg.rb

@@ -0,0 +1,96 @@
+$:.unshift('./lib')
+require 'eventmachine'
+require 'em-synchrony'
+require 'pg/em/connection_pool'
+require "em-synchrony/fiber_iterator"
+require 'pp'
+require 'benchmark'
+
+def benchmark(repeat=100)
+  Benchmark.bm(20) do |b|
+    b.report('single:')         { single(repeat) }
+    puts
+    b.report('parallel 90000/1:')  { parallel(repeat, 90000, 1) }
+    b.report('parallel 5000/5:')  { parallel(repeat, 5000, 5) }
+    b.report('parallel 2000/10:') { parallel(repeat, 2000, 10) }
+    b.report('parallel 1000/20:') { parallel(repeat, 1000, 20) }
+    puts
+    patch_blocking
+    b.report('blocking 90000/1:')  { parallel(repeat, 90000, 1) }
+    b.report('blocking 5000/5:')  { parallel(repeat, 5000, 5) }
+    b.report('blocking 2000/10:') { parallel(repeat, 2000, 10) }
+    b.report('blocking 1000/20:') { parallel(repeat, 1000, 20) }
+    patch_remove_blocking
+  end
+end
+
+def patch_remove_blocking
+  PG::EM::Client::Watcher.module_eval <<-EOE
+    alias_method :fetch_results, :original_fetch_results
+    alias_method :notify_readable, :fetch_results
+    undef :original_fetch_results
+  EOE
+end
+
+def patch_blocking
+  PG::Connection.class_eval <<-EOE
+    alias_method :blocking_get_last_result, :get_last_result
+  EOE
+  PG::EM::Client::Watcher.module_eval <<-EOE
+    alias_method :original_fetch_results, :fetch_results
+    def fetch_results
+      self.notify_readable = false
+      begin
+        result = @client.blocking_get_last_result
+      rescue Exception => e
+        @deferrable.fail(e)
+      else
+        @deferrable.succeed(result)
+      end
+    end
+    alias_method :notify_readable, :fetch_results
+  EOE
+end
+
+# retrieve resources using single select query
+def single(repeat=1)
+  rowcount = 0
+  p = PGconn.new
+  p.query('select count(*) from resources') do |result|
+    rowcount = result.getvalue(0,0).to_i
+  end
+  repeat.times do
+    p.query('select * from resources order by cdate') do |result|
+      $resources = result.values
+    end
+  end
+  # raise "invalid count #{$resources.length} != #{rowcount}" if $resources.length != rowcount
+end
+
+# retrieve resources using parallel queries
+def parallel(repeat=1, chunk_size=2000, concurrency=10)
+  resources = []
+  rowcount = 0
+  EM.synchrony do
+    p = PG::EM::ConnectionPool.new size: concurrency
+    p.query('select count(*) from resources') do |result|
+      rowcount = result.getvalue(0,0).to_i
+    end
+    offsets = (rowcount / chunk_size.to_f).ceil.times.map {|n| n*chunk_size }
+    repeat.times do
+      EM::Synchrony::FiberIterator.new(offsets, concurrency).each do |offset|
+        p.query('select * from resources order by cdate limit $1 offset $2', [chunk_size, offset]) do |result|
+          resources[offset, chunk_size] = result.values
+        end
+      end
+    end
+    EM.stop
+  end
+  # raise "invalid count #{resources.length} != #{rowcount}" if resources.length != rowcount
+  # raise "resources != $resources" if resources != $resources
+  resources
+end
+
+if $0 == __FILE__
+  benchmark ARGV[0].to_i.nonzero? || 10
+end

data/benchmarks/single_row_mode.rb

@@ -0,0 +1,88 @@
+$:.unshift('./lib')
+require 'pg/em/connection_pool'
+require 'em-synchrony'
+require 'em-synchrony/fiber_iterator'
+require 'pp'
+require 'benchmark'
+
+TABLE_NAME = 'resources'
+LIMIT_ROWS = 5000
+
+include EM::Synchrony
+
+unless PG::EM::Client.single_row_mode?
+  raise 'compile pg against pqlib >= 9.2 to support single row mode'
+end
+
+def benchmark(repeat=40)
+  Benchmark.bm(20) do |b|
+    puts
+    b.report("threads #{repeat/1}x1:")   { threads(repeat, 1) }
+    b.report("threads #{repeat/5}x5:")   { threads(repeat, 5) }
+    b.report("threads #{repeat/10}x10:") { threads(repeat, 10) }
+    b.report("threads #{repeat/20}x20:") { threads(repeat, 20) }
+    b.report("threads #{repeat/40}x40:") { threads(repeat, 40) }
+    puts
+    b.report("fibers  #{repeat/1}x1:")   { fibers(repeat, 1) }
+    b.report("fibers  #{repeat/5}x5:")   { fibers(repeat, 5) }
+    b.report("fibers  #{repeat/10}x10:") { fibers(repeat, 10) }
+    b.report("fibers  #{repeat/20}x20:") { fibers(repeat, 20) }
+    b.report("fibers  #{repeat/40}x40:") { fibers(repeat, 40) }
+  end
+end
+
+def threads(repeat, concurrency)
+  db = Hash.new { |pool, id| pool[id] = PG::Connection.new }
+  (0...concurrency).map do |i|
+    Thread.new do
+      (repeat/concurrency).times do
+        stream_results(db[i])
+      end
+    end
+  end.each(&:join)
+  db.each_value(&:finish).clear
+end
+
+def fibers(repeat, concurrency)
+  EM.synchrony do
+    db = PG::EM::ConnectionPool.new size: concurrency, lazy: true
+    FiberIterator.new((0...concurrency), concurrency).each do
+      db.hold do |pg|
+        (repeat/concurrency).times do
+          stream_results(pg)
+        end
+      end
+    end
+    db.finish
+    EM.stop
+  end
+end
+
+def stream_results(pg)
+  pg.send_query("select * from #{TABLE_NAME}")
+  pg.set_single_row_mode
+  rows = 0
+  last_time = Time.now
+  while result = pg.get_result
+    begin
+      result.check
+      result.each do |tuple|
+        rows += 1
+        if rows >= LIMIT_ROWS
+          pg.reset
+          break
+        end
+      end
+    rescue PG::Error => e
+      pg.get_last_result
+      raise e
+    ensure
+      result.clear
+    end
+  end
+end
+
+if $0 == __FILE__
+  benchmark ARGV[0].to_i.nonzero? || 40
+end
+

data/em-pg-client.gemspec

@@ -0,0 +1,34 @@
+$:.unshift "lib"
+require 'pg/em-version'
+
+files = `git ls-files`.split("\n")
+
+Gem::Specification.new do |s|
+  s.name = "em-pg-client-12"
+  s.version = PG::EM::VERSION
+  s.required_ruby_version = ">= 1.9.2"
+  s.date = "#{Time.now.strftime("%Y-%m-%d")}"
+  s.summary = "EventMachine PostgreSQL client"
+  s.email = "rafal@yeondir.com"
+  s.homepage = "http://github.com/royaltm/ruby-em-pg-client"
+  s.license = "MIT"
+  s.require_path = "lib"
+  s.description = "PostgreSQL asynchronous EventMachine client, based on pg interface (PG::Connection)"
+  s.authors = ["Rafal Michalski"]
+  s.files = files - ['.gitignore']
+  s.test_files = Dir.glob("spec/**/*")
+  s.rdoc_options << "--title" << "em-pg-client" <<
+    "--main" << "README.md"
+  s.has_rdoc = true
+  s.extra_rdoc_files = [
+      files.grep(/^benchmarks\/.*\.rb$/),
+      "README.md", "BENCHMARKS.md", "LICENSE", "HISTORY.md"
+    ].flatten
+  s.requirements << "PostgreSQL server"
+  s.add_runtime_dependency "pg", ">= 0.17.0"
+  s.add_runtime_dependency "eventmachine", "~> 1.2"
+  s.add_development_dependency "rspec", "~> 2.14"
+  s.add_development_dependency "em-synchrony", "~> 1.0"
+  s.add_development_dependency "coveralls", ">= 0.7.0"
+  s.add_development_dependency "simplecov", ">= 0.8.2"
+end

data/examples/single_row_mode.rb

@@ -0,0 +1,57 @@
+# Demonstation of fully asynchronous query data streaming.
+#
+# This is a low-level method which has its:
+#
+# upside:   it's the same way you would work with PG::Connection
+# downside: it's a little verbose and doesn't support automatic re-connects
+gem 'em-pg-client', '>= 0.3.2'
+require 'pg/em/connection_pool'
+require 'em-synchrony'
+require 'em-synchrony/fiber_iterator'
+
+TABLE_NAME = 'resources'
+
+unless PG::EM::Client.single_row_mode?
+  raise 'compile pg against pqlib >= 9.2 to support single row mode'
+end
+
+EM.synchrony do
+  EM.add_periodic_timer(0.01) { print ' ' }
+
+  db = PG::EM::ConnectionPool.new size: 3
+
+  10.times do
+
+    EM::Synchrony::FiberIterator.new(%w[@ * #], 3).each do |mark|
+
+      db.hold do |pg|
+        pg.send_query("select * from #{TABLE_NAME}")
+        pg.set_single_row_mode
+        rows = 0
+        while result = pg.get_result
+          begin
+            result.check
+            result.each do |tuple|
+              rows += 1
+              # process tuple
+              print mark
+              # break stream cleanly
+              pg.reset if rows > 1000
+            end
+          rescue PG::Error => e
+            # cleanup connection
+            pg.get_last_result
+            raise e
+          ensure
+            result.clear
+          end
+        end
+      end
+
+    end
+
+    puts
+    puts '='*80
+  end
+  EM.stop
+end

data/lib/em-pg-client.rb

@@ -0,0 +1 @@
+require 'pg/em'

data/lib/em-synchrony/pg.rb

@@ -0,0 +1,3 @@
+# for backward compatibility with
+# require 'em-synchrony/pg'
+require 'pg/em'

data/lib/pg/em-version.rb

@@ -0,0 +1,5 @@
+module PG
+  module EM
+    VERSION = '0.3.4'
+  end
+end