em-pg-client-12 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rspec +1 -0
- data/.travis.yml +32 -0
- data/.yardopts +1 -0
- data/BENCHMARKS.md +91 -0
- data/Gemfile +5 -0
- data/HISTORY.md +107 -0
- data/LICENSE +21 -0
- data/README.md +456 -0
- data/Rakefile +157 -0
- data/benchmarks/em_pg.rb +96 -0
- data/benchmarks/single_row_mode.rb +88 -0
- data/em-pg-client.gemspec +34 -0
- data/examples/single_row_mode.rb +57 -0
- data/lib/em-pg-client.rb +1 -0
- data/lib/em-synchrony/pg.rb +3 -0
- data/lib/pg/em-version.rb +5 -0
- data/lib/pg/em.rb +1129 -0
- data/lib/pg/em/client/connect_watcher.rb +89 -0
- data/lib/pg/em/client/watcher.rb +204 -0
- data/lib/pg/em/connection_pool.rb +480 -0
- data/lib/pg/em/featured_deferrable.rb +43 -0
- data/spec/connection_pool_helpers.rb +89 -0
- data/spec/em_client.rb +33 -0
- data/spec/em_client_autoreconnect.rb +672 -0
- data/spec/em_client_common.rb +619 -0
- data/spec/em_client_on_connect.rb +171 -0
- data/spec/em_connection_pool.rb +200 -0
- data/spec/em_synchrony_client.rb +787 -0
- data/spec/em_synchrony_client_autoreconnect.rb +560 -0
- data/spec/pg_em_client_connect_finish.rb +54 -0
- data/spec/pg_em_client_connect_timeout.rb +91 -0
- data/spec/pg_em_client_options.rb +133 -0
- data/spec/pg_em_connection_pool.rb +679 -0
- data/spec/pg_em_featured_deferrable.rb +125 -0
- data/spec/spec_helper.rb +9 -0
- metadata +187 -0
data/Rakefile
ADDED
@@ -0,0 +1,157 @@
|
|
1
|
+
require 'coveralls/rake/task'
|
2
|
+
Coveralls::RakeTask.new
|
3
|
+
task :test_with_coveralls => ['test:all', 'coveralls:push']
|
4
|
+
|
5
|
+
$:.unshift "lib"
|
6
|
+
|
7
|
+
task :default => [:test]
|
8
|
+
|
9
|
+
$gem_name = "em-pg-client"
|
10
|
+
|
11
|
+
def windows_os?
|
12
|
+
RbConfig::CONFIG['host_os'] =~ /cygwin|mswin|mingw|bccwin|wince|emx/
|
13
|
+
end
|
14
|
+
|
15
|
+
desc "Run tests"
|
16
|
+
task :test => :'test:safe'
|
17
|
+
|
18
|
+
namespace :test do
|
19
|
+
env_common = {'PGDATABASE' => 'test'}
|
20
|
+
env_unix = env_common.merge('PGHOST' => ENV['PGHOST_UNIX'] || '/tmp')
|
21
|
+
env_inet = env_common.merge('PGHOST' => ENV['PGHOST_INET'] || 'localhost')
|
22
|
+
|
23
|
+
task :warn do
|
24
|
+
puts "WARNING: The tests needs to be run with an available local PostgreSQL server"
|
25
|
+
end
|
26
|
+
|
27
|
+
desc "Run specs only"
|
28
|
+
task :spec => [:spec_defer, :spec_pool, :spec_client]
|
29
|
+
|
30
|
+
task :spec_client do
|
31
|
+
sh({'COVNAME'=>'spec:client'}, "rspec spec/pg_em_client_*.rb")
|
32
|
+
end
|
33
|
+
|
34
|
+
task :spec_pool do
|
35
|
+
sh({'COVNAME'=>'spec:pool'}, "rspec spec/pg_em_connection_pool.rb")
|
36
|
+
end
|
37
|
+
|
38
|
+
task :spec_defer do
|
39
|
+
sh({'COVNAME'=>'spec:deferrable'}, "rspec spec/pg_em_featured_deferrable.rb")
|
40
|
+
end
|
41
|
+
|
42
|
+
desc "Run safe tests only"
|
43
|
+
task :safe => [:warn, :spec, :async, :fiber, :on_connect, :pool]
|
44
|
+
task :async => [:async_inet, :async_unix]
|
45
|
+
task :fiber => [:fiber_inet, :fiber_unix]
|
46
|
+
task :on_connect => [:on_connect_inet, :on_connect_unix]
|
47
|
+
task :pool => [:pool_inet, :pool_unix]
|
48
|
+
|
49
|
+
task :pool_inet do
|
50
|
+
sh env_inet.merge('COVNAME'=>'pool:inet'), "rspec spec/em_connection_pool.rb"
|
51
|
+
end
|
52
|
+
|
53
|
+
task :pool_unix do
|
54
|
+
sh env_unix.merge('COVNAME'=>'pool:unix'), "rspec spec/em_connection_pool.rb" unless windows_os?
|
55
|
+
end
|
56
|
+
|
57
|
+
task :on_connect_inet do
|
58
|
+
sh env_inet.merge('COVNAME'=>'on_connect:inet'), "rspec spec/em_client_on_connect.rb"
|
59
|
+
end
|
60
|
+
|
61
|
+
task :on_connect_unix do
|
62
|
+
sh env_unix.merge('COVNAME'=>'on_connect:unix'), "rspec spec/em_client_on_connect.rb" unless windows_os?
|
63
|
+
end
|
64
|
+
|
65
|
+
task :async_inet do
|
66
|
+
sh env_inet.merge('COVNAME'=>'async:inet'), "rspec spec/em_client.rb"
|
67
|
+
end
|
68
|
+
|
69
|
+
task :async_unix do
|
70
|
+
sh env_unix.merge('COVNAME'=>'async:unix'), "rspec spec/em_client.rb" unless windows_os?
|
71
|
+
end
|
72
|
+
|
73
|
+
task :fiber_inet do
|
74
|
+
sh env_inet.merge('COVNAME'=>'fiber:inet'), "rspec spec/em_synchrony_client.rb"
|
75
|
+
end
|
76
|
+
|
77
|
+
task :fiber_unix do
|
78
|
+
sh env_unix.merge('COVNAME'=>'fiber:unix'), "rspec spec/em_synchrony_client.rb" unless windows_os?
|
79
|
+
end
|
80
|
+
|
81
|
+
task :pgdata_check do
|
82
|
+
unless ENV['PGDATA'] || (ENV['PG_CTL_STOP_CMD'] && ENV['PG_CTL_START_CMD'])
|
83
|
+
raise "Set PGDATA environment variable before running the autoreconnect tests."
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
desc "Run unsafe tests only"
|
88
|
+
task :unsafe => [:warn, :pgdata_check,
|
89
|
+
:async_autoreconnect_inet,
|
90
|
+
:async_autoreconnect_unix,
|
91
|
+
:fiber_autoreconnect_inet,
|
92
|
+
:fiber_autoreconnect_unix]
|
93
|
+
|
94
|
+
task :async_autoreconnect_inet do
|
95
|
+
sh env_inet.merge('COVNAME'=>'async:autoreconnect:inet'), "rspec spec/em_client_autoreconnect.rb"
|
96
|
+
end
|
97
|
+
|
98
|
+
task :async_autoreconnect_unix do
|
99
|
+
sh env_unix.merge('COVNAME'=>'async:autoreconnect:unix'), "rspec spec/em_client_autoreconnect.rb"
|
100
|
+
end
|
101
|
+
|
102
|
+
task :fiber_autoreconnect_inet do
|
103
|
+
sh env_inet.merge('COVNAME'=>'fiber:autoreconnect:inet'), "rspec spec/em_synchrony_client_autoreconnect.rb"
|
104
|
+
end
|
105
|
+
|
106
|
+
task :fiber_autoreconnect_unix do
|
107
|
+
sh env_unix.merge('COVNAME'=>'fiber:autoreconnect:unix'), "rspec spec/em_synchrony_client_autoreconnect.rb"
|
108
|
+
end
|
109
|
+
|
110
|
+
desc "Run safe and unsafe tests"
|
111
|
+
task :all => [:spec, :safe, :unsafe]
|
112
|
+
end
|
113
|
+
|
114
|
+
desc "Build the gem"
|
115
|
+
task :gem do
|
116
|
+
sh "gem build #$gem_name.gemspec"
|
117
|
+
end
|
118
|
+
|
119
|
+
desc "Install the library at local machnie"
|
120
|
+
task :install => :gem do
|
121
|
+
sh "gem install #$gem_name -l"
|
122
|
+
end
|
123
|
+
|
124
|
+
desc "Uninstall the library from local machnie"
|
125
|
+
task :uninstall do
|
126
|
+
sh "gem uninstall #$gem_name"
|
127
|
+
end
|
128
|
+
|
129
|
+
desc "Clean"
|
130
|
+
task :clean do
|
131
|
+
sh "rm #$gem_name*.gem"
|
132
|
+
end
|
133
|
+
|
134
|
+
desc "Documentation"
|
135
|
+
task :doc do
|
136
|
+
sh "yardoc"
|
137
|
+
end
|
138
|
+
|
139
|
+
desc "Benchmark"
|
140
|
+
task :benchmark do
|
141
|
+
require "./benchmarks/em_pg.rb"
|
142
|
+
[10, 100, 1000].each do |i|
|
143
|
+
puts "Repeat: #{i}"
|
144
|
+
benchmark(i)
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
desc "Console"
|
149
|
+
task :console do
|
150
|
+
require 'irb'
|
151
|
+
require 'irb/completion'
|
152
|
+
require 'em-synchrony'
|
153
|
+
require 'em-pg-client'
|
154
|
+
require 'pg/em/connection_pool'
|
155
|
+
ARGV.clear
|
156
|
+
IRB.start
|
157
|
+
end
|
data/benchmarks/em_pg.rb
ADDED
@@ -0,0 +1,96 @@
|
|
1
|
+
$:.unshift('./lib')
|
2
|
+
require 'eventmachine'
|
3
|
+
require 'em-synchrony'
|
4
|
+
require 'pg/em/connection_pool'
|
5
|
+
require "em-synchrony/fiber_iterator"
|
6
|
+
require 'pp'
|
7
|
+
require 'benchmark'
|
8
|
+
|
9
|
+
def benchmark(repeat=100)
|
10
|
+
Benchmark.bm(20) do |b|
|
11
|
+
b.report('single:') { single(repeat) }
|
12
|
+
puts
|
13
|
+
b.report('parallel 90000/1:') { parallel(repeat, 90000, 1) }
|
14
|
+
b.report('parallel 5000/5:') { parallel(repeat, 5000, 5) }
|
15
|
+
b.report('parallel 2000/10:') { parallel(repeat, 2000, 10) }
|
16
|
+
b.report('parallel 1000/20:') { parallel(repeat, 1000, 20) }
|
17
|
+
puts
|
18
|
+
patch_blocking
|
19
|
+
b.report('blocking 90000/1:') { parallel(repeat, 90000, 1) }
|
20
|
+
b.report('blocking 5000/5:') { parallel(repeat, 5000, 5) }
|
21
|
+
b.report('blocking 2000/10:') { parallel(repeat, 2000, 10) }
|
22
|
+
b.report('blocking 1000/20:') { parallel(repeat, 1000, 20) }
|
23
|
+
patch_remove_blocking
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def patch_remove_blocking
|
28
|
+
PG::EM::Client::Watcher.module_eval <<-EOE
|
29
|
+
alias_method :fetch_results, :original_fetch_results
|
30
|
+
alias_method :notify_readable, :fetch_results
|
31
|
+
undef :original_fetch_results
|
32
|
+
EOE
|
33
|
+
end
|
34
|
+
|
35
|
+
def patch_blocking
|
36
|
+
PG::Connection.class_eval <<-EOE
|
37
|
+
alias_method :blocking_get_last_result, :get_last_result
|
38
|
+
EOE
|
39
|
+
PG::EM::Client::Watcher.module_eval <<-EOE
|
40
|
+
alias_method :original_fetch_results, :fetch_results
|
41
|
+
def fetch_results
|
42
|
+
self.notify_readable = false
|
43
|
+
begin
|
44
|
+
result = @client.blocking_get_last_result
|
45
|
+
rescue Exception => e
|
46
|
+
@deferrable.fail(e)
|
47
|
+
else
|
48
|
+
@deferrable.succeed(result)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
alias_method :notify_readable, :fetch_results
|
52
|
+
EOE
|
53
|
+
end
|
54
|
+
|
55
|
+
# retrieve resources using single select query
|
56
|
+
def single(repeat=1)
|
57
|
+
rowcount = 0
|
58
|
+
p = PGconn.new
|
59
|
+
p.query('select count(*) from resources') do |result|
|
60
|
+
rowcount = result.getvalue(0,0).to_i
|
61
|
+
end
|
62
|
+
repeat.times do
|
63
|
+
p.query('select * from resources order by cdate') do |result|
|
64
|
+
$resources = result.values
|
65
|
+
end
|
66
|
+
end
|
67
|
+
# raise "invalid count #{$resources.length} != #{rowcount}" if $resources.length != rowcount
|
68
|
+
end
|
69
|
+
|
70
|
+
# retrieve resources using parallel queries
|
71
|
+
def parallel(repeat=1, chunk_size=2000, concurrency=10)
|
72
|
+
resources = []
|
73
|
+
rowcount = 0
|
74
|
+
EM.synchrony do
|
75
|
+
p = PG::EM::ConnectionPool.new size: concurrency
|
76
|
+
p.query('select count(*) from resources') do |result|
|
77
|
+
rowcount = result.getvalue(0,0).to_i
|
78
|
+
end
|
79
|
+
offsets = (rowcount / chunk_size.to_f).ceil.times.map {|n| n*chunk_size }
|
80
|
+
repeat.times do
|
81
|
+
EM::Synchrony::FiberIterator.new(offsets, concurrency).each do |offset|
|
82
|
+
p.query('select * from resources order by cdate limit $1 offset $2', [chunk_size, offset]) do |result|
|
83
|
+
resources[offset, chunk_size] = result.values
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
EM.stop
|
88
|
+
end
|
89
|
+
# raise "invalid count #{resources.length} != #{rowcount}" if resources.length != rowcount
|
90
|
+
# raise "resources != $resources" if resources != $resources
|
91
|
+
resources
|
92
|
+
end
|
93
|
+
|
94
|
+
if $0 == __FILE__
|
95
|
+
benchmark ARGV[0].to_i.nonzero? || 10
|
96
|
+
end
|
@@ -0,0 +1,88 @@
|
|
1
|
+
$:.unshift('./lib')
|
2
|
+
require 'pg/em/connection_pool'
|
3
|
+
require 'em-synchrony'
|
4
|
+
require 'em-synchrony/fiber_iterator'
|
5
|
+
require 'pp'
|
6
|
+
require 'benchmark'
|
7
|
+
|
8
|
+
TABLE_NAME = 'resources'
|
9
|
+
LIMIT_ROWS = 5000
|
10
|
+
|
11
|
+
include EM::Synchrony
|
12
|
+
|
13
|
+
unless PG::EM::Client.single_row_mode?
|
14
|
+
raise 'compile pg against pqlib >= 9.2 to support single row mode'
|
15
|
+
end
|
16
|
+
|
17
|
+
def benchmark(repeat=40)
|
18
|
+
Benchmark.bm(20) do |b|
|
19
|
+
puts
|
20
|
+
b.report("threads #{repeat/1}x1:") { threads(repeat, 1) }
|
21
|
+
b.report("threads #{repeat/5}x5:") { threads(repeat, 5) }
|
22
|
+
b.report("threads #{repeat/10}x10:") { threads(repeat, 10) }
|
23
|
+
b.report("threads #{repeat/20}x20:") { threads(repeat, 20) }
|
24
|
+
b.report("threads #{repeat/40}x40:") { threads(repeat, 40) }
|
25
|
+
puts
|
26
|
+
b.report("fibers #{repeat/1}x1:") { fibers(repeat, 1) }
|
27
|
+
b.report("fibers #{repeat/5}x5:") { fibers(repeat, 5) }
|
28
|
+
b.report("fibers #{repeat/10}x10:") { fibers(repeat, 10) }
|
29
|
+
b.report("fibers #{repeat/20}x20:") { fibers(repeat, 20) }
|
30
|
+
b.report("fibers #{repeat/40}x40:") { fibers(repeat, 40) }
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def threads(repeat, concurrency)
|
35
|
+
db = Hash.new { |pool, id| pool[id] = PG::Connection.new }
|
36
|
+
(0...concurrency).map do |i|
|
37
|
+
Thread.new do
|
38
|
+
(repeat/concurrency).times do
|
39
|
+
stream_results(db[i])
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end.each(&:join)
|
43
|
+
db.each_value(&:finish).clear
|
44
|
+
end
|
45
|
+
|
46
|
+
def fibers(repeat, concurrency)
|
47
|
+
EM.synchrony do
|
48
|
+
db = PG::EM::ConnectionPool.new size: concurrency, lazy: true
|
49
|
+
FiberIterator.new((0...concurrency), concurrency).each do
|
50
|
+
db.hold do |pg|
|
51
|
+
(repeat/concurrency).times do
|
52
|
+
stream_results(pg)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
db.finish
|
57
|
+
EM.stop
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def stream_results(pg)
|
62
|
+
pg.send_query("select * from #{TABLE_NAME}")
|
63
|
+
pg.set_single_row_mode
|
64
|
+
rows = 0
|
65
|
+
last_time = Time.now
|
66
|
+
while result = pg.get_result
|
67
|
+
begin
|
68
|
+
result.check
|
69
|
+
result.each do |tuple|
|
70
|
+
rows += 1
|
71
|
+
if rows >= LIMIT_ROWS
|
72
|
+
pg.reset
|
73
|
+
break
|
74
|
+
end
|
75
|
+
end
|
76
|
+
rescue PG::Error => e
|
77
|
+
pg.get_last_result
|
78
|
+
raise e
|
79
|
+
ensure
|
80
|
+
result.clear
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
if $0 == __FILE__
|
86
|
+
benchmark ARGV[0].to_i.nonzero? || 40
|
87
|
+
end
|
88
|
+
|
@@ -0,0 +1,34 @@
|
|
1
|
+
$:.unshift "lib"
|
2
|
+
require 'pg/em-version'
|
3
|
+
|
4
|
+
files = `git ls-files`.split("\n")
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = "em-pg-client-12"
|
8
|
+
s.version = PG::EM::VERSION
|
9
|
+
s.required_ruby_version = ">= 1.9.2"
|
10
|
+
s.date = "#{Time.now.strftime("%Y-%m-%d")}"
|
11
|
+
s.summary = "EventMachine PostgreSQL client"
|
12
|
+
s.email = "rafal@yeondir.com"
|
13
|
+
s.homepage = "http://github.com/royaltm/ruby-em-pg-client"
|
14
|
+
s.license = "MIT"
|
15
|
+
s.require_path = "lib"
|
16
|
+
s.description = "PostgreSQL asynchronous EventMachine client, based on pg interface (PG::Connection)"
|
17
|
+
s.authors = ["Rafal Michalski"]
|
18
|
+
s.files = files - ['.gitignore']
|
19
|
+
s.test_files = Dir.glob("spec/**/*")
|
20
|
+
s.rdoc_options << "--title" << "em-pg-client" <<
|
21
|
+
"--main" << "README.md"
|
22
|
+
s.has_rdoc = true
|
23
|
+
s.extra_rdoc_files = [
|
24
|
+
files.grep(/^benchmarks\/.*\.rb$/),
|
25
|
+
"README.md", "BENCHMARKS.md", "LICENSE", "HISTORY.md"
|
26
|
+
].flatten
|
27
|
+
s.requirements << "PostgreSQL server"
|
28
|
+
s.add_runtime_dependency "pg", ">= 0.17.0"
|
29
|
+
s.add_runtime_dependency "eventmachine", "~> 1.2"
|
30
|
+
s.add_development_dependency "rspec", "~> 2.14"
|
31
|
+
s.add_development_dependency "em-synchrony", "~> 1.0"
|
32
|
+
s.add_development_dependency "coveralls", ">= 0.7.0"
|
33
|
+
s.add_development_dependency "simplecov", ">= 0.8.2"
|
34
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
# Demonstation of fully asynchronous query data streaming.
|
2
|
+
#
|
3
|
+
# This is a low-level method which has its:
|
4
|
+
#
|
5
|
+
# upside: it's the same way you would work with PG::Connection
|
6
|
+
# downside: it's a little verbose and doesn't support automatic re-connects
|
7
|
+
gem 'em-pg-client', '>= 0.3.2'
|
8
|
+
require 'pg/em/connection_pool'
|
9
|
+
require 'em-synchrony'
|
10
|
+
require 'em-synchrony/fiber_iterator'
|
11
|
+
|
12
|
+
TABLE_NAME = 'resources'
|
13
|
+
|
14
|
+
unless PG::EM::Client.single_row_mode?
|
15
|
+
raise 'compile pg against pqlib >= 9.2 to support single row mode'
|
16
|
+
end
|
17
|
+
|
18
|
+
EM.synchrony do
|
19
|
+
EM.add_periodic_timer(0.01) { print ' ' }
|
20
|
+
|
21
|
+
db = PG::EM::ConnectionPool.new size: 3
|
22
|
+
|
23
|
+
10.times do
|
24
|
+
|
25
|
+
EM::Synchrony::FiberIterator.new(%w[@ * #], 3).each do |mark|
|
26
|
+
|
27
|
+
db.hold do |pg|
|
28
|
+
pg.send_query("select * from #{TABLE_NAME}")
|
29
|
+
pg.set_single_row_mode
|
30
|
+
rows = 0
|
31
|
+
while result = pg.get_result
|
32
|
+
begin
|
33
|
+
result.check
|
34
|
+
result.each do |tuple|
|
35
|
+
rows += 1
|
36
|
+
# process tuple
|
37
|
+
print mark
|
38
|
+
# break stream cleanly
|
39
|
+
pg.reset if rows > 1000
|
40
|
+
end
|
41
|
+
rescue PG::Error => e
|
42
|
+
# cleanup connection
|
43
|
+
pg.get_last_result
|
44
|
+
raise e
|
45
|
+
ensure
|
46
|
+
result.clear
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
|
53
|
+
puts
|
54
|
+
puts '='*80
|
55
|
+
end
|
56
|
+
EM.stop
|
57
|
+
end
|
data/lib/em-pg-client.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require 'pg/em'
|