em-pg-client 0.1.1 → 0.2.0.pre.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/HISTORY.rdoc +16 -9
- data/README.rdoc +294 -258
- data/benchmarks/em_pg.rb +93 -93
- data/em-pg-client.gemspec +1 -1
- data/lib/em-synchrony/pg.rb +81 -45
- data/lib/pg/em.rb +348 -175
- metadata +5 -5
data/HISTORY.rdoc
CHANGED
@@ -1,9 +1,16 @@
|
|
1
|
-
|
2
|
-
- added
|
3
|
-
-
|
4
|
-
-
|
5
|
-
-
|
6
|
-
- added
|
7
|
-
|
8
|
-
0.1.
|
9
|
-
-
|
1
|
+
next release
|
2
|
+
- added connect_timeout property with for async connect/reset
|
3
|
+
- fix: async_autoreconnect for tcp/ip connections
|
4
|
+
- fix: async_* does not raise errors; errors handled by deferrable
|
5
|
+
- rework async_autoreconnect in fully async manner
|
6
|
+
- added async_connect() and #async_reset()
|
7
|
+
- API change: on_reconnect -> on_autoreconnect
|
8
|
+
0.1.1
|
9
|
+
- added on_reconnect callback
|
10
|
+
- docs updated
|
11
|
+
- added development dependency for eventmachine >= 1.0.0.beta.1
|
12
|
+
- added separate client specs for eventmachine = 0.12.10
|
13
|
+
- added error checking to eventmachine specs
|
14
|
+
|
15
|
+
0.1.0
|
16
|
+
- first release
|
data/README.rdoc
CHANGED
@@ -1,258 +1,294 @@
|
|
1
|
-
= em-pg-client
|
2
|
-
|
3
|
-
Author:: Rafał Michalski (mailto:rafal@yeondir.com)
|
4
|
-
|
5
|
-
* http://github.com/royaltm/ruby-em-pg-client
|
6
|
-
|
7
|
-
== DESCRIPTION
|
8
|
-
|
9
|
-
*em-pg-client* is a PostgreSQL EventMachine client wrapper for Ruby
|
10
|
-
based on ruby-pg[https://bitbucket.org/ged/ruby-pg]
|
11
|
-
|
12
|
-
== FEATURES
|
13
|
-
|
14
|
-
* minimal changes to PG::Connection API
|
15
|
-
* auto reconnects on socket connection loss (like server restarts)
|
16
|
-
* true non-blocking asynchronous processing
|
17
|
-
* EM-Synchrony[https://github.com/igrigorik/em-synchrony] support
|
18
|
-
|
19
|
-
== BUGS/LIMITATIONS
|
20
|
-
|
21
|
-
* actually no ActiveRecord
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
==
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
pg.
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
EM.run do
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
EM.synchrony do
|
215
|
-
|
216
|
-
pg
|
217
|
-
|
218
|
-
end
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
EM.
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
1
|
+
= em-pg-client
|
2
|
+
|
3
|
+
Author:: Rafał Michalski (mailto:rafal@yeondir.com)
|
4
|
+
|
5
|
+
* http://github.com/royaltm/ruby-em-pg-client
|
6
|
+
|
7
|
+
== DESCRIPTION
|
8
|
+
|
9
|
+
*em-pg-client* is a PostgreSQL EventMachine client wrapper for Ruby
|
10
|
+
based on ruby-pg[https://bitbucket.org/ged/ruby-pg]
|
11
|
+
|
12
|
+
== FEATURES
|
13
|
+
|
14
|
+
* minimal changes to PG::Connection API
|
15
|
+
* fully async auto reconnects on socket connection loss (like server restarts)
|
16
|
+
* true non-blocking asynchronous processing
|
17
|
+
* EM-Synchrony[https://github.com/igrigorik/em-synchrony] support
|
18
|
+
|
19
|
+
== BUGS/LIMITATIONS
|
20
|
+
|
21
|
+
* actually no ActiveRecord nor Sequel support (you are welcome to contribute).
|
22
|
+
|
23
|
+
== API Changes
|
24
|
+
|
25
|
+
* +on_reconnect+ renamed to more accurate +on_autoreconnect+
|
26
|
+
(well, it's not used by +#reset+ call)
|
27
|
+
|
28
|
+
== TODO:
|
29
|
+
|
30
|
+
* em-synchrony ORM (ActiveRecord, Sequel and maybe Datamapper) support
|
31
|
+
as separate projects
|
32
|
+
|
33
|
+
== REQUIREMENTS
|
34
|
+
|
35
|
+
* ruby >= 1.9
|
36
|
+
* https://bitbucket.org/ged/ruby-pg (>= 0.13)
|
37
|
+
* http://rubyeventmachine.com
|
38
|
+
* (optional) EM-Synchrony[https://github.com/igrigorik/em-synchrony]
|
39
|
+
|
40
|
+
== INSTALL
|
41
|
+
|
42
|
+
=== Legacy
|
43
|
+
|
44
|
+
$ [sudo] gem install em-pg-client
|
45
|
+
|
46
|
+
==== Gemfile
|
47
|
+
|
48
|
+
# eventmachine
|
49
|
+
gem "em-pg-client", "~> 0.1.1", :require => 'pg/em'
|
50
|
+
# em-synchrony
|
51
|
+
gem "em-pg-client", "~> 0.1.1", :require => ['pg/em', 'em-synchrony/pg']
|
52
|
+
|
53
|
+
=== Latest branch (fully-async)
|
54
|
+
|
55
|
+
$ [sudo] gem install em-pg-client --prerelease
|
56
|
+
|
57
|
+
==== Gemfile
|
58
|
+
|
59
|
+
# eventmachine
|
60
|
+
gem "em-pg-client", "~> 0.2.0.pre", :require => 'pg/em'
|
61
|
+
# em-synchrony
|
62
|
+
gem "em-pg-client", "~> 0.2.0.pre", :require => ['pg/em', 'em-synchrony/pg']
|
63
|
+
|
64
|
+
==== Github
|
65
|
+
|
66
|
+
git clone git://github.com/royaltm/ruby-em-pg-client.git
|
67
|
+
git checkout fully-async
|
68
|
+
|
69
|
+
== WHY?
|
70
|
+
|
71
|
+
Because until now nobody did it to fit my needs.
|
72
|
+
I've found at least 3 other implementations of EM postgres client:
|
73
|
+
|
74
|
+
* https://github.com/jzimmek/em-postgresql-sequel
|
75
|
+
* https://github.com/leftbee/em-postgresql-adapter
|
76
|
+
* https://github.com/jtoy/em-postgres
|
77
|
+
|
78
|
+
and (except the bundled one which uses no longer maintained postgres-pr library)
|
79
|
+
all of them have similiar flaws:
|
80
|
+
|
81
|
+
* 2 of them are designed to support some ORM (ActiveRecord or Sequel)
|
82
|
+
so they are EM-Synchrony only,
|
83
|
+
* non-standard API method names,
|
84
|
+
* no (nonexistent or non-working) autoreconnect implementation,
|
85
|
+
* not fully supporting asynchronous PG::Connection API.
|
86
|
+
|
87
|
+
The last one is worth some comment:
|
88
|
+
|
89
|
+
They all use blocking methods to retrieve whole result from server
|
90
|
+
(PGConn#block() or PGConn#get_result() which also
|
91
|
+
blocks when there is not enough buffered data on socket).
|
92
|
+
|
93
|
+
This implementation makes use of non-blocking: PGConn#is_busy and PGConn#consume_input methods.
|
94
|
+
Depending on size of result sets and concurrency level the gain in overall speed and responsiveness of your
|
95
|
+
application might be actually quite huge. I've done some tests[link:BENCHMARKS.rdoc] already.
|
96
|
+
|
97
|
+
== Thanks
|
98
|
+
|
99
|
+
The greetz go to:
|
100
|
+
- Authors[https://bitbucket.org/ged/ruby-pg/wiki/Home#!copying] of +pg+ driver (especially for its async-api)
|
101
|
+
- Francis Cianfrocca for great reactor framework (EventMachine[https://github.com/eventmachine/eventmachine])
|
102
|
+
- Ilya Grigorik (igrigorik[https://github.com/igrigorik]) for (untangling)[http://www.igvita.com/2010/03/22/untangling-evented-code-with-ruby-fibers/] EM with Fibers
|
103
|
+
|
104
|
+
== USAGE
|
105
|
+
|
106
|
+
=== BASIC
|
107
|
+
|
108
|
+
require 'pg/em'
|
109
|
+
|
110
|
+
# no async
|
111
|
+
pg = PG::EM::Client.new dbname: 'test'
|
112
|
+
pq.query('select * from foo') do |result|
|
113
|
+
puts Array(result).inspect
|
114
|
+
end
|
115
|
+
|
116
|
+
# asynchronous
|
117
|
+
EM.run do
|
118
|
+
pq.query('select * from foo') do |result|
|
119
|
+
raise result if result.is_a? ::Exception
|
120
|
+
puts Array(result).inspect
|
121
|
+
EM.stop
|
122
|
+
end
|
123
|
+
puts "sent"
|
124
|
+
end
|
125
|
+
|
126
|
+
=== AUTORECONNECTING IN ASYNC MODE
|
127
|
+
Autoreconnecting is done in non-blocking manner using #async_reset internally.
|
128
|
+
|
129
|
+
EM.run do
|
130
|
+
pg = PG::EM::Client.new dbname: 'test',
|
131
|
+
connect_timeout: 5, query_timeout: 50
|
132
|
+
try_query = lambda do |&blk|
|
133
|
+
pg.query('select * from foo') do |result|
|
134
|
+
raise result if result.is_a? ::Exception
|
135
|
+
puts Array(result).inspect
|
136
|
+
blk.call
|
137
|
+
end
|
138
|
+
end
|
139
|
+
try_query.call {
|
140
|
+
system 'pg_ctl stop -m fast'
|
141
|
+
system 'pg_ctl start -w'
|
142
|
+
try_query.call { EM.stop }
|
143
|
+
}
|
144
|
+
end
|
145
|
+
|
146
|
+
to disable this feature call:
|
147
|
+
|
148
|
+
pg.async_autoreconnect = false
|
149
|
+
|
150
|
+
or
|
151
|
+
|
152
|
+
pg = PG::EM::Client.new dbname: 'test',
|
153
|
+
async_autoreconnect: false
|
154
|
+
|
155
|
+
It's also possible to define +on_autoreconnect+ callback to be invoked
|
156
|
+
while the connection has been reset. It's called just before the send query
|
157
|
+
command is executed:
|
158
|
+
|
159
|
+
EM.run do
|
160
|
+
pg = PG::EM::Client.new dbname: 'test'
|
161
|
+
pg.prepare('bar', 'select * from foo order by cdate desc') do
|
162
|
+
pg.on_autoreconnect = proc { |c, e|
|
163
|
+
c.prepare('bar', 'select * from foo order by cdate desc')
|
164
|
+
}
|
165
|
+
try_query = lambda do |&blk|
|
166
|
+
pg.exec_prepared('bar') do |result|
|
167
|
+
raise result if result.is_a? ::Exception
|
168
|
+
puts Array(result).inspect
|
169
|
+
blk.call
|
170
|
+
end
|
171
|
+
end
|
172
|
+
try_query.call {
|
173
|
+
system 'pg_ctl stop -m fast'
|
174
|
+
system 'pg_ctl start -w'
|
175
|
+
try_query.call { EM.stop }
|
176
|
+
}
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
As you can see it's possible to send async query from inside on_autoreconnect
|
181
|
+
proc. However you have to pass +Deferrable+ from the async callback to the
|
182
|
+
caller. See +#on_autoreconnect+ docs for details.
|
183
|
+
|
184
|
+
=== TRUE ASYNC
|
185
|
+
For non-blocking connect use PG::EM::Client.async_connect and #async_reset for
|
186
|
+
asynchronous connection reset. Like other async methods they return deferrable object.
|
187
|
+
Use #callback to obtain already connected Client.
|
188
|
+
|
189
|
+
EM.run do
|
190
|
+
pool = (1..10).map {
|
191
|
+
PG::EM::Client.async_connect dbname: 'test',
|
192
|
+
connect_timeout: 5, query_timeout: 50 }
|
193
|
+
|
194
|
+
togo = pool.length
|
195
|
+
|
196
|
+
pool.each_with_index do |df, i|
|
197
|
+
df.callback do |pg|
|
198
|
+
pg.query("select * from foo") do |result|
|
199
|
+
puts "recv: #{i}"
|
200
|
+
EM.stop if (togo-=1).zero?
|
201
|
+
end
|
202
|
+
puts "sent: #{i}"
|
203
|
+
end
|
204
|
+
df.errback { |ex| raise ex }
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
=== EM-Synchrony
|
209
|
+
Under +em-synchrony+ PG::EM::Client.new is fully asynchronous and blocks only current fiber.
|
210
|
+
This also applies to #reset.
|
211
|
+
|
212
|
+
require 'em-synchrony/pg'
|
213
|
+
|
214
|
+
EM.synchrony do
|
215
|
+
pg = PG::EM::Client.new dbname: 'test'
|
216
|
+
pg.query('select * from foo') do |result|
|
217
|
+
puts Array(result).inspect
|
218
|
+
end
|
219
|
+
EM.stop
|
220
|
+
end
|
221
|
+
|
222
|
+
==== Handling errors
|
223
|
+
|
224
|
+
EM.synchrony do
|
225
|
+
begin
|
226
|
+
pg.query('select * from foo') do |result|
|
227
|
+
puts result
|
228
|
+
end
|
229
|
+
rescue PG::Error => e
|
230
|
+
puts "PSQL error: #{e.inspect}"
|
231
|
+
end
|
232
|
+
EM.stop
|
233
|
+
end
|
234
|
+
|
235
|
+
==== Parallel async queries
|
236
|
+
|
237
|
+
EM.synchrony do
|
238
|
+
pg = EM::Synchrony::ConnectionPool.new(size: 2) do
|
239
|
+
PG::EM::Client.new :dbname => 'test'
|
240
|
+
end
|
241
|
+
multi = EventMachine::Synchrony::Multi.new
|
242
|
+
multi.add :foo, pg.aquery('select * from foo') # or #async_query()
|
243
|
+
multi.add :bar, pg.aquery('select * from bar') # #aquery() is just an alias
|
244
|
+
res = multi.perform
|
245
|
+
p res
|
246
|
+
EM.stop
|
247
|
+
end
|
248
|
+
|
249
|
+
==== Fiber Concurrency
|
250
|
+
|
251
|
+
EM.synchrony do
|
252
|
+
# use ConnectionPool when more Fibers will be querying at the same time!
|
253
|
+
pg = EM::Synchrony::ConnectionPool.new(size: 5) do
|
254
|
+
PG::EM::Client.new :dbname => 'test'
|
255
|
+
end
|
256
|
+
counter = 0
|
257
|
+
EM::Synchrony::FiberIterator.new(['select * from foo']*10, 5) do |query|
|
258
|
+
i = counter
|
259
|
+
pg.query(query) do |result|
|
260
|
+
puts "recv: #{i}"
|
261
|
+
end
|
262
|
+
puts "sent: #{i}"
|
263
|
+
counter += 1
|
264
|
+
end
|
265
|
+
EM.stop
|
266
|
+
end
|
267
|
+
|
268
|
+
==== Async reconnect with on_autoreconnect callback
|
269
|
+
|
270
|
+
EM.synchrony do
|
271
|
+
on_autoreconnect = proc do |c, e|
|
272
|
+
c.prepare('bar', 'select * from foo order by cdate desc')
|
273
|
+
end
|
274
|
+
pg = EM::Synchrony::ConnectionPool.new(size: 5) do
|
275
|
+
p = PG::EM::Client.new dbname: 'test', on_autoreconnect: on_autoreconnect
|
276
|
+
on_autoreconnect.call p
|
277
|
+
p
|
278
|
+
end
|
279
|
+
try_query = lambda do
|
280
|
+
pg.exec_prepared('bar') do |result|
|
281
|
+
raise result if result.is_a? ::Exception
|
282
|
+
puts Array(result).inspect
|
283
|
+
end
|
284
|
+
end
|
285
|
+
try_query.call
|
286
|
+
system 'pg_ctl stop -m fast'
|
287
|
+
system 'pg_ctl start -w'
|
288
|
+
try_query.call
|
289
|
+
EM.stop
|
290
|
+
end
|
291
|
+
|
292
|
+
== LICENCE
|
293
|
+
|
294
|
+
The MIT License - Copyright (c) 2012 Rafał Michalski
|