pg_helper 0.3.1 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.ruby-version +1 -0
- data/Gemfile +1 -1
- data/HISTORY.md +3 -0
- data/README.md +4 -0
- data/lib/pg_helper.rb +5 -4
- data/lib/pg_helper/connection_pool.rb +361 -0
- data/lib/pg_helper/query_builder.rb +80 -0
- data/lib/pg_helper/query_helper.rb +137 -148
- data/lib/pg_helper/support_classes.rb +44 -0
- data/lib/pg_helper/version.rb +2 -2
- data/pg_helper.gemspec +4 -2
- data/spec/lib/connection_pool_spec.rb +50 -0
- data/spec/lib/connection_pool_test.rb +310 -0
- data/spec/lib/pg_helper_spec.rb +147 -89
- data/spec/lib/query_builder_spec.rb +91 -0
- data/spec/spec_helper.rb +6 -0
- metadata +58 -40
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 2d1859b5d93f74a231783cba64bd601d701d0485
|
4
|
+
data.tar.gz: 500e32c9a63334b30a967e3b7b35054fca89ad32
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 1945e53defa8da6e57974514d567eeb9d3a0236db0eb26bd15b26d2d63c79ae3f7b173dd31a6488ce834418a9648cf7860179603e7ad1b2b9c89dc1076998602
|
7
|
+
data.tar.gz: a63b6e170f7bbfd367a5676758fd0fe1869202f25d9e5f95071d04cd9f9963a23f5a95d09a0b0e1846e47749a01e3fd149864181ae2417c25d477772914df5c5
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.3.0
|
data/Gemfile
CHANGED
data/HISTORY.md
CHANGED
data/README.md
CHANGED
@@ -1,5 +1,9 @@
|
|
1
1
|
PgHelper
|
2
2
|
=============
|
3
|
+
[![Build Status](https://travis-ci.org/webervin/pg_helper.png)]
|
4
|
+
(https://travis-ci.org/webervin/pg_helper)
|
5
|
+
[![Code Climate](https://codeclimate.com/github/webervin/pg_helper.png)](https://codeclimate.com/github/webervin/pg_helper)
|
6
|
+
[![Dependency Status](https://gemnasium.com/webervin/pg_helper.png)](https://gemnasium.com/webervin/pg_helper)
|
3
7
|
|
4
8
|
Because sometimes I don't want ActiveRecord to parse all fields, nor think about connection.
|
5
9
|
All features are actually provided by [Pg gem](http://rubygems.org/gems/pg)
|
data/lib/pg_helper.rb
CHANGED
@@ -1,7 +1,8 @@
|
|
1
|
-
|
2
|
-
require 'rubygems'
|
1
|
+
# require 'rubygems' < - this will be required by package manager
|
3
2
|
require 'pg'
|
4
3
|
|
5
4
|
# require all of the library files
|
6
|
-
# note, you may need to specify these explicitly if there are any
|
7
|
-
|
5
|
+
# note, you may need to specify these explicitly if there are any
|
6
|
+
# load order dependencies
|
7
|
+
require 'pg_helper/support_classes'
|
8
|
+
require 'pg_helper/query_helper'
|
@@ -0,0 +1,361 @@
|
|
1
|
+
# This file is simplified version of ActiveRecord::Base::ConnectionPool
|
2
|
+
# (from Ruby on Rails framework).
|
3
|
+
#
|
4
|
+
# Original code is:
|
5
|
+
# https://github.com/rails
|
6
|
+
# Distributed under MIT license.
|
7
|
+
#
|
8
|
+
# Rails is developed and maintained by
|
9
|
+
# David Heinemeier Hansson
|
10
|
+
# http://www.rubyonrails.org
|
11
|
+
#
|
12
|
+
# Please note that as code is modified by me Ruby on Rails team is not responsible for any bugs,
|
13
|
+
# bugs are added by me :)
|
14
|
+
# Also note that pg_helper is NOT related to Ruby on Rails in any way
|
15
|
+
|
16
|
+
require 'thread'
|
17
|
+
require 'thread_safe'
|
18
|
+
require 'monitor'
|
19
|
+
require 'set'
|
20
|
+
require 'pg'
|
21
|
+
|
22
|
+
module PgHelper
|
23
|
+
# Raised when a connection could not be obtained within the connection
|
24
|
+
# acquisition timeout period: because max connections in pool
|
25
|
+
# are in use.
|
26
|
+
class CouldNotObtainConnection < RuntimeError
|
27
|
+
end
|
28
|
+
|
29
|
+
class ConnectionPool
|
30
|
+
# nearly standard queue, but with timeout on wait
|
31
|
+
# FIXME: custom class inherit from
|
32
|
+
# http://www.ruby-doc.org/stdlib-2.0/libdoc/thread/rdoc/Queue.html
|
33
|
+
class Queue
|
34
|
+
def initialize(lock = Monitor.new)
|
35
|
+
@lock = lock
|
36
|
+
@cond = @lock.new_cond
|
37
|
+
@num_waiting = 0
|
38
|
+
@queue = []
|
39
|
+
end
|
40
|
+
|
41
|
+
# Test if any threads are currently waiting on the queue.
|
42
|
+
def any_waiting?
|
43
|
+
synchronize do
|
44
|
+
@num_waiting > 0
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
# Return the number of threads currently waiting on this
|
49
|
+
# queue.
|
50
|
+
def num_waiting
|
51
|
+
synchronize do
|
52
|
+
$DEBUG && warn(@num_waiting.to_s)
|
53
|
+
@num_waiting
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
# Add +element+ to the queue. Never blocks.
|
58
|
+
def add(element)
|
59
|
+
synchronize do
|
60
|
+
@queue.push element
|
61
|
+
@cond.signal
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
# If +element+ is in the queue, remove and return it, or nil.
|
66
|
+
def delete(element)
|
67
|
+
synchronize do
|
68
|
+
@queue.delete(element)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
# Remove all elements from the queue.
|
73
|
+
def clear
|
74
|
+
synchronize do
|
75
|
+
@queue.clear
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
# Remove the head of the queue.
|
80
|
+
#
|
81
|
+
# If +timeout+ is not given, remove and return the head the
|
82
|
+
# queue if the number of available elements is strictly
|
83
|
+
# greater than the number of threads currently waiting (that
|
84
|
+
# is, don't jump ahead in line). Otherwise, return nil.
|
85
|
+
#
|
86
|
+
# If +timeout+ is given, block if it there is no element
|
87
|
+
# available, waiting up to +timeout+ seconds for an element to
|
88
|
+
# become available.
|
89
|
+
#
|
90
|
+
# Raises:
|
91
|
+
# - ConnectionTimeoutError if +timeout+ is given and no element
|
92
|
+
# becomes available after +timeout+ seconds,
|
93
|
+
def poll(timeout = nil)
|
94
|
+
synchronize do
|
95
|
+
if timeout
|
96
|
+
no_wait_poll || wait_poll(timeout)
|
97
|
+
else
|
98
|
+
no_wait_poll
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
private
|
104
|
+
|
105
|
+
def synchronize(&block)
|
106
|
+
@lock.synchronize(&block)
|
107
|
+
end
|
108
|
+
|
109
|
+
# Test if the queue currently contains any elements.
|
110
|
+
def any?
|
111
|
+
!@queue.empty?
|
112
|
+
end
|
113
|
+
|
114
|
+
# A thread can remove an element from the queue without
|
115
|
+
# waiting if an only if the number of currently available
|
116
|
+
# connections is strictly greater than the number of waiting
|
117
|
+
# threads.
|
118
|
+
def can_remove_no_wait?
|
119
|
+
@queue.size > @num_waiting
|
120
|
+
end
|
121
|
+
|
122
|
+
# Removes and returns the head of the queue if possible, or nil.
|
123
|
+
def remove
|
124
|
+
@queue.shift
|
125
|
+
end
|
126
|
+
|
127
|
+
# Remove and return the head the queue if the number of
|
128
|
+
# available elements is strictly greater than the number of
|
129
|
+
# threads currently waiting. Otherwise, return nil.
|
130
|
+
def no_wait_poll
|
131
|
+
remove if can_remove_no_wait?
|
132
|
+
end
|
133
|
+
|
134
|
+
# Waits on the queue up to +timeout+ seconds, then removes and
|
135
|
+
# returns the head of the queue.
|
136
|
+
def wait_poll(timeout)
|
137
|
+
@num_waiting += 1
|
138
|
+
|
139
|
+
t0 = Time.now
|
140
|
+
elapsed = 0
|
141
|
+
loop do
|
142
|
+
@cond.wait(timeout - elapsed)
|
143
|
+
|
144
|
+
return remove if any?
|
145
|
+
|
146
|
+
elapsed = Time.now - t0
|
147
|
+
if elapsed >= timeout
|
148
|
+
msg = 'could not obtain a database connection within %0.3f seconds (waited %0.3f seconds)' %
|
149
|
+
[timeout, elapsed]
|
150
|
+
fail CouldNotObtainConnection, msg
|
151
|
+
end
|
152
|
+
end
|
153
|
+
ensure
|
154
|
+
@num_waiting -= 1
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
include MonitorMixin
|
159
|
+
|
160
|
+
attr_accessor :auto_connect, :checkout_timeout
|
161
|
+
attr_reader :connection_options, :connections, :size
|
162
|
+
|
163
|
+
# all opts but :checkout_timeout, :pool, :auto_connect will be passed to PGConn.new
|
164
|
+
def initialize(opts)
|
165
|
+
super()
|
166
|
+
|
167
|
+
connection_opts = opts.dup
|
168
|
+
@checkout_timeout = opts.delete(:checkout_timeout) || 5
|
169
|
+
@size = opts.delete(:pool) || 5
|
170
|
+
@auto_connect = opts.delete(:auto_connect) || true
|
171
|
+
|
172
|
+
@connection_options = connection_opts
|
173
|
+
|
174
|
+
# The cache of reserved connections mapped to threads
|
175
|
+
@reserved_connections = ThreadSafe::Cache.new(initial_capacity: @size)
|
176
|
+
|
177
|
+
@connections = []
|
178
|
+
|
179
|
+
@available = Queue.new self
|
180
|
+
end
|
181
|
+
|
182
|
+
# Retrieve the connection associated with the current thread, or call
|
183
|
+
# #checkout to obtain one if necessary.
|
184
|
+
#
|
185
|
+
# #connection can be called any number of times; the connection is
|
186
|
+
# held in a hash keyed by the thread id.
|
187
|
+
def connection
|
188
|
+
# this is correctly done double-checked locking
|
189
|
+
# (ThreadSafe::Cache's lookups have volatile semantics)
|
190
|
+
@reserved_connections[current_connection_id] || synchronize do
|
191
|
+
@reserved_connections[current_connection_id] ||= checkout
|
192
|
+
end
|
193
|
+
end
|
194
|
+
|
195
|
+
# Is there an open connection that is being used for the current thread?
|
196
|
+
def active_connection?
|
197
|
+
synchronize do
|
198
|
+
@reserved_connections.fetch(current_connection_id) do
|
199
|
+
return false
|
200
|
+
end
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
# Signal that the thread is finished with the current connection.
|
205
|
+
# #release_connection releases the connection-thread association
|
206
|
+
# and returns the connection to the pool.
|
207
|
+
def release_connection(with_id = current_connection_id)
|
208
|
+
synchronize do
|
209
|
+
conn = @reserved_connections.delete(with_id)
|
210
|
+
checkin conn if conn
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
# If a connection already exists yield it to the block. If no connection
|
215
|
+
# exists checkout a connection, yield it to the block, and checkin the
|
216
|
+
# connection when finished.
|
217
|
+
def with_connection
|
218
|
+
connection_id = current_connection_id
|
219
|
+
fresh_connection = true unless active_connection?
|
220
|
+
yield connection
|
221
|
+
ensure
|
222
|
+
release_connection(connection_id) if fresh_connection
|
223
|
+
end
|
224
|
+
|
225
|
+
# Returns true if a connection has already been opened.
|
226
|
+
def connected?
|
227
|
+
synchronize { @connections.any? }
|
228
|
+
end
|
229
|
+
|
230
|
+
# Disconnects all connections in the pool, and clears the pool.
|
231
|
+
def disconnect!
|
232
|
+
synchronize do
|
233
|
+
@reserved_connections.clear
|
234
|
+
@connections.each do |conn|
|
235
|
+
checkin conn
|
236
|
+
$DEBUG && warn("Closing pg connection: #{conn.object_id}")
|
237
|
+
conn.close
|
238
|
+
end
|
239
|
+
@connections = []
|
240
|
+
@available.clear
|
241
|
+
end
|
242
|
+
end
|
243
|
+
|
244
|
+
# Clears the cache which maps classes.
|
245
|
+
def clear_reloadable_connections!
|
246
|
+
synchronize do
|
247
|
+
@reserved_connections.clear
|
248
|
+
@connections.each do |conn|
|
249
|
+
checkin conn
|
250
|
+
end
|
251
|
+
|
252
|
+
@connections.delete_if(&:finished?)
|
253
|
+
|
254
|
+
@available.clear
|
255
|
+
@connections.each do |conn|
|
256
|
+
@available.add conn
|
257
|
+
end
|
258
|
+
end
|
259
|
+
end
|
260
|
+
|
261
|
+
# Check-out a database connection from the pool, indicating that you want
|
262
|
+
# to use it. You should call #checkin when you no longer need this.
|
263
|
+
#
|
264
|
+
# This is done by either returning and leasing existing connection, or by
|
265
|
+
# creating a new connection and leasing it.
|
266
|
+
#
|
267
|
+
# If all connections are leased and the pool is at capacity (meaning the
|
268
|
+
# number of currently leased connections is greater than or equal to the
|
269
|
+
# size limit set), an ActiveRecord::ConnectionTimeoutError exception will be raised.
|
270
|
+
#
|
271
|
+
# Returns: an AbstractAdapter object.
|
272
|
+
#
|
273
|
+
# Raises:
|
274
|
+
# - ConnectionTimeoutError: no connection can be obtained from the pool.
|
275
|
+
def checkout
|
276
|
+
synchronize do
|
277
|
+
conn = acquire_connection
|
278
|
+
checkout_and_verify(conn)
|
279
|
+
end
|
280
|
+
end
|
281
|
+
|
282
|
+
# Check-in a database connection back into the pool, indicating that you
|
283
|
+
# no longer need this connection.
|
284
|
+
#
|
285
|
+
# +conn+: an AbstractAdapter object, which was obtained by earlier by
|
286
|
+
# calling +checkout+ on this pool.
|
287
|
+
def checkin(conn)
|
288
|
+
synchronize do
|
289
|
+
release conn
|
290
|
+
@available.add conn
|
291
|
+
end
|
292
|
+
end
|
293
|
+
# # Remove a connection from the connection pool. The connection will
|
294
|
+
# # remain open and active but will no longer be managed by this pool.
|
295
|
+
# def remove(conn)
|
296
|
+
# synchronize do
|
297
|
+
# @connections.delete conn
|
298
|
+
# @available.delete conn
|
299
|
+
#
|
300
|
+
# # FIXME: we might want to store the key on the connection so that removing
|
301
|
+
# # from the reserved hash will be a little easier.
|
302
|
+
# release conn
|
303
|
+
#
|
304
|
+
# @available.add checkout_new_connection if @available.any_waiting?
|
305
|
+
# end
|
306
|
+
# end
|
307
|
+
|
308
|
+
private
|
309
|
+
|
310
|
+
# Acquire a connection by one of 1) immediately removing one
|
311
|
+
# from the queue of available connections, 2) creating a new
|
312
|
+
# connection if the pool is not at capacity, 3) waiting on the
|
313
|
+
# queue for a connection to become available.
|
314
|
+
#
|
315
|
+
# Raises:
|
316
|
+
# - ConnectionTimeoutError if a connection could not be acquired
|
317
|
+
def acquire_connection
|
318
|
+
if conn = @available.poll
|
319
|
+
conn
|
320
|
+
elsif @connections.size < @size
|
321
|
+
checkout_new_connection
|
322
|
+
else
|
323
|
+
@available.poll(@checkout_timeout)
|
324
|
+
end
|
325
|
+
end
|
326
|
+
|
327
|
+
def release(conn)
|
328
|
+
thread_id = if @reserved_connections[current_connection_id] == conn
|
329
|
+
current_connection_id
|
330
|
+
else
|
331
|
+
@reserved_connections.keys.find do |k|
|
332
|
+
@reserved_connections[k] == conn
|
333
|
+
end
|
334
|
+
end
|
335
|
+
|
336
|
+
@reserved_connections.delete thread_id if thread_id
|
337
|
+
end
|
338
|
+
|
339
|
+
def new_connection
|
340
|
+
conn = PGconn.open(connection_options)
|
341
|
+
$DEBUG && warn("Connected to PostgreSQL #{conn.server_version} (#{conn.object_id})")
|
342
|
+
conn
|
343
|
+
end
|
344
|
+
|
345
|
+
def current_connection_id #:nodoc:
|
346
|
+
Thread.current.object_id
|
347
|
+
end
|
348
|
+
|
349
|
+
def checkout_new_connection
|
350
|
+
fail ConnectionNotEstablished unless @auto_connect
|
351
|
+
c = new_connection
|
352
|
+
@connections << c
|
353
|
+
c
|
354
|
+
end
|
355
|
+
|
356
|
+
def checkout_and_verify(c)
|
357
|
+
c.reset
|
358
|
+
c
|
359
|
+
end
|
360
|
+
end
|
361
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
module PgHelper
|
2
|
+
class QueryBuilder
|
3
|
+
attr_reader :table_name
|
4
|
+
|
5
|
+
def self.from(table_name)
|
6
|
+
new(table_name)
|
7
|
+
end
|
8
|
+
|
9
|
+
def initialize(table_name)
|
10
|
+
@table_name = table_name
|
11
|
+
@selects = []
|
12
|
+
@where = []
|
13
|
+
@cte_list = []
|
14
|
+
@join_list = []
|
15
|
+
end
|
16
|
+
|
17
|
+
# http://www.postgresql.org/docs/9.2/static/sql-select.html
|
18
|
+
def to_sql
|
19
|
+
"#{with_list}"\
|
20
|
+
"SELECT #{column_list} "\
|
21
|
+
"FROM #{table_name}"\
|
22
|
+
"#{join_list}"\
|
23
|
+
"#{where_list}"
|
24
|
+
end
|
25
|
+
|
26
|
+
def select(value)
|
27
|
+
@selects << value
|
28
|
+
self
|
29
|
+
end
|
30
|
+
|
31
|
+
def where(condition)
|
32
|
+
@where << condition
|
33
|
+
self
|
34
|
+
end
|
35
|
+
|
36
|
+
def with(cte_name, cte_query)
|
37
|
+
@cte_list << "#{cte_name} AS (#{cte_query})"
|
38
|
+
self
|
39
|
+
end
|
40
|
+
|
41
|
+
def join(join_sql)
|
42
|
+
@join_list << join_sql
|
43
|
+
self
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
def join_list
|
49
|
+
if @join_list.empty?
|
50
|
+
nil
|
51
|
+
else
|
52
|
+
" #{@join_list.join(' ')}"
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def with_list
|
57
|
+
if @cte_list.empty?
|
58
|
+
nil
|
59
|
+
else
|
60
|
+
"WITH #{@cte_list.join(',')} "
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def where_list
|
65
|
+
if @where.empty?
|
66
|
+
nil
|
67
|
+
else
|
68
|
+
" WHERE #{@where.join(' AND ')}"
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def column_list
|
73
|
+
if @selects.empty?
|
74
|
+
'*'
|
75
|
+
else
|
76
|
+
@selects.join(',')
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|