pg_versions 1.0 → 2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/pg_versions/pg_versions.rb +193 -88
- data/lib/pg_versions/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f9b1174e7cd188f4950927c333ada56e38efa995668e613faac794dfe1a12f99
|
4
|
+
data.tar.gz: 93bef8d80d24f74577c88789291d17c1ad7b585b53aa65ed7b639a6e9bed039d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a883f123cb0ba892d37fd27ad9fa795a399c0903719670f011226b638429dd719990c2334d0bac1e65c167493130350df050e3ee6ce10c51f8bb52a9194336e8
|
7
|
+
data.tar.gz: 1daf1d513b2de942cfae39b74c2113a75129ccc2b47e9ed87f97fee20e3dc4b20742d808ff56bd6a4edac6d61a545a116bd7cf4621eebd4ea94ee6ab616fa3ba
|
@@ -1,5 +1,5 @@
|
|
1
1
|
# Operations on versions:
|
2
|
-
# 1. bump - increase or assert version. ensures new version is unique and higher than any previous version, even if table entry existed
|
2
|
+
# 1. bump - increase or assert version. ensures new version is unique and higher than any previous version, even if table entry existed before and got removed
|
3
3
|
# 2. read - always returns a version, even if table entry is missing
|
4
4
|
# 3. remove (clean) - for periodic pruning of old entries
|
5
5
|
#
|
@@ -32,22 +32,28 @@ require 'set'
|
|
32
32
|
module PgVersions
|
33
33
|
|
34
34
|
|
35
|
-
class
|
35
|
+
class InvalidParameters < StandardError; end
|
36
36
|
|
37
37
|
def self.timestamp_to_integers(input)
|
38
38
|
"to_char(%s, 'YYYYMMDD')::integer || ',' || to_char(%s, 'HH24MISS')::integer || ',' || to_char(%s, 'US')::integer"%[input, input, input]
|
39
39
|
end
|
40
40
|
|
41
41
|
|
42
|
-
def self.with_connection(pg_connection)
|
43
|
-
if pg_connection
|
44
|
-
|
45
|
-
|
42
|
+
def self.with_connection(pg_connection, reset: false, &block)
|
43
|
+
if pg_connection.kind_of? PG::Connection
|
44
|
+
if reset
|
45
|
+
pg_connection.sync_reset
|
46
|
+
pg_connection.exec("select;")
|
47
|
+
end
|
48
|
+
block.call(pg_connection)
|
49
|
+
elsif pg_connection.respond_to? :call
|
50
|
+
pg_connection.call(reset, &block)
|
51
|
+
elsif pg_connection.nil? and defined? ActiveRecord
|
46
52
|
ActiveRecord::Base.connection_pool.with_connection { |ar_connection|
|
47
|
-
|
53
|
+
block.call(ar_connection.instance_variable_get(:@connection))
|
48
54
|
}
|
49
55
|
else
|
50
|
-
raise
|
56
|
+
raise InvalidParameters, "Missing connection. Either pass pg connection object or import ActiveRecord."
|
51
57
|
end
|
52
58
|
end
|
53
59
|
|
@@ -135,89 +141,193 @@ module PgVersions
|
|
135
141
|
|
136
142
|
|
137
143
|
class Notification
|
138
|
-
attr_reader :
|
139
|
-
def initialize(
|
140
|
-
@
|
144
|
+
attr_reader :changed_versions, :changed, :all_versions, :versions
|
145
|
+
def initialize(changed_versions, all_versions)
|
146
|
+
@changed_versions, @all_versions = changed_versions, all_versions
|
147
|
+
@changed, @versions = changed_versions, all_versions
|
141
148
|
end
|
142
149
|
end
|
143
150
|
|
144
151
|
|
145
|
-
class Connection
|
146
|
-
|
147
|
-
def initialize(connection=nil)
|
148
|
-
@actor_commands = Queue.new
|
149
|
-
actor_notify_r, @actor_notify_w = IO.pipe
|
150
152
|
|
151
|
-
|
152
|
-
|
153
|
+
class ConnectionThread
|
154
|
+
|
155
|
+
class Closing < StandardError
|
156
|
+
attr_reader :retpipe
|
157
|
+
def initialize(retpipe)
|
158
|
+
@retpipe = retpipe
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
|
163
|
+
attr_reader :status
|
164
|
+
|
165
|
+
def initialize(connection)
|
166
|
+
retry_on_exceptions = [ PG::ConnectionBad, PG::UnableToSend ]
|
167
|
+
retry_on_exceptions << ActiveRecord::ConnectionNotEstablished if defined? ActiveRecord
|
168
|
+
|
169
|
+
@subscribers = Hash.new { |h,k| h[k] = Set.new }
|
170
|
+
@status = :disconnected
|
171
|
+
|
172
|
+
@thread_requests_mutex = Mutex.new
|
173
|
+
@thread_requests = []
|
174
|
+
thread_requests_notify_r, @thread_requests_notify_w = IO.pipe
|
175
|
+
|
176
|
+
@thread = Thread.new {
|
177
|
+
reset_connection = false
|
178
|
+
retry_delay = 0
|
153
179
|
begin
|
154
|
-
PgVersions.with_connection(connection)
|
155
|
-
|
156
|
-
|
180
|
+
PgVersions.with_connection(connection, reset: reset_connection) do |pg_connection|
|
181
|
+
|
182
|
+
@status = :connected
|
183
|
+
retry_delay = 0
|
184
|
+
|
185
|
+
@subscribers.each_key { |channel|
|
186
|
+
listen(pg_connection, channel)
|
187
|
+
}
|
188
|
+
read(pg_connection, @subscribers.keys).each_pair { |channel, version|
|
189
|
+
#p channel, version
|
190
|
+
@subscribers[channel].each { |subscriber|
|
191
|
+
subscriber.notify({ channel => version })
|
192
|
+
}
|
193
|
+
}
|
194
|
+
|
157
195
|
loop {
|
196
|
+
@thread_requests_mutex.synchronize {
|
197
|
+
@thread_requests.each { |function, retpipe, params|
|
198
|
+
raise Closing, retpipe if function == :stop
|
199
|
+
retpipe << send(function, pg_connection, *params)
|
200
|
+
}
|
201
|
+
@thread_requests.clear
|
202
|
+
}
|
203
|
+
|
204
|
+
while notification = pg_connection.notifies
|
205
|
+
channel, payload = notification[:relname], notification[:extra]
|
206
|
+
@subscribers[channel].each { |subscriber|
|
207
|
+
subscriber.notify({ channel => PgVersions.string_to_version(payload) })
|
208
|
+
}
|
209
|
+
end
|
210
|
+
|
158
211
|
#TODO: handle errors
|
159
|
-
reads,_writes,_errors = IO::select([pg_connection.socket_io,
|
212
|
+
reads,_writes,_errors = IO::select([pg_connection.socket_io, thread_requests_notify_r])
|
160
213
|
|
161
214
|
if reads.include?(pg_connection.socket_io)
|
162
215
|
pg_connection.consume_input
|
163
216
|
end
|
164
217
|
|
165
|
-
if reads.include?(
|
166
|
-
|
167
|
-
actor_notify_r.read(1)
|
168
|
-
end
|
169
|
-
|
170
|
-
while notification = pg_connection.notifies
|
171
|
-
channel, payload = notification[:relname], notification[:extra]
|
172
|
-
subscribers[channel].each { |subscriber|
|
173
|
-
subscriber.notify(channel, PgVersions.string_to_version(payload))
|
174
|
-
}
|
218
|
+
if reads.include?(thread_requests_notify_r)
|
219
|
+
thread_requests_notify_r.read(1)
|
175
220
|
end
|
176
221
|
}
|
177
|
-
|
178
|
-
|
179
|
-
|
222
|
+
rescue Closing => e
|
223
|
+
e.retpipe << true
|
224
|
+
end
|
225
|
+
rescue *retry_on_exceptions => e
|
226
|
+
reset_connection = true
|
227
|
+
@status = :disconnected
|
228
|
+
$stderr.puts "Pg connection failed. Retrying in #{retry_delay/1000.0}s."
|
229
|
+
sleep retry_delay/1000.0
|
230
|
+
retry_delay = { 0=>100, 100=>1000, 1000=>2000, 2000=>2000 }[retry_delay]
|
231
|
+
retry
|
232
|
+
end
|
233
|
+
}
|
234
|
+
@thread.abort_on_exception = true
|
235
|
+
end
|
236
|
+
|
237
|
+
private def listen(pg_connection, channel)
|
238
|
+
pg_connection.exec("LISTEN #{PG::Connection.quote_ident(channel)}")
|
239
|
+
end
|
240
|
+
|
241
|
+
private def unlisten(pg_connection, channel)
|
242
|
+
pg_connection.exec("UNLISTEN #{PG::Connection.quote_ident(channel)}")
|
243
|
+
end
|
244
|
+
|
245
|
+
private def subscribe(pg_connection, subscriber, channels)
|
246
|
+
channels.each { |channel|
|
247
|
+
@subscribers[channel] << subscriber
|
248
|
+
listen(pg_connection, channel) if @subscribers[channel].size == 1
|
249
|
+
}
|
250
|
+
subscriber.notify(PgVersions.read(channels, connection: pg_connection))
|
251
|
+
true
|
252
|
+
end
|
253
|
+
|
254
|
+
private def unsubscribe(pg_connection, subscriber, channels)
|
255
|
+
channels.each { |channel|
|
256
|
+
@subscribers[channel].delete(subscriber)
|
257
|
+
if @subscribers[channel].size == 0
|
258
|
+
unlisten(pg_connection, channel)
|
259
|
+
@subscribers.delete(channel)
|
180
260
|
end
|
181
261
|
}
|
182
|
-
|
262
|
+
true
|
183
263
|
end
|
184
264
|
|
265
|
+
private def read(pg_connection, channels)
|
266
|
+
PgVersions.read(channels, connection: pg_connection)
|
267
|
+
end
|
185
268
|
|
186
|
-
def
|
187
|
-
|
188
|
-
@actor_commands << proc { |pg_connection, subscribers|
|
189
|
-
done << block.call(pg_connection, subscribers)
|
190
|
-
}
|
191
|
-
@actor_notify_w.write('!')
|
192
|
-
done.shift
|
269
|
+
private def bump(pg_connection, channels)
|
270
|
+
PgVersions.bump(channels, connection: pg_connection)
|
193
271
|
end
|
194
272
|
|
273
|
+
def request(function, *params)
|
274
|
+
request_nonblock(function, *params).pop
|
275
|
+
end
|
195
276
|
|
196
|
-
def
|
197
|
-
|
198
|
-
|
277
|
+
def request_nonblock(function, *params)
|
278
|
+
retpipe = Queue.new
|
279
|
+
@thread_requests_mutex.synchronize {
|
280
|
+
@thread_requests.push [function, retpipe, params]
|
199
281
|
}
|
282
|
+
@thread_requests_notify_w.write('!')
|
283
|
+
retpipe
|
284
|
+
end
|
285
|
+
|
286
|
+
end
|
287
|
+
|
288
|
+
class Connection
|
289
|
+
|
290
|
+
def initialize(connection=nil)
|
291
|
+
@connection_thread = ConnectionThread.new(connection)
|
292
|
+
end
|
293
|
+
|
294
|
+
def close()
|
295
|
+
@connection_thread.request(:stop)
|
296
|
+
end
|
297
|
+
|
298
|
+
def bump(*channels)
|
299
|
+
@connection_thread.request(:bump, channels)
|
200
300
|
end
|
201
301
|
|
202
302
|
|
203
303
|
def read(*channels)
|
204
|
-
|
205
|
-
PgVersions.read(channels, connection: pg_connection)
|
206
|
-
}
|
304
|
+
@connection_thread.request(:read, channels)
|
207
305
|
end
|
208
306
|
|
209
307
|
|
210
308
|
def subscribe(*channels, known: {})
|
211
|
-
subscription = Subscription.new(
|
309
|
+
subscription = Subscription.new(@connection_thread)
|
212
310
|
subscription.subscribe([channels].flatten, known: known)
|
213
|
-
|
311
|
+
if block_given?
|
312
|
+
Thread.handle_interrupt(Object => :never) {
|
313
|
+
begin
|
314
|
+
Thread.handle_interrupt(Object => :immediate) {
|
315
|
+
yield subscription
|
316
|
+
}
|
317
|
+
ensure
|
318
|
+
subscription.drop
|
319
|
+
end
|
320
|
+
}
|
321
|
+
else
|
322
|
+
subscription
|
323
|
+
end
|
214
324
|
end
|
215
325
|
|
216
326
|
|
217
327
|
class Subscription
|
218
328
|
|
219
|
-
def initialize(
|
220
|
-
@
|
329
|
+
def initialize(connection_thread)
|
330
|
+
@connection_thread = connection_thread
|
221
331
|
@notifications = Queue.new
|
222
332
|
@already_known_versions = Hash.new { |h,k| h[k] = [] }
|
223
333
|
@channels = Hash.new(0)
|
@@ -231,15 +341,7 @@ module PgVersions
|
|
231
341
|
(@channels[channel] += 1) == 1
|
232
342
|
}
|
233
343
|
if channels.size > 0
|
234
|
-
@
|
235
|
-
channels.each { |channel|
|
236
|
-
subscribers[channel] << self
|
237
|
-
pg_connection.exec("LISTEN #{PG::Connection.quote_ident(channel)}") if subscribers[channel].size == 1
|
238
|
-
}
|
239
|
-
PgVersions.read(channels, connection: pg_connection).each_pair { |channel, version|
|
240
|
-
notify(channel, version)
|
241
|
-
}
|
242
|
-
}
|
344
|
+
@connection_thread.request(:subscribe, self, channels)
|
243
345
|
end
|
244
346
|
end
|
245
347
|
|
@@ -252,59 +354,62 @@ module PgVersions
|
|
252
354
|
@channels.delete(channel) if @channels[channel] == 0
|
253
355
|
not @channels.has_key?(channel)
|
254
356
|
}
|
255
|
-
@
|
256
|
-
channels.each { |channel|
|
257
|
-
subscribers[channel].delete(self)
|
258
|
-
if subscribers[channel].size == 0
|
259
|
-
pg_connection.exec("UNLISTEN #{PG::Connection.quote_ident(channel)}")
|
260
|
-
subscribers.delete(channel)
|
261
|
-
end
|
262
|
-
}
|
263
|
-
}
|
357
|
+
@connection_thread.request(:unsubscribe, self, channels)
|
264
358
|
end
|
265
359
|
|
266
360
|
|
267
|
-
def read(*channels, notify:
|
361
|
+
def read(*channels, notify: false)
|
268
362
|
channels = @channels.keys if channels.size == 0
|
269
|
-
versions = @
|
270
|
-
PgVersions.read(channels, connection: pg_connection)
|
271
|
-
}
|
363
|
+
versions = @connection_thread.request(:read, channels)
|
272
364
|
update_already_known_versions(versions) if not notify
|
273
365
|
versions
|
274
366
|
end
|
275
367
|
|
276
368
|
|
277
|
-
def bump(*channels, notify:
|
369
|
+
def bump(*channels, notify: false)
|
278
370
|
channels = @channels.keys if channels.size == 0
|
279
|
-
versions = @
|
280
|
-
PgVersions.bump(channels, connection: pg_connection)
|
281
|
-
}
|
371
|
+
versions = @connection_thread.request(:bump, channels)
|
282
372
|
update_already_known_versions(versions) if not notify
|
283
373
|
versions
|
284
374
|
end
|
285
375
|
|
286
376
|
|
377
|
+
#TODO: make this resume-able after forced exception
|
287
378
|
def wait(new_already_known_versions = {})
|
288
379
|
update_already_known_versions(new_already_known_versions)
|
289
380
|
loop {
|
290
|
-
|
291
|
-
return nil if not
|
292
|
-
|
293
|
-
@already_known_versions[channel]
|
294
|
-
|
381
|
+
versions = @notifications.shift
|
382
|
+
return nil if not versions #termination
|
383
|
+
changed_versions = versions.to_a.map { |channel, version|
|
384
|
+
if (@already_known_versions[channel] <=> version) == -1
|
385
|
+
@already_known_versions[channel] = version
|
386
|
+
[channel, version]
|
387
|
+
end
|
388
|
+
}.compact.to_h
|
389
|
+
if changed_versions.size > 0
|
390
|
+
return Notification.new(changed_versions, @already_known_versions.dup)
|
295
391
|
end
|
296
392
|
}
|
297
393
|
end
|
298
394
|
|
299
395
|
|
300
|
-
def
|
301
|
-
|
396
|
+
def each(new_already_known_versions = {})
|
397
|
+
update_already_known_versions(new_already_known_versions)
|
398
|
+
while notification = wait()
|
399
|
+
yield notification
|
400
|
+
end
|
401
|
+
end
|
402
|
+
|
403
|
+
|
404
|
+
def notify(versions)
|
405
|
+
@notifications << versions
|
302
406
|
end
|
303
407
|
|
304
408
|
|
305
409
|
def drop
|
306
|
-
@notifications <<
|
307
|
-
unsubscribe
|
410
|
+
@notifications << nil
|
411
|
+
@connection_thread.request_nonblock(:unsubscribe, self, @channels.keys)
|
412
|
+
#TODO: what to do if this object gets used after drop?
|
308
413
|
end
|
309
414
|
|
310
415
|
|
data/lib/pg_versions/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pg_versions
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '
|
4
|
+
version: '2.0'
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- yunta
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-01-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rspec
|
@@ -100,7 +100,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
100
100
|
- !ruby/object:Gem::Version
|
101
101
|
version: '0'
|
102
102
|
requirements: []
|
103
|
-
rubygems_version: 3.
|
103
|
+
rubygems_version: 3.3.26
|
104
104
|
signing_key:
|
105
105
|
specification_version: 4
|
106
106
|
summary: Persistent timestamped postgres notification library
|