pg_versions 1.0 → 2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/pg_versions/pg_versions.rb +209 -93
- data/lib/pg_versions/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f6eab4d5b85196e9a17fe15efdb09cb6ef83d6821a060c40a5c68e3fae7c9fed
|
4
|
+
data.tar.gz: ff1c5c4157163144f7bb1808115c52e50438d8119415b7ea19df18e82e9d5bcd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 12c82c86f3138b9b27eaade5a3eb97779b246de26a680de3e23e6d99daa576cc56a1b04ce2c8b82b81f72f8402a73f5eb2e5e7d4babb57651d458317f40eeea5
|
7
|
+
data.tar.gz: 70c41180db0eff19a524f1b65165800c08a50a22ddf25f07daaacb58a0c0b901e0bbf22b620f61499f52788ec4d9120de373b2cdbb3a1aaae3d02481dc211bd3
|
@@ -1,5 +1,5 @@
|
|
1
1
|
# Operations on versions:
|
2
|
-
# 1. bump - increase or assert version. ensures new version is unique and higher than any previous version, even if table entry existed
|
2
|
+
# 1. bump - increase or assert version. ensures new version is unique and higher than any previous version, even if table entry existed before and got removed
|
3
3
|
# 2. read - always returns a version, even if table entry is missing
|
4
4
|
# 3. remove (clean) - for periodic pruning of old entries
|
5
5
|
#
|
@@ -32,22 +32,28 @@ require 'set'
|
|
32
32
|
module PgVersions
|
33
33
|
|
34
34
|
|
35
|
-
class
|
35
|
+
class InvalidParameters < StandardError; end
|
36
36
|
|
37
37
|
def self.timestamp_to_integers(input)
|
38
38
|
"to_char(%s, 'YYYYMMDD')::integer || ',' || to_char(%s, 'HH24MISS')::integer || ',' || to_char(%s, 'US')::integer"%[input, input, input]
|
39
39
|
end
|
40
40
|
|
41
41
|
|
42
|
-
def self.with_connection(pg_connection)
|
43
|
-
if pg_connection
|
44
|
-
|
45
|
-
|
42
|
+
def self.with_connection(pg_connection, reset: false, &block)
|
43
|
+
if pg_connection.kind_of? PG::Connection
|
44
|
+
if reset
|
45
|
+
pg_connection.sync_reset
|
46
|
+
pg_connection.exec("select;")
|
47
|
+
end
|
48
|
+
block.call(pg_connection)
|
49
|
+
elsif pg_connection.respond_to? :call
|
50
|
+
pg_connection.call(reset, &block)
|
51
|
+
elsif pg_connection.nil? and defined? ActiveRecord
|
46
52
|
ActiveRecord::Base.connection_pool.with_connection { |ar_connection|
|
47
|
-
|
53
|
+
block.call(ar_connection.instance_variable_get(:@connection))
|
48
54
|
}
|
49
55
|
else
|
50
|
-
raise
|
56
|
+
raise InvalidParameters, "Missing connection. Either pass pg connection object or import ActiveRecord."
|
51
57
|
end
|
52
58
|
end
|
53
59
|
|
@@ -75,6 +81,7 @@ module PgVersions
|
|
75
81
|
|
76
82
|
#TODO: ensure this is called only once per transaction, or that all bumps occur in the same order in all transactions, to avoid deadlocks
|
77
83
|
def self.bump(*channels, connection: nil)
|
84
|
+
#TODO: pg_connection.exec returned nil once during testing.
|
78
85
|
PgVersions.with_connection(connection) { |pg_connection|
|
79
86
|
channels = [channels].flatten.sort
|
80
87
|
return {} if channels.size == 0
|
@@ -101,7 +108,9 @@ module PgVersions
|
|
101
108
|
FROM
|
102
109
|
to_bump
|
103
110
|
JOIN updated ON to_bump.channel = updated.channel;
|
104
|
-
")
|
111
|
+
") { |result|
|
112
|
+
result.map { |row| [channels[Integer(row["i"])], string_to_version(row["version"])] }.to_h
|
113
|
+
}
|
105
114
|
}
|
106
115
|
end
|
107
116
|
|
@@ -124,8 +133,10 @@ module PgVersions
|
|
124
133
|
JOIN pg_versions ON pg_versions.channel = channels.channel
|
125
134
|
ORDER BY
|
126
135
|
i DESC;
|
127
|
-
")
|
128
|
-
|
136
|
+
") { |result|
|
137
|
+
result.each { |row|
|
138
|
+
versions[channels.delete_at(Integer(row["i"]))] = string_to_version(row["version"])
|
139
|
+
}
|
129
140
|
}
|
130
141
|
#TODO: bump in the same query instead of calling bump
|
131
142
|
versions.merge!(self.bump(channels, connection: pg_connection)) if channels.size > 0
|
@@ -135,89 +146,193 @@ module PgVersions
|
|
135
146
|
|
136
147
|
|
137
148
|
class Notification
|
138
|
-
attr_reader :
|
139
|
-
def initialize(
|
140
|
-
@
|
149
|
+
attr_reader :changed_versions, :changed, :all_versions, :versions
|
150
|
+
def initialize(changed_versions, all_versions)
|
151
|
+
@changed_versions, @all_versions = changed_versions, all_versions
|
152
|
+
@changed, @versions = changed_versions, all_versions
|
141
153
|
end
|
142
154
|
end
|
143
155
|
|
144
156
|
|
145
|
-
class Connection
|
146
|
-
|
147
|
-
def initialize(connection=nil)
|
148
|
-
@actor_commands = Queue.new
|
149
|
-
actor_notify_r, @actor_notify_w = IO.pipe
|
150
157
|
|
151
|
-
|
152
|
-
|
158
|
+
class ConnectionThread
|
159
|
+
|
160
|
+
class Closing < StandardError
|
161
|
+
attr_reader :retpipe
|
162
|
+
def initialize(retpipe)
|
163
|
+
@retpipe = retpipe
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
167
|
+
|
168
|
+
attr_reader :status
|
169
|
+
|
170
|
+
def initialize(connection)
|
171
|
+
retry_on_exceptions = [ PG::ConnectionBad, PG::UnableToSend ]
|
172
|
+
retry_on_exceptions << ActiveRecord::ConnectionNotEstablished if defined? ActiveRecord
|
173
|
+
|
174
|
+
@subscribers = Hash.new { |h,k| h[k] = Set.new }
|
175
|
+
@status = :disconnected
|
176
|
+
|
177
|
+
@thread_requests_mutex = Mutex.new
|
178
|
+
@thread_requests = []
|
179
|
+
thread_requests_notify_r, @thread_requests_notify_w = IO.pipe
|
180
|
+
|
181
|
+
@thread = Thread.new {
|
182
|
+
reset_connection = false
|
183
|
+
retry_delay = 0
|
153
184
|
begin
|
154
|
-
PgVersions.with_connection(connection)
|
155
|
-
|
156
|
-
|
185
|
+
PgVersions.with_connection(connection, reset: reset_connection) do |pg_connection|
|
186
|
+
|
187
|
+
@status = :connected
|
188
|
+
retry_delay = 0
|
189
|
+
|
190
|
+
@subscribers.each_key { |channel|
|
191
|
+
listen(pg_connection, channel)
|
192
|
+
}
|
193
|
+
read(pg_connection, @subscribers.keys).each_pair { |channel, version|
|
194
|
+
@subscribers[channel].each { |subscriber|
|
195
|
+
subscriber.notify({ channel => version })
|
196
|
+
}
|
197
|
+
}
|
198
|
+
|
157
199
|
loop {
|
200
|
+
@thread_requests_mutex.synchronize {
|
201
|
+
@thread_requests.each { |function, retpipe, params|
|
202
|
+
raise Closing, retpipe if function == :stop
|
203
|
+
retpipe << send(function, pg_connection, *params)
|
204
|
+
}
|
205
|
+
@thread_requests.clear
|
206
|
+
}
|
207
|
+
|
208
|
+
while notification = pg_connection.notifies
|
209
|
+
channel, payload = notification[:relname], notification[:extra]
|
210
|
+
@subscribers[channel].each { |subscriber|
|
211
|
+
subscriber.notify({ channel => PgVersions.string_to_version(payload) })
|
212
|
+
}
|
213
|
+
end
|
214
|
+
|
158
215
|
#TODO: handle errors
|
159
|
-
reads,_writes,_errors = IO::select([pg_connection.socket_io,
|
216
|
+
reads,_writes,_errors = IO::select([pg_connection.socket_io, thread_requests_notify_r])
|
160
217
|
|
161
218
|
if reads.include?(pg_connection.socket_io)
|
162
219
|
pg_connection.consume_input
|
163
220
|
end
|
164
221
|
|
165
|
-
if reads.include?(
|
166
|
-
|
167
|
-
actor_notify_r.read(1)
|
168
|
-
end
|
169
|
-
|
170
|
-
while notification = pg_connection.notifies
|
171
|
-
channel, payload = notification[:relname], notification[:extra]
|
172
|
-
subscribers[channel].each { |subscriber|
|
173
|
-
subscriber.notify(channel, PgVersions.string_to_version(payload))
|
174
|
-
}
|
222
|
+
if reads.include?(thread_requests_notify_r)
|
223
|
+
thread_requests_notify_r.read(1)
|
175
224
|
end
|
176
225
|
}
|
177
|
-
|
178
|
-
|
179
|
-
|
226
|
+
rescue Closing => e
|
227
|
+
e.retpipe << true
|
228
|
+
end
|
229
|
+
rescue *retry_on_exceptions => e
|
230
|
+
reset_connection = true
|
231
|
+
@status = :disconnected
|
232
|
+
$stderr.puts "Pg connection failed. Retrying in #{retry_delay/1000.0}s."
|
233
|
+
sleep retry_delay/1000.0
|
234
|
+
retry_delay = { 0=>100, 100=>1000, 1000=>2000, 2000=>2000 }[retry_delay]
|
235
|
+
retry
|
236
|
+
end
|
237
|
+
}
|
238
|
+
@thread.abort_on_exception = true
|
239
|
+
end
|
240
|
+
|
241
|
+
private def listen(pg_connection, channel)
|
242
|
+
pg_connection.exec("LISTEN #{PG::Connection.quote_ident(channel)}")
|
243
|
+
end
|
244
|
+
|
245
|
+
private def unlisten(pg_connection, channel)
|
246
|
+
pg_connection.exec("UNLISTEN #{PG::Connection.quote_ident(channel)}")
|
247
|
+
end
|
248
|
+
|
249
|
+
private def subscribe(pg_connection, subscriber, channels)
|
250
|
+
channels.each { |channel|
|
251
|
+
@subscribers[channel] << subscriber
|
252
|
+
listen(pg_connection, channel) if @subscribers[channel].size == 1
|
253
|
+
}
|
254
|
+
subscriber.notify(PgVersions.read(channels, connection: pg_connection))
|
255
|
+
true
|
256
|
+
end
|
257
|
+
|
258
|
+
private def unsubscribe(pg_connection, subscriber, channels)
|
259
|
+
channels.each { |channel|
|
260
|
+
@subscribers[channel].delete(subscriber)
|
261
|
+
if @subscribers[channel].size == 0
|
262
|
+
unlisten(pg_connection, channel)
|
263
|
+
@subscribers.delete(channel)
|
180
264
|
end
|
181
265
|
}
|
182
|
-
|
266
|
+
true
|
183
267
|
end
|
184
268
|
|
269
|
+
private def read(pg_connection, channels)
|
270
|
+
PgVersions.read(channels, connection: pg_connection)
|
271
|
+
end
|
185
272
|
|
186
|
-
def
|
187
|
-
|
188
|
-
@actor_commands << proc { |pg_connection, subscribers|
|
189
|
-
done << block.call(pg_connection, subscribers)
|
190
|
-
}
|
191
|
-
@actor_notify_w.write('!')
|
192
|
-
done.shift
|
273
|
+
private def bump(pg_connection, channels)
|
274
|
+
PgVersions.bump(channels, connection: pg_connection)
|
193
275
|
end
|
194
276
|
|
277
|
+
def request(function, *params)
|
278
|
+
request_nonblock(function, *params).pop
|
279
|
+
end
|
195
280
|
|
196
|
-
def
|
197
|
-
|
198
|
-
|
281
|
+
def request_nonblock(function, *params)
|
282
|
+
retpipe = Queue.new
|
283
|
+
@thread_requests_mutex.synchronize {
|
284
|
+
@thread_requests.push [function, retpipe, params]
|
199
285
|
}
|
286
|
+
@thread_requests_notify_w.write('!')
|
287
|
+
retpipe
|
288
|
+
end
|
289
|
+
|
290
|
+
end
|
291
|
+
|
292
|
+
class Connection
|
293
|
+
|
294
|
+
def initialize(connection=nil)
|
295
|
+
@connection_thread = ConnectionThread.new(connection)
|
296
|
+
end
|
297
|
+
|
298
|
+
def close()
|
299
|
+
@connection_thread.request(:stop)
|
300
|
+
end
|
301
|
+
|
302
|
+
def bump(*channels)
|
303
|
+
@connection_thread.request(:bump, channels)
|
200
304
|
end
|
201
305
|
|
202
306
|
|
203
307
|
def read(*channels)
|
204
|
-
|
205
|
-
PgVersions.read(channels, connection: pg_connection)
|
206
|
-
}
|
308
|
+
@connection_thread.request(:read, channels)
|
207
309
|
end
|
208
310
|
|
209
311
|
|
210
|
-
def subscribe(*channels, known: {})
|
211
|
-
subscription = Subscription.new(
|
312
|
+
def subscribe(*channels, known: {}, batch_delay: 0.01)
|
313
|
+
subscription = Subscription.new(@connection_thread, batch_delay)
|
212
314
|
subscription.subscribe([channels].flatten, known: known)
|
213
|
-
|
315
|
+
if block_given?
|
316
|
+
Thread.handle_interrupt(Object => :never) {
|
317
|
+
begin
|
318
|
+
Thread.handle_interrupt(Object => :immediate) {
|
319
|
+
yield subscription
|
320
|
+
}
|
321
|
+
ensure
|
322
|
+
subscription.drop
|
323
|
+
end
|
324
|
+
}
|
325
|
+
else
|
326
|
+
subscription
|
327
|
+
end
|
214
328
|
end
|
215
329
|
|
216
330
|
|
217
331
|
class Subscription
|
218
332
|
|
219
|
-
def initialize(
|
220
|
-
@
|
333
|
+
def initialize(connection_thread, batch_delay)
|
334
|
+
@connection_thread = connection_thread
|
335
|
+
@batch_delay = batch_delay
|
221
336
|
@notifications = Queue.new
|
222
337
|
@already_known_versions = Hash.new { |h,k| h[k] = [] }
|
223
338
|
@channels = Hash.new(0)
|
@@ -231,15 +346,7 @@ module PgVersions
|
|
231
346
|
(@channels[channel] += 1) == 1
|
232
347
|
}
|
233
348
|
if channels.size > 0
|
234
|
-
@
|
235
|
-
channels.each { |channel|
|
236
|
-
subscribers[channel] << self
|
237
|
-
pg_connection.exec("LISTEN #{PG::Connection.quote_ident(channel)}") if subscribers[channel].size == 1
|
238
|
-
}
|
239
|
-
PgVersions.read(channels, connection: pg_connection).each_pair { |channel, version|
|
240
|
-
notify(channel, version)
|
241
|
-
}
|
242
|
-
}
|
349
|
+
@connection_thread.request(:subscribe, self, channels)
|
243
350
|
end
|
244
351
|
end
|
245
352
|
|
@@ -252,59 +359,68 @@ module PgVersions
|
|
252
359
|
@channels.delete(channel) if @channels[channel] == 0
|
253
360
|
not @channels.has_key?(channel)
|
254
361
|
}
|
255
|
-
@
|
256
|
-
channels.each { |channel|
|
257
|
-
subscribers[channel].delete(self)
|
258
|
-
if subscribers[channel].size == 0
|
259
|
-
pg_connection.exec("UNLISTEN #{PG::Connection.quote_ident(channel)}")
|
260
|
-
subscribers.delete(channel)
|
261
|
-
end
|
262
|
-
}
|
263
|
-
}
|
362
|
+
@connection_thread.request(:unsubscribe, self, channels)
|
264
363
|
end
|
265
364
|
|
266
365
|
|
267
|
-
def read(*channels, notify:
|
366
|
+
def read(*channels, notify: false)
|
268
367
|
channels = @channels.keys if channels.size == 0
|
269
|
-
versions = @
|
270
|
-
PgVersions.read(channels, connection: pg_connection)
|
271
|
-
}
|
368
|
+
versions = @connection_thread.request(:read, channels)
|
272
369
|
update_already_known_versions(versions) if not notify
|
273
370
|
versions
|
274
371
|
end
|
275
372
|
|
276
373
|
|
277
|
-
def bump(*channels, notify:
|
374
|
+
def bump(*channels, notify: false)
|
278
375
|
channels = @channels.keys if channels.size == 0
|
279
|
-
versions = @
|
280
|
-
PgVersions.bump(channels, connection: pg_connection)
|
281
|
-
}
|
376
|
+
versions = @connection_thread.request(:bump, channels)
|
282
377
|
update_already_known_versions(versions) if not notify
|
283
378
|
versions
|
284
379
|
end
|
285
380
|
|
286
381
|
|
287
|
-
|
382
|
+
#TODO: make this resume-able after forced exception
|
383
|
+
def wait(new_already_known_versions = {}, batch_delay: nil)
|
384
|
+
batch_delay = @batch_delay if batch_delay.nil?
|
288
385
|
update_already_known_versions(new_already_known_versions)
|
289
386
|
loop {
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
387
|
+
events = [@notifications.shift]
|
388
|
+
sleep batch_delay if batch_delay
|
389
|
+
events << @notifications.shift while not @notifications.empty?
|
390
|
+
changed_versions = {}
|
391
|
+
events.each { |versions|
|
392
|
+
return nil if not versions #termination
|
393
|
+
versions.each { |channel, version|
|
394
|
+
if (@already_known_versions[channel] <=> version) == -1
|
395
|
+
@already_known_versions[channel] = version
|
396
|
+
changed_versions[channel] = version
|
397
|
+
end
|
398
|
+
}
|
399
|
+
}
|
400
|
+
if changed_versions.size > 0
|
401
|
+
return Notification.new(changed_versions, @already_known_versions.dup)
|
295
402
|
end
|
296
403
|
}
|
297
404
|
end
|
298
405
|
|
299
406
|
|
300
|
-
def
|
301
|
-
|
407
|
+
def each(new_already_known_versions = {}, batch_delay: nil)
|
408
|
+
update_already_known_versions(new_already_known_versions)
|
409
|
+
while notification = wait(batch_delay: batch_delay)
|
410
|
+
yield notification
|
411
|
+
end
|
412
|
+
end
|
413
|
+
|
414
|
+
|
415
|
+
def notify(versions)
|
416
|
+
@notifications << versions
|
302
417
|
end
|
303
418
|
|
304
419
|
|
305
420
|
def drop
|
306
|
-
@notifications <<
|
307
|
-
unsubscribe
|
421
|
+
@notifications << nil
|
422
|
+
@connection_thread.request_nonblock(:unsubscribe, self, @channels.keys)
|
423
|
+
#TODO: what to do if this object gets used after drop?
|
308
424
|
end
|
309
425
|
|
310
426
|
|
data/lib/pg_versions/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pg_versions
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '1
|
4
|
+
version: '2.1'
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- yunta
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-01-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rspec
|
@@ -100,7 +100,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
100
100
|
- !ruby/object:Gem::Version
|
101
101
|
version: '0'
|
102
102
|
requirements: []
|
103
|
-
rubygems_version: 3.
|
103
|
+
rubygems_version: 3.3.26
|
104
104
|
signing_key:
|
105
105
|
specification_version: 4
|
106
106
|
summary: Persistent timestamped postgres notification library
|