pg_versions 1.0 → 2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 272ebc261a3bbb1c96eb4545b2a215a55366b950ead6dd4b4547dbcf1ee6ea6a
-  data.tar.gz: 5ba78eabb8f3fca3b7d1302b3e032660c625c294ae6cef31eed252e801558acb
+  metadata.gz: f6eab4d5b85196e9a17fe15efdb09cb6ef83d6821a060c40a5c68e3fae7c9fed
+  data.tar.gz: ff1c5c4157163144f7bb1808115c52e50438d8119415b7ea19df18e82e9d5bcd
 SHA512:
-  metadata.gz: 2fb87931f3595b7938328461397901244ae2d357f80b93e5698bc813ba25e4ee542228afd68e17e2e3fc96e523ac0f86a22a6fc24e1bad81eb5557020b330716
-  data.tar.gz: 7ecac0582c70f42295e870c4dcb8df35f23b94e9f5e5461717f50e4a8d8b561cc60513bf67eb02f9918f2b5887b2d70b4f9a59226cdf6a99cb08ebb9831d5234
+  metadata.gz: 12c82c86f3138b9b27eaade5a3eb97779b246de26a680de3e23e6d99daa576cc56a1b04ce2c8b82b81f72f8402a73f5eb2e5e7d4babb57651d458317f40eeea5
+  data.tar.gz: 70c41180db0eff19a524f1b65165800c08a50a22ddf25f07daaacb58a0c0b901e0bbf22b620f61499f52788ec4d9120de373b2cdbb3a1aaae3d02481dc211bd3

data/lib/pg_versions/pg_versions.rb

@@ -1,5 +1,5 @@
 # Operations on versions:
-# 1. bump - increase or assert version. ensures new version is unique and higher than any previous version, even if table entry existed  before and got removed
+# 1. bump - increase or assert version. ensures new version is unique and higher than any previous version, even if table entry existed before and got removed
 # 2. read - always returns a version, even if table entry is missing
 # 3. remove (clean) - for periodic pruning of old entries
 #
@@ -32,22 +32,28 @@ require 'set'
 module PgVersions
 
 
-	class ConnectionAcquisitionFailedError < StandardError; end
+	class InvalidParameters < StandardError; end
 
 	def self.timestamp_to_integers(input)
 		"to_char(%s, 'YYYYMMDD')::integer || ',' || to_char(%s, 'HH24MISS')::integer || ',' || to_char(%s, 'US')::integer"%[input, input, input]
 	end
 
 
-	def self.with_connection(pg_connection)
-		if pg_connection
-			yield pg_connection
-		elsif defined? ActiveRecord
+	def self.with_connection(pg_connection, reset: false, &block)
+		if pg_connection.kind_of? PG::Connection
+			if reset
+				pg_connection.sync_reset
+				pg_connection.exec("select;")
+			end
+			block.call(pg_connection)
+		elsif pg_connection.respond_to? :call
+			pg_connection.call(reset, &block)
+		elsif pg_connection.nil? and defined? ActiveRecord
 			ActiveRecord::Base.connection_pool.with_connection { |ar_connection|
-				yield ar_connection.instance_variable_get(:@connection)
+				block.call(ar_connection.instance_variable_get(:@connection))
 			}
 		else
-			raise ConnectionAcquisitionFailedError, "Missing connection. Either pass pg connection object or import ActiveRecord."
+			raise InvalidParameters, "Missing connection. Either pass pg connection object or import ActiveRecord."
 		end
 	end	
 
@@ -75,6 +81,7 @@ module PgVersions
 	
 	#TODO: ensure this is called only once per transaction, or that all bumps occur in the same order in all transactions, to avoid deadlocks
 	def self.bump(*channels, connection: nil)
+		#TODO: pg_connection.exec returned nil once during testing.
 		PgVersions.with_connection(connection) { |pg_connection|
 			channels = [channels].flatten.sort
 			return {} if channels.size == 0
@@ -101,7 +108,9 @@ module PgVersions
 				FROM
 					to_bump
 					JOIN updated ON to_bump.channel = updated.channel;
-			").map { |row| [channels[Integer(row["i"])], string_to_version(row["version"])] }.to_h
+			") { |result| 
+				result.map { |row| [channels[Integer(row["i"])], string_to_version(row["version"])] }.to_h
+			}
 		}
 	end
 
@@ -124,8 +133,10 @@ module PgVersions
 					JOIN pg_versions ON pg_versions.channel = channels.channel
 				ORDER BY
 					i DESC;
-			").each { |row|
-				versions[channels.delete_at(Integer(row["i"]))] = string_to_version(row["version"])
+			") { |result|
+				result.each { |row|
+					versions[channels.delete_at(Integer(row["i"]))] = string_to_version(row["version"])
+				}
 			}
 			#TODO: bump in the same query instead of calling bump
 			versions.merge!(self.bump(channels, connection: pg_connection))  if channels.size > 0
@@ -135,89 +146,193 @@ module PgVersions
 
 
 	class Notification
-		attr_reader :channel, :version
-		def initialize(channel,version)
-			@channel, @version = channel, version
+		attr_reader :changed_versions, :changed, :all_versions, :versions
+		def initialize(changed_versions, all_versions)
+			@changed_versions, @all_versions = changed_versions, all_versions
+			@changed, @versions = changed_versions, all_versions
 		end
 	end
 
 
-	class Connection
-		
-		def initialize(connection=nil)
-			@actor_commands = Queue.new
-			actor_notify_r, @actor_notify_w = IO.pipe
 
-			connection_error_queue = Queue.new
-			@actor = Thread.new {
+	class ConnectionThread
+
+		class Closing < StandardError
+			attr_reader :retpipe
+			def initialize(retpipe)
+				@retpipe = retpipe
+			end
+		end
+
+
+		attr_reader :status
+
+		def initialize(connection)
+			retry_on_exceptions = [ PG::ConnectionBad, PG::UnableToSend ]
+			retry_on_exceptions << ActiveRecord::ConnectionNotEstablished  if defined? ActiveRecord
+
+			@subscribers = Hash.new { |h,k| h[k] = Set.new }
+			@status = :disconnected
+
+			@thread_requests_mutex = Mutex.new
+			@thread_requests = []
+			thread_requests_notify_r, @thread_requests_notify_w = IO.pipe
+
+			@thread = Thread.new {
+				reset_connection = false
+				retry_delay = 0
 				begin
-					PgVersions.with_connection(connection) { |pg_connection|
-						connection_error_queue << false
-						subscribers = Hash.new { |h,k| h[k] = Set.new }
+					PgVersions.with_connection(connection, reset: reset_connection) do |pg_connection|
+						
+						@status = :connected
+						retry_delay = 0
+
+						@subscribers.each_key { |channel|
+							listen(pg_connection, channel)
+						}
+						read(pg_connection, @subscribers.keys).each_pair { |channel, version|
+							@subscribers[channel].each { |subscriber|
+								subscriber.notify({ channel => version })
+							}
+						}
+
 						loop {
+							@thread_requests_mutex.synchronize { 
+								@thread_requests.each { |function, retpipe, params|
+									raise Closing, retpipe  if function == :stop
+									retpipe << send(function, pg_connection, *params)
+								}
+								@thread_requests.clear
+							}
+
+							while notification = pg_connection.notifies
+								channel, payload = notification[:relname], notification[:extra]
+								@subscribers[channel].each { |subscriber|
+									subscriber.notify({ channel => PgVersions.string_to_version(payload) })
+								}
+							end
+
 							#TODO: handle errors
-							reads,_writes,_errors = IO::select([pg_connection.socket_io, actor_notify_r])
+							reads,_writes,_errors = IO::select([pg_connection.socket_io, thread_requests_notify_r])
 							
 							if reads.include?(pg_connection.socket_io)
 								pg_connection.consume_input
 							end
 
-							if reads.include?(actor_notify_r)
-								@actor_commands.shift.call(pg_connection, subscribers)
-								actor_notify_r.read(1)
-							end
-
-							while notification = pg_connection.notifies
-								channel, payload = notification[:relname], notification[:extra]
-								subscribers[channel].each { |subscriber|
-									subscriber.notify(channel, PgVersions.string_to_version(payload))
-								}
+							if reads.include?(thread_requests_notify_r)
+								thread_requests_notify_r.read(1)
 							end
 						}
-					}
-				rescue ConnectionAcquisitionFailedError => e
-					connection_error_queue << e
+					rescue Closing => e
+						e.retpipe << true
+					end
+				rescue *retry_on_exceptions => e
+					reset_connection = true
+					@status = :disconnected
+					$stderr.puts "Pg connection failed. Retrying in #{retry_delay/1000.0}s."
+					sleep retry_delay/1000.0
+					retry_delay = { 0=>100, 100=>1000, 1000=>2000, 2000=>2000 }[retry_delay]
+					retry
+				end
+			} 
+			@thread.abort_on_exception = true
+		end
+
+		private def listen(pg_connection, channel)
+			pg_connection.exec("LISTEN #{PG::Connection.quote_ident(channel)}")
+		end
+
+		private def unlisten(pg_connection, channel)
+			pg_connection.exec("UNLISTEN #{PG::Connection.quote_ident(channel)}")
+		end
+
+		private def subscribe(pg_connection, subscriber, channels)
+			channels.each { |channel|
+				@subscribers[channel] << subscriber
+				listen(pg_connection, channel)  if @subscribers[channel].size == 1
+			}
+			subscriber.notify(PgVersions.read(channels, connection: pg_connection))
+			true
+		end
+		
+		private def unsubscribe(pg_connection, subscriber, channels)
+			channels.each { |channel|
+				@subscribers[channel].delete(subscriber)
+				if @subscribers[channel].size == 0
+					unlisten(pg_connection, channel)
+					@subscribers.delete(channel)
 				end
 			}
-			(connection_error = connection_error_queue.shift) and raise connection_error
+			true
 		end
 
+		private def read(pg_connection, channels)
+			PgVersions.read(channels, connection: pg_connection)
+		end
 
-		def actor_call(&block)
-			done = Queue.new
-			@actor_commands << proc { |pg_connection, subscribers|
-				done << block.call(pg_connection, subscribers)
-			} 
-			@actor_notify_w.write('!')
-			done.shift
+		private def bump(pg_connection, channels)
+			PgVersions.bump(channels, connection: pg_connection)
 		end
 
+		def request(function, *params)
+			request_nonblock(function, *params).pop
+		end
 
-		def bump(*channels)
-			actor_call { |pg_connection, _subscribers|
-				PgVersions.bump(channels, connection: pg_connection)
+		def request_nonblock(function, *params)
+			retpipe = Queue.new
+			@thread_requests_mutex.synchronize {
+				@thread_requests.push [function, retpipe, params]
 			}
+			@thread_requests_notify_w.write('!')
+			retpipe
+		end
+		
+	end
+
+	class Connection
+		
+		def initialize(connection=nil)
+			@connection_thread = ConnectionThread.new(connection)
+		end
+
+		def close()
+			@connection_thread.request(:stop)
+		end
+
+		def bump(*channels)
+			@connection_thread.request(:bump, channels)
 		end
 
 
 		def read(*channels)
-			actor_call { |pg_connection, _subscribers|
-				PgVersions.read(channels, connection: pg_connection)
-			}
+			@connection_thread.request(:read, channels)
 		end
 
 
-		def subscribe(*channels, known: {})
-			subscription = Subscription.new(self)
+		def subscribe(*channels, known: {}, batch_delay: 0.01)
+			subscription = Subscription.new(@connection_thread, batch_delay)
 			subscription.subscribe([channels].flatten, known: known)
-			subscription
+			if block_given?
+				Thread.handle_interrupt(Object => :never) {
+					begin
+						Thread.handle_interrupt(Object => :immediate) {
+							yield subscription
+						}
+					ensure
+						subscription.drop
+					end
+				}
+			else
+				subscription
+			end
 		end
 
 
 		class Subscription
 
-			def initialize(connection)
-				@connection = connection
+			def initialize(connection_thread, batch_delay)
+				@connection_thread = connection_thread
+				@batch_delay = batch_delay
 				@notifications = Queue.new
 				@already_known_versions = Hash.new { |h,k| h[k] = [] }
 				@channels = Hash.new(0)
@@ -231,15 +346,7 @@ module PgVersions
 					(@channels[channel] += 1) == 1
 				}
 				if channels.size > 0
-					@connection.actor_call { |pg_connection, subscribers|
-						channels.each { |channel|
-							subscribers[channel] << self
-							pg_connection.exec("LISTEN #{PG::Connection.quote_ident(channel)}")  if subscribers[channel].size == 1
-						}
-						PgVersions.read(channels, connection: pg_connection).each_pair { |channel, version|
-							notify(channel, version)
-						}
-					}
+					@connection_thread.request(:subscribe, self, channels)
 				end
 			end
 			
@@ -252,59 +359,68 @@ module PgVersions
 					@channels.delete(channel)  if @channels[channel] == 0
 					not @channels.has_key?(channel)
 				}
-				@connection.actor_call { |pg_connection, subscribers|
-					channels.each { |channel|
-						subscribers[channel].delete(self)
-						if subscribers[channel].size == 0
-							pg_connection.exec("UNLISTEN #{PG::Connection.quote_ident(channel)}")
-							subscribers.delete(channel)
-						end
-					}
-				}
+				@connection_thread.request(:unsubscribe, self, channels)
 			end
 
 
-			def read(*channels, notify: true)
+			def read(*channels, notify: false)
 				channels = @channels.keys  if channels.size == 0
-				versions = @connection.actor_call { |pg_connection, subscribers|
-					PgVersions.read(channels, connection: pg_connection)
-				}
+				versions = @connection_thread.request(:read, channels)
 				update_already_known_versions(versions)  if not notify
 				versions
 			end
 
 
-			def bump(*channels, notify: true)
+			def bump(*channels, notify: false)
 				channels = @channels.keys  if channels.size == 0
-				versions = @connection.actor_call { |pg_connection, subscribers|
-					PgVersions.bump(channels, connection: pg_connection)
-				}
+				versions = @connection_thread.request(:bump, channels)
 				update_already_known_versions(versions)  if not notify
 				versions
 			end
 
 
-			def wait(new_already_known_versions = {})
+			#TODO: make this resume-able after forced exception
+			def wait(new_already_known_versions = {}, batch_delay: nil)
+				batch_delay = @batch_delay  if batch_delay.nil?
 				update_already_known_versions(new_already_known_versions)
 				loop {
-					channel, version = @notifications.shift
-					return nil  if not channel #termination
-					if (@already_known_versions[channel] <=> version) == -1
-						@already_known_versions[channel] = version
-						return Notification.new(channel, version)
+					events = [@notifications.shift]
+					sleep batch_delay  if batch_delay
+					events << @notifications.shift  while not @notifications.empty?
+					changed_versions = {}
+					events.each { |versions|
+						return nil  if not versions #termination
+						versions.each { |channel, version|
+							if (@already_known_versions[channel] <=> version) == -1
+								@already_known_versions[channel] = version
+								changed_versions[channel] = version
+							end
+						}
+					}
+					if changed_versions.size > 0
+						return Notification.new(changed_versions, @already_known_versions.dup)
 					end
 				}
 			end
 
 
-			def notify(channel, payload)
-				@notifications << [channel, payload]
+			def each(new_already_known_versions = {}, batch_delay: nil)
+				update_already_known_versions(new_already_known_versions)
+				while notification = wait(batch_delay: batch_delay)
+					yield notification
+				end
+			end
+
+
+			def notify(versions)
+				@notifications << versions
 			end
 
 
 			def drop
-				@notifications << [nil, nil]
-				unsubscribe(@channels.keys)
+				@notifications << nil
+				@connection_thread.request_nonblock(:unsubscribe, self, @channels.keys)
+				#TODO: what to do if this object gets used after drop?
 			end
 
 

data/lib/pg_versions/version.rb

@@ -1,3 +1,3 @@
 module PgVersions
-	VERSION = "1.0"
+	VERSION = "2.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_versions
 version: !ruby/object:Gem::Version
-  version: '1.0'
+  version: '2.1'
 platform: ruby
 authors:
 - yunta
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-16 00:00:00.000000000 Z
+date: 2023-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -100,7 +100,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.3.26
 signing_key:
 specification_version: 4
 summary: Persistent timestamped postgres notification library