pg_versions 1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 272ebc261a3bbb1c96eb4545b2a215a55366b950ead6dd4b4547dbcf1ee6ea6a
+  data.tar.gz: 5ba78eabb8f3fca3b7d1302b3e032660c625c294ae6cef31eed252e801558acb
+SHA512:
+  metadata.gz: 2fb87931f3595b7938328461397901244ae2d357f80b93e5698bc813ba25e4ee542228afd68e17e2e3fc96e523ac0f86a22a6fc24e1bad81eb5557020b330716
+  data.tar.gz: 7ecac0582c70f42295e870c4dcb8df35f23b94e9f5e5461717f50e4a8d8b561cc60513bf67eb02f9918f2b5887b2d70b4f9a59226cdf6a99cb08ebb9831d5234

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2020-2021 yunta
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/create-table.sql

@@ -0,0 +1 @@
+CREATE UNLOGGED TABLE pg_versions (channel text NOT NULL PRIMARY KEY, instant timestamptz NOT NULL, counter smallint NOT NULL);

data/db/migrate/1_create_pg_versions_table.rb

@@ -0,0 +1,11 @@
+class CreatePgVersionsTable < ActiveRecord::Migration[7.0]
+
+        def up
+                PgVersions.create_table
+        end
+
+        def down
+                PgVersions.drop_table
+        end
+
+end

data/drop-table.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS pg_versions;

data/lib/pg_versions/pg_versions.rb

@@ -0,0 +1,321 @@
+# Operations on versions:
+# 1. bump - increase or assert version. ensures new version is unique and higher than any previous version, even if table entry existed  before and got removed
+# 2. read - always returns a version, even if table entry is missing
+# 3. remove (clean) - for periodic pruning of old entries
+#
+# A "version" (and database schema) contains both: timestamp and a counter (smallint).
+#
+# We need the timestamp because entries may vanish from the table (either the table is unlogged and a crash occured, or entries were removed for periodic cleanup).
+# If an entry has vanished, and bump or read wants to generate a new version, they have to ensure that the new version is higher than any version the key may have had before.
+# Using current timestamp of microsecond precision gives near-certainty of having unique first component of the version.
+# There remains a theoretical risk that a timestamp was placed in the table, then got deleted (periodic cleanup), and then bump or read generated identical timestamp.
+# All those actions would have to happen in the same microsecond, which, having accurate time source, is likely impossible with postgres (yet!).
+# To eliminate the risk, delete operation on the table should be followed by at least one microsecond of sleep (or longer if time source is of lower precission), while holding an exclusive lock on the table.
+# Bumps and reads always acquire shared lock on the table before reading the timestamp, to ensure that no delete operation gets executed between timestamp generation and version bump.
+# Also, if using unlogged table, administrator should ensure that new database instance will not restart, recover, and resume servicing requests faster than one microsecond after the crash XD
+#
+# Besides the timestamp, we also need the counter. Counter helps solving two scenarios:
+# 1) Hypothetical: Two transactions read out the timestamp at the same microsecond. Without the counter the versions asserted by both of those transactions would be identical.
+# 2) Practical: Timestamp read-out and row-update are separate operations. It's possible that entire other bump fits between them, leading to assertion of lower version by the transaction which started first and ended last.
+# With the counter, bump operation can detect that its timestamp is older than the one already present in the row, and bump the counter instead, leaving newer timestamp in place.
+# TODO: think if early row lock wouldn't be better
+#
+# Read of non-existent row should trigger a bump and reread. Otherwise following scenario may happen:
+# Empty table. Bump starts, gets timestamp, loses cpu. Reader reads, gets nothing, reports (current) timestamp higher than the one acquired by bump. Bump commits. Subsequent read reports older timestamp than previous read (loss of monotonicity).
+#
+
+
+require 'set'
+
+#TODO: prepared statements?
+
+module PgVersions
+
+
+	class ConnectionAcquisitionFailedError < StandardError; end
+
+	def self.timestamp_to_integers(input)
+		"to_char(%s, 'YYYYMMDD')::integer || ',' || to_char(%s, 'HH24MISS')::integer || ',' || to_char(%s, 'US')::integer"%[input, input, input]
+	end
+
+
+	def self.with_connection(pg_connection)
+		if pg_connection
+			yield pg_connection
+		elsif defined? ActiveRecord
+			ActiveRecord::Base.connection_pool.with_connection { |ar_connection|
+				yield ar_connection.instance_variable_get(:@connection)
+			}
+		else
+			raise ConnectionAcquisitionFailedError, "Missing connection. Either pass pg connection object or import ActiveRecord."
+		end
+	end	
+
+
+	def self.string_to_version(version_str) 
+		version_str.split(",").map { |str| Integer(str) }
+	end
+
+
+	def self.create_table(connection=nil)
+		PgVersions.with_connection(connection) { |pg_connection|
+			open(File.dirname(__FILE__)+"/../../create-table.sql") { |sql_file|
+				pg_connection.exec sql_file.read
+			}
+		}
+	end
+
+	def self.drop_table(connection=nil)
+		PgVersions.with_connection(connection) { |pg_connection|
+			open(File.dirname(__FILE__)+"/../../drop-table.sql") { |sql_file|
+				pg_connection.exec sql_file.read
+			}
+		}
+	end
+	
+	#TODO: ensure this is called only once per transaction, or that all bumps occur in the same order in all transactions, to avoid deadlocks
+	def self.bump(*channels, connection: nil)
+		PgVersions.with_connection(connection) { |pg_connection|
+			channels = [channels].flatten.sort
+			return {} if channels.size == 0
+			quoted_channels = channels.map.with_index { |channel, i| "(#{i},'#{pg_connection.escape_string(channel)}')" }.join(", ")
+			# table-wide share lock is there to mutually exclude table cleaner
+			# clock_timestamp() - this has to be a timestamp after table lock got acquired
+			pg_connection.exec("
+				LOCK TABLE pg_versions IN ACCESS SHARE MODE;
+				WITH 
+					to_bump(i, channel) AS (VALUES #{quoted_channels})
+					, current_instant(ts) AS (VALUES (clock_timestamp()))
+					, updated AS (
+							INSERT INTO pg_versions(channel, instant, counter)
+							SELECT to_bump.channel, (SELECT ts FROM current_instant), 0 FROM to_bump
+							ON CONFLICT (channel) DO UPDATE SET
+								instant = GREATEST(pg_versions.instant, EXCLUDED.instant),
+								counter = CASE WHEN pg_versions.instant < EXCLUDED.instant THEN 0 ELSE pg_versions.counter + 1 END
+							RETURNING channel, instant, pg_versions.counter
+							)
+				SELECT DISTINCT
+					i
+					, #{timestamp_to_integers('updated.instant')} || ',' || updated.counter::text AS version
+					, pg_notify(updated.channel::text, #{timestamp_to_integers('updated.instant')} || ',' || updated.counter::text)::text
+				FROM
+					to_bump
+					JOIN updated ON to_bump.channel = updated.channel;
+			").map { |row| [channels[Integer(row["i"])], string_to_version(row["version"])] }.to_h
+		}
+	end
+
+
+	def self.read(*channels, connection: nil)
+		PgVersions.with_connection(connection) { |pg_connection|
+			channels = [channels].flatten.sort
+			return {} if channels.size == 0
+			versions = {}
+			quoted_channels = channels.map.with_index { |channel, i| "(#{i},'#{pg_connection.escape_string(channel)}')" }.join(", ")
+			not_found_channels = pg_connection.exec("
+				LOCK TABLE pg_versions IN ACCESS SHARE MODE;
+				WITH
+					channels(i, channel) AS (VALUES #{quoted_channels})
+				SELECT
+					i
+					, #{timestamp_to_integers('instant')} || ',' || counter AS version
+				FROM
+					channels
+					JOIN pg_versions ON pg_versions.channel = channels.channel
+				ORDER BY
+					i DESC;
+			").each { |row|
+				versions[channels.delete_at(Integer(row["i"]))] = string_to_version(row["version"])
+			}
+			#TODO: bump in the same query instead of calling bump
+			versions.merge!(self.bump(channels, connection: pg_connection))  if channels.size > 0
+			versions
+		}
+	end
+
+
+	class Notification
+		attr_reader :channel, :version
+		def initialize(channel,version)
+			@channel, @version = channel, version
+		end
+	end
+
+
+	class Connection
+		
+		def initialize(connection=nil)
+			@actor_commands = Queue.new
+			actor_notify_r, @actor_notify_w = IO.pipe
+
+			connection_error_queue = Queue.new
+			@actor = Thread.new {
+				begin
+					PgVersions.with_connection(connection) { |pg_connection|
+						connection_error_queue << false
+						subscribers = Hash.new { |h,k| h[k] = Set.new }
+						loop {
+							#TODO: handle errors
+							reads,_writes,_errors = IO::select([pg_connection.socket_io, actor_notify_r])
+							
+							if reads.include?(pg_connection.socket_io)
+								pg_connection.consume_input
+							end
+
+							if reads.include?(actor_notify_r)
+								@actor_commands.shift.call(pg_connection, subscribers)
+								actor_notify_r.read(1)
+							end
+
+							while notification = pg_connection.notifies
+								channel, payload = notification[:relname], notification[:extra]
+								subscribers[channel].each { |subscriber|
+									subscriber.notify(channel, PgVersions.string_to_version(payload))
+								}
+							end
+						}
+					}
+				rescue ConnectionAcquisitionFailedError => e
+					connection_error_queue << e
+				end
+			}
+			(connection_error = connection_error_queue.shift) and raise connection_error
+		end
+
+
+		def actor_call(&block)
+			done = Queue.new
+			@actor_commands << proc { |pg_connection, subscribers|
+				done << block.call(pg_connection, subscribers)
+			} 
+			@actor_notify_w.write('!')
+			done.shift
+		end
+
+
+		def bump(*channels)
+			actor_call { |pg_connection, _subscribers|
+				PgVersions.bump(channels, connection: pg_connection)
+			}
+		end
+
+
+		def read(*channels)
+			actor_call { |pg_connection, _subscribers|
+				PgVersions.read(channels, connection: pg_connection)
+			}
+		end
+
+
+		def subscribe(*channels, known: {})
+			subscription = Subscription.new(self)
+			subscription.subscribe([channels].flatten, known: known)
+			subscription
+		end
+
+
+		class Subscription
+
+			def initialize(connection)
+				@connection = connection
+				@notifications = Queue.new
+				@already_known_versions = Hash.new { |h,k| h[k] = [] }
+				@channels = Hash.new(0)
+			end
+			
+
+			def subscribe(channels, known: {})
+				update_already_known_versions(known)
+				channels = [channels].flatten
+				channels.select! { |channel|
+					(@channels[channel] += 1) == 1
+				}
+				if channels.size > 0
+					@connection.actor_call { |pg_connection, subscribers|
+						channels.each { |channel|
+							subscribers[channel] << self
+							pg_connection.exec("LISTEN #{PG::Connection.quote_ident(channel)}")  if subscribers[channel].size == 1
+						}
+						PgVersions.read(channels, connection: pg_connection).each_pair { |channel, version|
+							notify(channel, version)
+						}
+					}
+				end
+			end
+			
+
+			def unsubscribe(*channels)
+				channels = [channels].flatten
+				channels.select! { |channel|
+					@channels[channel] -= 1
+					raise "Trying to unsubscribe from channel (%p) more times than it was subscribed to"%[channel]  if @channels[channel] < 0
+					@channels.delete(channel)  if @channels[channel] == 0
+					not @channels.has_key?(channel)
+				}
+				@connection.actor_call { |pg_connection, subscribers|
+					channels.each { |channel|
+						subscribers[channel].delete(self)
+						if subscribers[channel].size == 0
+							pg_connection.exec("UNLISTEN #{PG::Connection.quote_ident(channel)}")
+							subscribers.delete(channel)
+						end
+					}
+				}
+			end
+
+
+			def read(*channels, notify: true)
+				channels = @channels.keys  if channels.size == 0
+				versions = @connection.actor_call { |pg_connection, subscribers|
+					PgVersions.read(channels, connection: pg_connection)
+				}
+				update_already_known_versions(versions)  if not notify
+				versions
+			end
+
+
+			def bump(*channels, notify: true)
+				channels = @channels.keys  if channels.size == 0
+				versions = @connection.actor_call { |pg_connection, subscribers|
+					PgVersions.bump(channels, connection: pg_connection)
+				}
+				update_already_known_versions(versions)  if not notify
+				versions
+			end
+
+
+			def wait(new_already_known_versions = {})
+				update_already_known_versions(new_already_known_versions)
+				loop {
+					channel, version = @notifications.shift
+					return nil  if not channel #termination
+					if (@already_known_versions[channel] <=> version) == -1
+						@already_known_versions[channel] = version
+						return Notification.new(channel, version)
+					end
+				}
+			end
+
+
+			def notify(channel, payload)
+				@notifications << [channel, payload]
+			end
+
+
+			def drop
+				@notifications << [nil, nil]
+				unsubscribe(@channels.keys)
+			end
+
+
+			def update_already_known_versions(new_already_known_versions)
+				new_already_known_versions.each { |channel, version|
+					@already_known_versions[channel] = version  if (version <=> @already_known_versions[channel]) == 1
+				}
+			end
+
+		end
+
+	end
+
+end

data/lib/pg_versions/rails.rb

@@ -0,0 +1,7 @@
+if defined? ::Rails
+	module PgVersions
+		class Engine < ::Rails::Engine
+			isolate_namespace PgVersions
+		end
+	end
+end

data/lib/pg_versions/version.rb

@@ -0,0 +1,3 @@
+module PgVersions
+	VERSION = "1.0"
+end

data/lib/pg_versions.rb

@@ -0,0 +1,5 @@
+require "pg_versions/pg_versions"
+require "pg_versions/rails"
+
+module PgVersions
+end

metadata

@@ -0,0 +1,107 @@
+--- !ruby/object:Gem::Specification
+name: pg_versions
+version: !ruby/object:Gem::Version
+  version: '1.0'
+platform: ruby
+authors:
+- yunta
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2022-01-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.10'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.21'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.21'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+description: PostgreSQL notifications with persistence.
+email:
+- maciej.blomberg@mikoton.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- create-table.sql
+- db/migrate/1_create_pg_versions_table.rb
+- drop-table.sql
+- lib/pg_versions.rb
+- lib/pg_versions/pg_versions.rb
+- lib/pg_versions/rails.rb
+- lib/pg_versions/version.rb
+homepage: https://gitlab.com/yunta/pg-versions
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.22
+signing_key:
+specification_version: 4
+summary: Persistent timestamped postgres notification library
+test_files: []