honker 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 26a3d929e2e2266567868784cc70e65fe06a60b8e93956f9d65721f39b4ed5c0
-  data.tar.gz: 41d9411865cb1d0774d3fd3f4147606b508980ac66f39f7cda339a71e6ca9981
+  metadata.gz: 4186dca28701dc5b7f2654da008e49ba84c51218a3d7141d6ff5c5d842ef3875
+  data.tar.gz: 12e5f86f35bc2832405fa98a7c60ca0667b9577ab2041ce571893b7596f46613
 SHA512:
-  metadata.gz: c20a4106adbe01b12b380991cf3dea7ffb6fdf175d1f0aa430ef0cf7abd029d755535109c67bf69315cc7060d0e335dee45534fa4ea3858ae27daebabda8a10f
-  data.tar.gz: ec076836d653243e5a50204943263f0dfecb8e664704b95f28c3edc14644ba6a7051d8cc73a163c90486f5a9e778e7dc002b0528858369b018dc979a7f78b915
+  metadata.gz: c0efdcaf60dac27bc103ccc09bf367e16a8b55b41001355ac230fdab14361d2d1a8c31a768bbba53a3065bcee1280760ea5cf30d95d72428af709ab5e554b3ad
+  data.tar.gz: 245e0ed000e237c10670e4822252a004bf3cf1897476696cbeee97a9bbd3eddf0c4ebc494fd172f24b99f9a86d52307fb121380e15fff35ae17e790d9ddf151a

data/README.md

@@ -230,6 +230,19 @@ if (job = q.claim_one("worker-1"))
 end
 ```
 
+Live pub/sub listeners are channel-filtered and skip notifications that
+already existed when they attached:
+
+```ruby
+db.listen("orders") do |notification|
+  puts notification.payload
+end
+```
+
+Without a block, `listen` returns an enumerable listener. Use
+`listener.next(timeout_s: 5)` for a bounded wait and call `listener.close`
+when finished. Closing the database also closes every listener created from it.
+
 Delayed jobs use `run_at:`:
 
 ```ruby

data/ext/honker/honker-core/src/lib.rs

@@ -3441,9 +3441,14 @@ while True:
         let _ = std::fs::remove_file(format!("{}-wal", tmp.display()));
         let _ = std::fs::remove_file(format!("{}-shm", tmp.display()));
 
+        // 300 ms = 6x RX_POLL_MS. The regression this guards is recv_timeout
+        // blocking on the safety-net interval, which lands at 500 ms or more,
+        // so this still catches it with room to spare. The old 150 ms bound
+        // was below CI scheduler jitter, not below the bug: it failed a
+        // Windows runner at 155 ms while the real failure is 3x further out.
         assert!(
-            elapsed < Duration::from_millis(150),
-            "kernel watcher shutdown took {elapsed:?}, expected < 150 ms \
+            elapsed < Duration::from_millis(300),
+            "kernel watcher shutdown took {elapsed:?}, expected < 300 ms \
              (RX_POLL_MS = 50 ms; if this exceeds 500 ms the recv_timeout \
              is blocking on the safety-net interval again)"
         );

data/lib/honker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Honker
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

data/lib/honker.rb

@@ -159,6 +159,295 @@ module Honker
     end
   end
 
+  # Fans one native watcher out to any number of Ruby waiters. The native
+  # watcher owns a single receiver, so callers must not race each other on
+  # CoreWatcher#wait directly.
+  class UpdateHub
+    WAIT_SLICE_S = 0.1
+
+    def initialize(watcher)
+      @watcher = watcher
+      @mutex = Mutex.new
+      @changed = ConditionVariable.new
+      @generation = 0
+      @seen_by_thread = {}
+      @stopping = false
+      @closed = false
+      @disposed = false
+      @error = nil
+      @thread = Thread.new { run }
+      @thread.name = "honker-update-hub" if @thread.respond_to?(:name=)
+    end
+
+    def snapshot
+      @mutex.synchronize { @generation }
+    end
+
+    def closed?
+      @mutex.synchronize { @closed }
+    end
+
+    def signal
+      @mutex.synchronize do
+        return if @closed
+
+        @generation += 1
+        @changed.broadcast
+      end
+    end
+
+    # Wake blocked waiters without claiming that SQLite changed. Listener
+    # cancellation uses this path so closing one listener cannot create a
+    # spurious Database#wait_for_update result for unrelated callers.
+    def wake
+      @mutex.synchronize { @changed.broadcast unless @closed }
+    end
+
+    def wait_after(generation, timeout_s = nil, &cancelled)
+      deadline = timeout_s.nil? ? nil : monotonic_now + [timeout_s.to_f, 0.0].max
+      @mutex.synchronize do
+        loop do
+          return false if cancelled&.call
+          raise @error if @error
+          return false if @closed
+          return true if @generation > generation
+
+          if deadline
+            remaining = deadline - monotonic_now
+            return false if remaining <= 0
+
+            @changed.wait(@mutex, remaining)
+          else
+            @changed.wait(@mutex)
+          end
+        end
+      end
+    end
+
+    def wait(timeout_s)
+      deadline = monotonic_now + [timeout_s.to_f, 0.0].max
+      key = Thread.current
+      @mutex.synchronize do
+        @seen_by_thread.delete_if { |thread, _generation| !thread.alive? }
+        loop do
+          raise @error if @error
+          return false if @closed
+          if @generation > @seen_by_thread.fetch(key, 0)
+            @seen_by_thread[key] = @generation
+            return true
+          end
+
+          remaining = deadline - monotonic_now
+          return false if remaining <= 0
+
+          @changed.wait(@mutex, remaining)
+        end
+      end
+    end
+
+    def close
+      @mutex.synchronize do
+        return if @disposed
+
+        if @stopping
+          @changed.wait(@mutex) until @disposed
+          return
+        end
+
+        @stopping = true
+      end
+      begin
+        @thread.join
+        @watcher.close
+      ensure
+        @mutex.synchronize do
+          @disposed = true
+          @closed = true
+          @changed.broadcast
+        end
+      end
+    end
+
+    private
+
+    def run
+      loop do
+        break if @mutex.synchronize { @stopping }
+
+        signal if @watcher.wait(WAIT_SLICE_S)
+      end
+    rescue StandardError => e
+      @mutex.synchronize do
+        @error = e
+        @closed = true
+        @changed.broadcast
+      end
+    ensure
+      @mutex.synchronize do
+        @closed = true unless @stopping
+        @changed.broadcast
+      end
+    end
+
+    def monotonic_now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+
+  # A live pub/sub notification. Payloads produced by Database#notify are
+  # JSON-decoded; raw non-JSON SQL payloads are returned unchanged.
+  Notification = Struct.new(:id, :channel, :payload, :created_at)
+
+  class Listener
+    include Enumerable
+
+    def initialize(db, channel, fallback_poll_s: 15.0)
+      raise ArgumentError, "channel must not be empty" if channel.to_s.empty?
+      if !fallback_poll_s.nil? && !fallback_poll_s.to_f.positive?
+        raise ArgumentError, "fallback_poll_s must be positive or nil"
+      end
+
+      @db = db
+      @channel = channel.to_s
+      @fallback_poll_s = fallback_poll_s&.to_f
+      @buffer = []
+      @state_mutex = Mutex.new
+      @state_changed = ConditionVariable.new
+      @active_calls = 0
+      @active_threads = Hash.new(0)
+      @closed = false
+      @read_db = SQLite3::Database.new(@db.path)
+      @read_db.busy_timeout = 5000
+      @read_db.execute("PRAGMA query_only = ON")
+      @last_seen = @read_db.get_first_value(
+        "SELECT COALESCE(MAX(id), 0) FROM _honker_notifications WHERE channel = ?",
+        [@channel],
+      ).to_i
+    rescue StandardError
+      @read_db&.close
+      raise
+    end
+
+    attr_reader :channel
+
+    def next(timeout_s: nil)
+      read_db = begin_call
+      return nil unless read_db
+
+      deadline = timeout_s.nil? ? nil : monotonic_now + [timeout_s.to_f, 0.0].max
+      loop do
+        return nil if closed?
+        return @buffer.shift unless @buffer.empty?
+
+        generation = @db.update_snapshot
+        refill(read_db)
+        next unless @buffer.empty?
+
+        remaining = deadline && deadline - monotonic_now
+        return nil if remaining && remaining <= 0
+
+        wait_s = [remaining, @fallback_poll_s].compact.min
+        @db.wait_for_update_after(generation, wait_s) { closed? }
+        if @db.updates_closed?
+          close
+          return nil
+        end
+      end
+    rescue StandardError
+      close
+      raise
+    ensure
+      end_call if read_db
+    end
+
+    def each
+      return enum_for(:each) unless block_given?
+
+      while (notification = self.next)
+        yield notification
+      end
+    end
+
+    def close
+      should_wake = @state_mutex.synchronize do
+        next false if @closed
+
+        @closed = true
+        close_read_db_if_idle
+        true
+      end
+
+      if should_wake
+        @db.wake_update_waiters
+        @db.unregister_listener(self)
+      end
+
+      @state_mutex.synchronize do
+        unless @active_threads.key?(Thread.current)
+          @state_changed.wait(@state_mutex) until @active_calls.zero?
+          close_read_db_if_idle
+        end
+      end
+      nil
+    end
+
+    def closed?
+      @state_mutex.synchronize { @closed }
+    end
+
+    private
+
+    def refill(read_db)
+      rows = read_db.execute(
+        "SELECT id, channel, payload, created_at " \
+        "FROM _honker_notifications " \
+        "WHERE channel = ? AND id > ? ORDER BY id LIMIT 1000",
+        [@channel, @last_seen],
+      )
+      rows.each do |id, channel, payload, created_at|
+        @last_seen = id.to_i
+        @buffer << Notification.new(id.to_i, channel, decode_payload(payload), created_at.to_i)
+      end
+    end
+
+    def begin_call
+      @state_mutex.synchronize do
+        return nil if @closed
+
+        @active_calls += 1
+        @active_threads[Thread.current] += 1
+        @read_db
+      end
+    end
+
+    def end_call
+      @state_mutex.synchronize do
+        @active_calls -= 1
+        @active_threads[Thread.current] -= 1
+        @active_threads.delete(Thread.current) if @active_threads[Thread.current].zero?
+        close_read_db_if_idle
+        @state_changed.broadcast if @active_calls.zero?
+      end
+    end
+
+    def close_read_db_if_idle
+      return unless @closed && @active_calls.zero? && @read_db
+
+      @read_db.close
+      @read_db = nil
+    end
+
+    def decode_payload(payload)
+      JSON.parse(payload)
+    rescue JSON::ParserError, TypeError
+      payload
+    end
+
+    def monotonic_now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+
   DEFAULT_PRAGMAS = <<~SQL
     PRAGMA journal_mode = WAL;
     PRAGMA synchronous = NORMAL;
@@ -174,7 +463,7 @@ module Honker
   # extension loaded. The constructor bootstraps the schema; safe to
   # open the same path from multiple processes.
   class Database
-    attr_reader :db
+    attr_reader :db, :path
 
     def initialize(path, extension_path: nil, watcher_backend: nil,
                    watcher_poll_interval_ms: nil,
@@ -187,8 +476,8 @@ module Honker
       end
 
       resolved_extension = extension_resolver.resolve(extension_path)
+      @path = path
       @db = SQLite3::Database.new(path)
-      @local_update_seq = 0
       @db.busy_timeout = 5000
       @db.execute("PRAGMA mmap_size = 0")
       @db.enable_load_extension(true)
@@ -196,24 +485,86 @@ module Honker
       @db.enable_load_extension(false)
       @db.execute_batch(DEFAULT_PRAGMAS)
       @db.execute("SELECT honker_bootstrap()")
-      @watcher = CoreWatcher.new(path, resolved_extension, watcher_backend, watcher_poll_interval_ms)
+      watcher = CoreWatcher.new(path, resolved_extension, watcher_backend, watcher_poll_interval_ms)
+      @updates = UpdateHub.new(watcher)
+      @listeners_mutex = Mutex.new
+      @listeners = {}
+      @closed = false
     end
 
     def close
-      @watcher&.close
+      listeners = @listeners_mutex.synchronize do
+        return if @closed
+
+        @closed = true
+        registered = @listeners.keys
+        @listeners.clear
+        registered
+      end
+      listeners.each(&:close)
+      @updates&.close
       @db&.close
     end
 
     def mark_updated
-      @local_update_seq += 1
+      @updates.signal
     end
 
     def update_snapshot
-      @local_update_seq
+      @updates.snapshot
     end
 
     def wait_for_update(timeout_s)
-      @watcher.wait(timeout_s)
+      @updates.wait(timeout_s)
+    end
+
+    def wait_for_update_after(generation, timeout_s, &cancelled)
+      @updates.wait_after(generation, timeout_s, &cancelled)
+    end
+
+    def wake_update_waiters
+      @updates.wake
+    end
+
+    def updates_closed?
+      @updates.closed?
+    end
+
+    def closed?
+      @listeners_mutex.synchronize { @closed }
+    end
+
+    def listen(channel, fallback_poll_s: 15.0)
+      raise Error, "database is closed" if closed?
+
+      listener = Listener.new(self, channel, fallback_poll_s: fallback_poll_s)
+      unless register_listener(listener)
+        listener.close
+        raise Error, "database is closed"
+      end
+      return listener unless block_given?
+
+      begin
+        listener.each { |notification| yield notification }
+      ensure
+        listener.close
+      end
+    end
+
+    # Internal listener lifecycle hooks. They are public only because the
+    # Listener is a separate object rather than a nested implementation detail.
+    def register_listener(listener)
+      @listeners_mutex.synchronize do
+        return false if @closed
+
+        @listeners[listener] = true
+        true
+      end
+    end
+
+    def unregister_listener(listener)
+      @listeners_mutex.synchronize { @listeners.delete(listener) }
+      nil
     end
 
     # Returns a Queue handle for a named queue.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: honker
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Russell Romney