wal 0.0.29 → 0.0.30

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7decfb78b8ccea001e781411dbcbbe939e9a7a30f29767a306e7c0c98107aba1
-  data.tar.gz: 976ec1815c1f379b966fcdd963e522644be2c7a5623939dc839e17673970a71f
+  metadata.gz: 629931aa6280e9b0f00fbc62ae9cc588771be9631fe2ee4ab5b346f884d52b27
+  data.tar.gz: 9144033bc75fceec8b94f33bfb67b3577bc0a1c9731353c56b6c499afd213b33
 SHA512:
-  metadata.gz: 1b8b9af4f0936ee756aeb492b3de17b68e17472af6d17ce05574c3bda747c28f1e73910490994a02d6f0255e59f2187d1e5e41927ddbdff39e113cc866950086
-  data.tar.gz: b0e185bf497f7dbc188eab4ef6935277a16e7a521c06a001b81374d68761954198852a7af1c68f51db5b126699b7f97666be66d8d7ea0ab14c95fd32a8120b95
+  metadata.gz: 3eb7fcf6d3ceea6f5d72cbcc75ffd67c99fcd688a06cfb8a8e412a428e4c6c5ef7dc44178ab4219d94e4a6e15ea8b4d3c45f00bbf96c33dec6103f688295ca50
+  data.tar.gz: f5b8260db91d66be7cf77a391d9a98285bb5773638778afffb2aa02b9b89a306937310104fffed49d8a658f33f0ae818ea35adb138461a3e0634ce341e63af2c

data/exe/wal

@@ -28,47 +28,6 @@ require "./config/environment"
 
 db_config = ActiveRecord::Base.configurations.configs_for(name: "primary").configuration_hash
 
-class Wal::LoggingReplicator
-  def initialize(slot, replicator)
-    @slot = slot
-    @replicator = replicator
-  end
-
-  def replicate_forever(watcher, publications:)
-    replication = @replicator.replicate(watcher, publications:)
-    count = 0
-    start = Time.now
-    loop do
-      case (event = replication.next)
-      when Wal::BeginTransactionEvent
-        start = Time.now
-        count = 0
-        if event.estimated_size > 0
-          Wal.logger&.info("[#{@slot}] Begin transaction=#{event.transaction_id} size=#{event.estimated_size}")
-        end
-      when Wal::CommitTransactionEvent
-        if count > 0
-          elapsed = ((Time.now - start) * 1000.0).round(1)
-          Wal.logger&.info("[#{@slot}] Commit transaction=#{event.transaction_id} elapsed=#{elapsed} events=#{count}")
-        end
-      when Wal::InsertEvent
-        Wal.logger&.debug("[#{@slot}] Insert transaction=#{event.transaction_id} table=#{event.table} primary_key=#{event.primary_key}")
-        count += 1
-      when Wal::UpdateEvent
-        Wal.logger&.debug("[#{@slot}] Update transaction=#{event.transaction_id} table=#{event.table} primary_key=#{event.primary_key}")
-        count += 1
-      when Wal::DeleteEvent
-        Wal.logger&.debug("[#{@slot}] Delete transaction=#{event.transaction_id} table=#{event.table} primary_key=#{event.primary_key}")
-        count += 1
-      else
-        count += 1
-      end
-    end
-  rescue StopIteration
-    nil
-  end
-end
-
 def watch(db_config, cli)
   watcher = cli["--watcher"].constantize.new
   use_temporary_slot = cli["--tmp-slot"] || false
@@ -78,90 +37,15 @@ def watch(db_config, cli)
   replicator_class = cli["--replicator"].presence&.constantize || Wal::Replicator
 
   puts "Watcher started for #{replication_slot} slot (#{publications.join(", ")})"
-  replicator = replicator_class.new(replication_slot:, use_temporary_slot:, db_config:)
-  replicator = Wal::LoggingReplicator.new(replication_slot, replicator)
-  replicator.replicate_forever(watcher, publications:)
+  replicator_class
+    .new(replication_slot:, use_temporary_slot:, db_config:)
+    .replicate_forever(Wal::LoggingWatcher.new(replication_slot, watcher), publications:)
   puts "Watcher finished for #{replication_slot}"
 end
 
-def start_worker(db_config, slot, config)
-  watcher = config["watcher"].constantize.new
-  temporary = config["temporary"] || false
-  publications = config["publications"] || []
-  replicator_class = config["replicator"].presence&.constantize || Wal::Replicator
-  auto_restart = config["auto_restart"].nil? || config["auto_restart"]
-  max_retries = config["retries"]&.to_i || (2**32 - 1)
-  retries = 0
-  backoff = config["retry_backoff"]&.to_f || 1
-  backoff_expoent = config["retry_backoff_expoent"]&.to_f
-
-  Thread.new(slot, watcher, temporary, publications) do |replication_slot, watcher, use_temporary_slot, publications|
-    replication_slot = "#{replication_slot}_#{SecureRandom.alphanumeric(4)}" if use_temporary_slot
-    puts "Watcher started for #{replication_slot} slot (#{publications.join(", ")})"
-
-    begin
-      replicator = replicator_class.new(replication_slot:, use_temporary_slot:, db_config:)
-      replicator = Wal::LoggingReplicator.new(replication_slot, replicator)
-      replicator.replicate_forever(watcher, publications:)
-      if auto_restart
-        backoff_time = backoff_expoent ? (backoff * retries) ** backoff_expoent : backoff
-        puts "Watcher finished for #{replication_slot}, auto restarting in #{backoff_time.floor(2)}..."
-        sleep backoff_time
-        puts "Restarting #{replication_slot}"
-        redo
-      end
-    rescue StandardError => err
-      if retries < max_retries
-        Wal.logger&.error("[#{replication_slot}] Error #{err}")
-        retries += 1
-        backoff_time = backoff_expoent ? (backoff * retries) ** backoff_expoent : backoff
-        puts "Restarting #{replication_slot} in #{backoff_time.floor(2)}s..."
-        sleep backoff_time
-        puts "Restarting #{replication_slot}"
-        retry
-      end
-      raise
-    end
-
-    puts "Watcher finished for #{replication_slot}"
-
-    Process.kill("TERM", Process.pid)
-  end
-end
-
-def start(db_config, cli)
-  slots = YAML.load_file(cli["<config-file>"])["slots"]
-
-  workers = slots.map do |slot, config|
-    start_worker(db_config, slot, config)
-  end
-
-  ping = Thread.new do
-    loop do
-      ActiveRecord::Base.connection_pool.with_connection do |conn|
-        conn.execute("SELECT pg_logical_emit_message(true, 'wal_ping', '{}')")
-      end
-      sleep 20
-    end
-  end
-
-  workers << ping
-
-  stop_workers = proc do
-    puts "Stopping WAL workers..."
-    workers.each(&:kill)
-    puts "WAL workers stopped"
-    exit 0
-  end
-
-  Signal.trap("TERM", &stop_workers)
-  Signal.trap("INT", &stop_workers)
-
-  workers.each(&:join)
-end
-
 if cli["watch"]
   watch(db_config, cli)
 elsif cli["start"]
-  start(db_config, cli)
+  runner = Wal::Runner.new(config_file: cli["<config-file>"], db_config:)
+  runner.start
 end

data/lib/wal/logging_watcher.rb

@@ -0,0 +1,46 @@
+module Wal
+  class LoggingWatcher
+    include Wal::Watcher
+
+    def initialize(slot, watcher)
+      @slot = slot
+      @watcher = watcher
+    end
+
+    def should_watch_table?(table)
+      @watcher.should_watch_table? table
+    end
+
+    def valid_context_prefix?(prefix)
+      @watcher.valid_context_prefix? prefix
+    end
+
+    def on_event(event)
+      case event
+      when Wal::BeginTransactionEvent
+        @start = Time.now
+        @count = 0
+        if event.estimated_size > 0
+          Wal.logger&.debug("[#{@slot}] Begin transaction=#{event.transaction_id} size=#{event.estimated_size}")
+        end
+      when Wal::CommitTransactionEvent
+        if @count > 0
+          elapsed = ((Time.now - @start) * 1000.0).round(1)
+          Wal.logger&.info("[#{@slot}] Commit transaction=#{event.transaction_id} elapsed=#{elapsed} events=#{@count}")
+        end
+      when Wal::InsertEvent
+        Wal.logger&.debug("[#{@slot}] Insert transaction=#{event.transaction_id} table=#{event.table} primary_key=#{event.primary_key}")
+        @count += 1
+      when Wal::UpdateEvent
+        Wal.logger&.debug("[#{@slot}] Update transaction=#{event.transaction_id} table=#{event.table} primary_key=#{event.primary_key}")
+        @count += 1
+      when Wal::DeleteEvent
+        Wal.logger&.debug("[#{@slot}] Delete transaction=#{event.transaction_id} table=#{event.table} primary_key=#{event.primary_key}")
+        @count += 1
+      else
+        @count += 1
+      end
+      @watcher.on_event(event)
+    end
+  end
+end

data/lib/wal/runner.rb

@@ -0,0 +1,164 @@
+module Wal
+  class Runner
+    class Worker
+      attr_reader :name, :slot_configs, :db_config
+
+      def initialize(name:, slot_configs:, db_config:)
+        @name = name
+        @slot_configs = slot_configs
+        @db_config = db_config
+        @threads = []
+      end
+
+      def run
+        @threads = slot_configs.map { |slot, config| start_worker(slot, config) }
+        setup_signal_handlers
+        @threads.each(&:join)
+      end
+
+      private
+
+      def start_worker(slot, config)
+        watcher = config["watcher"].constantize.new
+        temporary = config["temporary"] || false
+        publications = config["publications"] || []
+        replicator_class = config["replicator"].presence&.constantize || Wal::Replicator
+        auto_restart = config["auto_restart"].nil? || config["auto_restart"]
+        max_retries = config["retries"]&.to_i || (2**32 - 1)
+        backoff = config["retry_backoff"]&.to_f || 1
+        backoff_exponent = config["retry_backoff_expoent"]&.to_f
+
+        Thread.new(slot, watcher, temporary, publications) do |replication_slot, watcher, use_temporary_slot, publications|
+          retries = 0
+          replication_slot = "#{replication_slot}_#{SecureRandom.alphanumeric(4)}" if use_temporary_slot
+          puts "Watcher started for #{replication_slot} slot (#{publications.join(", ")})"
+
+          begin
+            replicator_class
+              .new(replication_slot:, use_temporary_slot:, db_config:)
+              .replicate_forever(Wal::LoggingWatcher.new(replication_slot, watcher), publications:)
+            if auto_restart
+              backoff_time = backoff_exponent ? (backoff * retries) ** backoff_exponent : backoff
+              puts "Watcher finished for #{replication_slot}, auto restarting in #{backoff_time.floor(2)}..."
+              sleep backoff_time
+              puts "Restarting #{replication_slot}"
+              redo
+            end
+          rescue StandardError => err
+            if retries < max_retries
+              Wal.logger&.error("[#{replication_slot}] Error #{err}")
+              Wal.logger&.error([err.message, *err.backtrace].join("\n"))
+              retries += 1
+              backoff_time = backoff_exponent ? (backoff * retries) ** backoff_exponent : backoff
+              puts "Restarting #{replication_slot} in #{backoff_time.floor(2)}s..."
+              sleep backoff_time
+              puts "Restarting #{replication_slot}"
+              retry
+            end
+            raise
+          end
+
+          puts "Watcher finished for #{replication_slot}"
+          Process.kill("TERM", Process.pid)
+        end
+      end
+
+      def setup_signal_handlers
+        stop_threads = proc do
+          puts "[#{name}] Stopping WAL threads..."
+          @threads.each(&:kill)
+          puts "[#{name}] WAL threads stopped"
+          exit 0
+        end
+
+        Signal.trap("TERM", &stop_threads)
+        Signal.trap("INT", &stop_threads)
+      end
+    end
+
+    attr_reader :config_file, :db_config
+
+    def initialize(config_file:, db_config:)
+      @config_file = config_file
+      @db_config = db_config
+      @child_pids = []
+    end
+
+    def start
+      slots = YAML.load_file(config_file)["slots"]
+      workers_slots = slots.group_by { |_slot, config| config["worker"] || "default" }
+
+      Wal.fork_hooks[:before_fork]&.call
+
+      workers_slots.each do |worker_name, slot_configs|
+        pid = fork_worker(worker_name, slot_configs)
+        @child_pids << pid
+        puts "Spawned worker '#{worker_name}' with PID #{pid}"
+      end
+
+      @ping_thread = start_ping_thread
+      setup_signal_handlers
+      wait_for_workers
+    end
+
+    private
+
+    def fork_worker(worker_name, slot_configs)
+      Process.fork do
+        Wal.fork_hooks[:after_fork]&.call
+        puts "[#{worker_name}] Starting worker process (PID: #{Process.pid})"
+        worker = Worker.new(name: worker_name, slot_configs: slot_configs, db_config: db_config)
+        worker.run
+      end
+    end
+
+    def start_ping_thread
+      Thread.new do
+        loop do
+          ActiveRecord::Base.connection_pool.with_connection do |conn|
+            conn.execute("SELECT pg_logical_emit_message(true, 'wal_ping', '{}')")
+          end
+          sleep 20
+        end
+      end
+    end
+
+    def setup_signal_handlers
+      stop_workers = proc do
+        puts "Stopping all worker processes..."
+        @ping_thread&.kill
+        @child_pids.each do |pid|
+          Process.kill("TERM", pid)
+        rescue Errno::ESRCH
+          # Process already dead
+        end
+        @child_pids.each do |pid|
+          Process.wait(pid)
+        rescue Errno::ECHILD
+          # Already reaped
+        end
+        puts "All worker processes stopped"
+        exit 0
+      end
+
+      Signal.trap("TERM", &stop_workers)
+      Signal.trap("INT", &stop_workers)
+
+      @stop_workers = stop_workers
+    end
+
+    def wait_for_workers
+      loop do
+        exited_pid = Process.wait(-1)
+        if @child_pids.include?(exited_pid)
+          @child_pids.delete(exited_pid)
+          puts "Worker process #{exited_pid} exited unexpectedly, shutting down..."
+          @stop_workers.call
+        end
+      rescue Errno::ECHILD
+        puts "All worker processes have exited"
+        break
+      end
+    end
+  end
+end

data/lib/wal/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Wal
-  VERSION = "0.0.29"
+  VERSION = "0.0.30"
 end

data/lib/wal.rb

@@ -4,20 +4,35 @@ require "active_support"
 require "active_record"
 require_relative "wal/watcher"
 require_relative "wal/noop_watcher"
+require_relative "wal/logging_watcher"
 require_relative "wal/record_watcher"
 require_relative "wal/streaming_watcher"
 require_relative "wal/replicator"
 require_relative "wal/active_record_context_extension"
 require_relative "wal/railtie"
+require_relative "wal/runner"
 require_relative "wal/version"
 
 module Wal
   class << self
     attr_accessor :logger
+    attr_accessor :fork_hooks
 
     def logger
       @logger ||= Logger.new($stdout, level: :info)
     end
+
+    def fork_hooks
+      @fork_hooks ||= {}
+    end
+
+    def before_fork(&block)
+      fork_hooks[:before_fork] = block
+    end
+
+    def after_fork(&block)
+      fork_hooks[:after_fork] = block
+    end
   end
 
   def self.configure(&block)

data/rbi/wal.rbi

@@ -7,11 +7,17 @@ module Wal
       UpdateEvent,
       DeleteEvent,
     ) }
-  VERSION = "0.0.29"
+  VERSION = "0.0.30"
 
   class << self
     sig { returns(T.class_of(Logger)) }
     attr_accessor :logger
+
+    sig { params(block: T.proc.void).void }
+    def before_fork(&block); end
+
+    sig { params(block: T.proc.void).void }
+    def after_fork(&block); end
   end
 
   sig { params(block: T.proc.params(config: T.class_of(Wal)).void).void }

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: wal
 version: !ruby/object:Gem::Version
-  version: 0.0.29
+  version: 0.0.30
 platform: ruby
 authors:
 - Rodrigo Navarro
 bindir: exe
 cert_chain: []
-date: 2026-01-20 00:00:00.000000000 Z
+date: 2026-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -95,11 +95,13 @@ files:
 - lib/wal/active_record_context_extension.rb
 - lib/wal/generators/migration.rb
 - lib/wal/generators/templates/migration.rb.erb
+- lib/wal/logging_watcher.rb
 - lib/wal/migration.rb
 - lib/wal/noop_watcher.rb
 - lib/wal/railtie.rb
 - lib/wal/record_watcher.rb
 - lib/wal/replicator.rb
+- lib/wal/runner.rb
 - lib/wal/streaming_watcher.rb
 - lib/wal/version.rb
 - lib/wal/watcher.rb