wal 0.0.32 → 0.0.34

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e8f18e327f41668911967dbb73d0f6c51ab43f2b5d1d25b7c39b5f5130a04081
-  data.tar.gz: d0ffc96073a5faa4fd3949a02348b029bd206196fdd61c71883e5115ee53d6fa
+  metadata.gz: f7a8f82addc9ffce03d2f1ee6a77bac1f3ae2074a5b427a8c60ca087df2aded6
+  data.tar.gz: 75559dccba1edf1d0f1a60ebf866726705fe48166d697a1859ec6da644e963f3
 SHA512:
-  metadata.gz: effddb80b5b8093e572e65a587f5b7975cb20750aa39e7c329cb7358bd099d3bd12110648ee5b11c4dd45d791c1d050e6b0e7e82c7585c1678f9b1c532d54395
-  data.tar.gz: 35e21330e4da4297adcf4cbfa67ce939b5894981b352acfbc7fe8c02b1083436d037e9f7be45445445ef621ec783c84ce866f8636379609d7ff8c62e7c9fd392
+  metadata.gz: 0d0d206e647386006b07e1a89e1a5596346a50894d0260d5def25876902abcbd398c07889a03ac43c88041343e576b2793eec8a1eecdcf47ff1f025587721a38
+  data.tar.gz: 62b2dbbeadf2c8076f2dee4ad1a95491552dfb0be3af646b03ffa449dd54009cfd940020fe4dfe3b69c2c5e7a707bdafdc2959baebe109ba28d288b31a086031

data/exe/wal

@@ -7,7 +7,7 @@ begin
     Usage:
       wal watch --watcher <watcher-class>
         (--slot <replication-slot> | --tmp-slot)
-        [--publication=<publication>...]
+        --publication <publication>...
         [--replicator <replicator-class>]
       wal start <config-file>
 
@@ -16,7 +16,7 @@ begin
       --watcher=<watcher-class>          The watcher class to be used to listen for WAL changes.
       --slot=<replication-slot>          The replication slot that will be used.
       --tmp-slot                         Use a temporary replication slot.
-      [--publication=<publication>...]   Force using the informed Postgres publications.
+      [--publication=<publication>...]   Postgres publications to subscribe to.
       [--replicator=<replicatior_class>] Change the replication driver class
   DOCOPT
 rescue Docopt::Exit => err
@@ -28,24 +28,18 @@ require "./config/environment"
 
 db_config = ActiveRecord::Base.configurations.configs_for(name: "primary").configuration_hash
 
-def watch(db_config, cli)
-  watcher = cli["--watcher"].constantize.new
-  use_temporary_slot = cli["--tmp-slot"] || false
-  replication_slot = cli["--slot"]
-  replication_slot = replication_slot.presence || "wal_watcher_#{SecureRandom.alphanumeric(4)}" if use_temporary_slot
-  publications = cli["--publication"] || []
-  replicator_class = cli["--replicator"].presence&.constantize || Wal::Replicator
-
-  puts "Watcher started for #{replication_slot} slot (#{publications.join(", ")})"
-  replicator_class
-    .new(replication_slot:, use_temporary_slot:, db_config:)
-    .replicate_forever(Wal::LoggingWatcher.new(replication_slot, watcher), publications:)
-  puts "Watcher finished for #{replication_slot}"
-end
-
 if cli["watch"]
-  watch(db_config, cli)
+  config = {
+    "watcher" => cli["--watcher"],
+    "temporary" => cli["--tmp-slot"],
+    "publications" => cli["--publication"] || [],
+    "replicator" => cli["--replicator"],
+  }
+  slot = cli["--slot"].presence || "wal_watcher_#{SecureRandom.alphanumeric(4)}"
+  runner = Wal::Runner.new(config: { "slots" => { slot => config } }, db_config:)
+  runner.start
 elsif cli["start"]
-  runner = Wal::Runner.new(config_file: cli["<config-file>"], db_config:)
+  config = YAML.load_file(cli["<config-file>"])
+  runner = Wal::Runner.new(config:, db_config:)
   runner.start
 end

data/lib/wal/logging_watcher.rb

@@ -5,6 +5,8 @@ module Wal
     def initialize(slot, watcher)
       @slot = slot
       @watcher = watcher
+      @actions = Set.new
+      @tables = Set.new
     end
 
     def should_watch_table?(table)
@@ -18,25 +20,40 @@ module Wal
     def on_event(event)
       case event
       when Wal::BeginTransactionEvent
-        @start = Time.now
+        @start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         @count = 0
-        if event.estimated_size > 0
-          Wal.logger&.debug("[#{@slot}] Begin transaction=#{event.transaction_id} size=#{event.estimated_size}")
-        end
+        Wal.logger&.debug("[#{@slot}] Begin transaction=#{event.transaction_id} size=#{event.estimated_size}")
       when Wal::CommitTransactionEvent
-        if @count > 0
-          elapsed = ((Time.now - @start) * 1000.0).round(1)
-          Wal.logger&.info("[#{@slot}] Commit transaction=#{event.transaction_id} elapsed=#{elapsed} events=#{@count}")
+        elapsed = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - @start) * 1000).round(3)
+        message = "[#{@slot}] Commit transaction=#{event.transaction_id} elapsed=#{elapsed} events=#{@count}"
+        message << " actions=#{@actions.sort.join(",")}" unless @actions.empty?
+        message << " tables=#{@tables.sort.join(",")}" unless @tables.empty?
+        if @count > 1
+          Wal.logger&.info(message)
+        else
+          Wal.logger&.debug(message)
         end
+        @actions.clear
+        @tables.clear
       when Wal::InsertEvent
         Wal.logger&.debug("[#{@slot}] Insert transaction=#{event.transaction_id} table=#{event.table} primary_key=#{event.primary_key}")
         @count += 1
+        @actions << :insert
+        @tables << event.table
       when Wal::UpdateEvent
-        Wal.logger&.debug("[#{@slot}] Update transaction=#{event.transaction_id} table=#{event.table} primary_key=#{event.primary_key}")
+        if Wal.logger&.level >= Logger::DEBUG
+          message = "[#{@slot}] Update transaction=#{event.transaction_id} table=#{event.table} primary_key=#{event.primary_key}"
+          message << " changed=#{event.diff.keys.join(",")}" if event.diff && !event.diff.empty?
+          Wal.logger&.debug(message)
+        end
         @count += 1
+        @actions << :update
+        @tables << event.table
       when Wal::DeleteEvent
         Wal.logger&.debug("[#{@slot}] Delete transaction=#{event.transaction_id} table=#{event.table} primary_key=#{event.primary_key}")
         @count += 1
+        @actions << :delete
+        @tables << event.table
       else
         @count += 1
       end

data/lib/wal/runner.rb

@@ -32,30 +32,38 @@ module Wal
         Thread.new(slot, watcher, temporary, publications) do |replication_slot, watcher, use_temporary_slot, publications|
           retries = 0
           replication_slot = "#{replication_slot}_#{SecureRandom.alphanumeric(4)}" if use_temporary_slot
-          puts "Watcher started for #{replication_slot} slot (#{publications.join(", ")})"
+          puts "[#{replication_slot}] Watcher started for #{replication_slot} slot (#{publications.join(", ")})"
 
           begin
+            Wal.hooks[:on_slot_start]&.call(slot, config)
+
             replicator_class
               .new(db_config:, **replicator_params, replication_slot:, use_temporary_slot:)
               .replicate_forever(Wal::LoggingWatcher.new(replication_slot, watcher), publications:)
+
+            Wal.hooks[:on_slot_finish]&.call(slot, config)
+
             if auto_restart
               backoff_time = backoff_exponent ? (backoff * retries) ** backoff_exponent : backoff
-              puts "Watcher finished for #{replication_slot}, auto restarting in #{backoff_time.floor(2)}..."
+              puts "[#{replication_slot}] Watcher finished for #{replication_slot}, auto restarting in #{backoff_time.floor(2)}..."
               sleep backoff_time
-              puts "Restarting #{replication_slot}"
+              puts "[#{replication_slot}] Restarting"
               redo
             end
           rescue ArgumentError
             raise
           rescue StandardError => err
+            Wal.hooks[:on_slot_error]&.call(err, slot, config)
+
+            Wal.logger&.error("[#{replication_slot}] Error #{err}")
+            Wal.logger&.error([err.message, *err.backtrace].join("\n"))
+
             if retries < max_retries
-              Wal.logger&.error("[#{replication_slot}] Error #{err}")
-              Wal.logger&.error([err.message, *err.backtrace].join("\n"))
               retries += 1
               backoff_time = backoff_exponent ? (backoff * retries) ** backoff_exponent : backoff
-              puts "Restarting #{replication_slot} in #{backoff_time.floor(2)}s..."
+              puts "#{replication_slot}] Restarting #{replication_slot} in #{backoff_time.floor(2)}s..."
               sleep backoff_time
-              puts "Restarting #{replication_slot}"
+              puts "#{replication_slot}] Restarting #{replication_slot}"
               retry
             end
             raise
@@ -79,24 +87,38 @@ module Wal
       end
     end
 
-    attr_reader :config_file, :db_config
+    attr_reader :config, :db_config
 
-    def initialize(config_file:, db_config:)
-      @config_file = config_file
+    def initialize(config:, db_config:)
+      @config = config
       @db_config = db_config
       @child_pids = []
     end
 
     def start
-      slots = YAML.load_file(config_file)["slots"]
-      workers_slots = slots.group_by { |_slot, config| config["worker"] || "default" }
+      workers_slots = config["slots"].group_by { |_slot, slot_config| slot_config["worker"] || "default" }
+
+      if workers_slots.size == 1
+        run_single_worker(workers_slots.first)
+      else
+        run_forked_workers(workers_slots)
+      end
+    end
+
+    def run_single_worker((worker_name, slot_configs))
+      @ping_thread = start_ping_thread
+      puts "[#{worker_name}] Starting worker process (PID: #{Process.pid})"
+      worker = Worker.new(name: worker_name, slot_configs: slot_configs, db_config: db_config)
+      worker.run
+    end
 
-      Wal.fork_hooks[:before_fork]&.call
+    def run_forked_workers(workers_slots)
+      Wal.hooks[:before_fork]&.call(workers_slots)
 
       workers_slots.each do |worker_name, slot_configs|
         pid = fork_worker(worker_name, slot_configs)
         @child_pids << pid
-        puts "Spawned worker '#{worker_name}' with PID #{pid}"
+        puts "[#{worker_name}] Spawned worker '#{worker_name}' with PID #{pid}"
       end
 
       @ping_thread = start_ping_thread
@@ -108,7 +130,7 @@ module Wal
 
     def fork_worker(worker_name, slot_configs)
       Process.fork do
-        Wal.fork_hooks[:after_fork]&.call
+        Wal.hooks[:after_fork]&.call(worker_name, slot_configs)
         puts "[#{worker_name}] Starting worker process (PID: #{Process.pid})"
         worker = Worker.new(name: worker_name, slot_configs: slot_configs, db_config: db_config)
         worker.run

data/lib/wal/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Wal
-  VERSION = "0.0.32"
+  VERSION = "0.0.34"
 end

data/lib/wal.rb

@@ -16,22 +16,30 @@ require_relative "wal/version"
 module Wal
   class << self
     attr_accessor :logger
-    attr_accessor :fork_hooks
+    attr_accessor :hooks
 
     def logger
       @logger ||= Logger.new($stdout, level: :info)
     end
 
-    def fork_hooks
-      @fork_hooks ||= {}
+    def hooks
+      @hooks ||= {}
+    end
+
+    def on_slot_finished(&block)
+      hooks[:on_slot_finished] = block
+    end
+
+    def on_slot_error(&block)
+      hooks[:on_slot_error] = block
     end
 
     def before_fork(&block)
-      fork_hooks[:before_fork] = block
+      hooks[:before_fork] = block
     end
 
     def after_fork(&block)
-      fork_hooks[:after_fork] = block
+      hooks[:after_fork] = block
     end
   end
 

data/rbi/wal.rbi

@@ -7,17 +7,26 @@ module Wal
       UpdateEvent,
       DeleteEvent,
     ) }
-  VERSION = "0.0.32"
+  VERSION = "0.0.34"
 
   class << self
     sig { returns(T.class_of(Logger)) }
     attr_accessor :logger
 
-    sig { params(block: T.proc.void).void }
+    sig { params(block: T.proc.params(worker_name: T.untyped).void).void }
     def before_fork(&block); end
 
-    sig { params(block: T.proc.void).void }
+    sig { params(block: T.proc.params(worker_name: String, slot_configs: T.untyped).void).void }
     def after_fork(&block); end
+
+    sig { params(block: T.proc.params(slot: String, config: T.untyped).void).void }
+    def on_slot_start(&block); end
+
+    sig { params(block: T.proc.params(error: StandardError, slot: String, config: T.untyped).void).void }
+    def on_slot_error(&block); end
+
+    sig { params(block: T.proc.params(slot: String, config: T.untyped).void).void }
+    def on_slot_finish(&block); end
   end
 
   sig { params(block: T.proc.params(config: T.class_of(Wal)).void).void }

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: wal
 version: !ruby/object:Gem::Version
-  version: 0.0.32
+  version: 0.0.34
 platform: ruby
 authors:
 - Rodrigo Navarro
 bindir: exe
 cert_chain: []
-date: 2026-01-28 00:00:00.000000000 Z
+date: 2026-02-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg