wal 0.0.16 → 0.0.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fad10e5a8aea823dba89b5f50ba13e60b771d442e9dbe682b0b03c8c0969b1c1
-  data.tar.gz: b1a2fff4a202dad05abe78d79fed809d18b6396fd97a8d6aef3438e7a30ab3c8
+  metadata.gz: 8547683e990bc915e382feecac5220a3b6b31a3d7e59c0891511f30c2eec5d19
+  data.tar.gz: e1bbac497d67a1cf76c8e56a002f45bf12849fe4fdfb4415ea3d8ee573e44e81
 SHA512:
-  metadata.gz: cb9cd1dcdb4afa3611597d6aac89470892d63d0fabe1db2cb1ba6c6e8013cf610cf27e20185a40a8aa0365ee5e5639b44789718aa46cded921dfa4c2c92c7938
-  data.tar.gz: 1655d1ee9a327383ddde0483a1e8a432d9b529601b7d3f700a427a0ac1560c01f786230e8fd93d62fc503d688cd4d02a92c69d7e53d05801c62106b2dfcb00da
+  metadata.gz: f119216774509c4056ed84ccd29d43c9d743d67c3365289b2b18be0b92dd2117f543bf282fb6071abdf27d3774a2e1326238ab73e37c4c7e55bd5267c6583b96
+  data.tar.gz: d938d438d7d7b79457dba62a4460fba9e9523f953a618eacf4706ac1cf01293aafdbbf71ec35cf8a0a5156ea4b38a4e138c0cc7a373c38f6208f6136d926bd17

data/README.md

@@ -4,7 +4,7 @@ Wal is a framework that lets you hook into Postgres WAL events directly from you
 
 Unlike using database triggers, Wal allows you to keep your logic in your application code while still reacting to persistence events coming from the database.
 
-Also, unlike ActiveRecord callbacks, these events are guaranteed by Postgre to be 100% consistent, ensuring you never miss one.
+Also, unlike ActiveRecord callbacks, these events are guaranteed by Postgres to be 100% consistent, ensuring you never miss one.
 
 # Getting started
 
@@ -45,42 +45,52 @@ class DenormalizePostWatcher < Wal::RecordWatcher
 
   # When a `Post` title or body is changed, we update its `DenormalizedPost` record
   on_update Post, changed: [:title, :body] do |event|
-    DenormalizedPost.where(post_id: event.primary_key).update_all(
-      title: event.new["title"],
-      body: event.new["body"],
-    )
+    DenormalizedPost
+      .where(post_id: event.primary_key)
+      .update_all(
+        title: event.new["title"],
+        body: event.new["body"],
+      )
   end
 
   # When a `Post` category changes, we also update its `DenormalizedPost` record
   on_update Post, changed: [:category_id] do |event|
-    DenormalizedPost.where(post_id: event.primary_key).update_all(
-      category_id: event.new["category_id"],
-      category_name: Category.find_by(id: event.new["category_id"])&.name,
-    )
+    DenormalizedPost
+      .where(post_id: event.primary_key)
+      .update_all(
+        category_id: event.new["category_id"],
+        category_name: Category.find_by(id: event.new["category_id"])&.name,
+      )
   end
 
   # When a `Category` changes, we update all the `DenormalizedPosts` referencing it
   on_update Category, changed: [:name] do |event|
-    DenormalizedPost.where(category_id: event.primary_key).update_all(
-      category_name: event.new["name"],
-    )
+    DenormalizedPost
+      .where(category_id: event.primary_key)
+      .update_all(
+        category_name: event.new["name"],
+      )
   end
 
   # Finally when a `Category` is deleted, we clear all the `DenormalizedPosts` referencing it
-  on_update Category, changed: [:name] do |event|
-    DenormalizedPost.where(category_id: event.primary_key).update_all(
-      category_id: nil,
-      category_name: nil,
-    )
+  on_delete Category do |event|
+    DenormalizedPost
+      .where(category_id: event.primary_key)
+      .update_all(
+        category_id: nil,
+        category_name: nil,
+      )
   end
 end
 ```
 
 You might wonder: *Why not just use ActiveRecord callbacks for this?*
 
-While callbacks seem simpler, they are not guaranteed to always run. Depending on the methods you use to perform the changes, it can be skipped.
+And while it is hard to justify that for our simple example, ActiveRecord callbacks are not guaranteed to always run. Depending on the methods you use to perform the changes, they can be skipped.
+
+Wal ensures every single change is captured. *Even when updates happen directly in the database and bypass Rails entirely*. That's the main reason to use it: when you need 100% consistency.
 
-Wal ensures every single change is captured. Even if updates happen directly in the database and bypass Rails entirely. That's the main reason to use it: when you need 100% consistency.
+Usually one could resort into database triggers when full consistency is required, but running and maintaining application level code on the database tends to be painful. Wal let's you do the same but at the application level.
 
 ## Configuring the Watcher
 

data/exe/wal

@@ -28,18 +28,59 @@ require "./config/environment"
 
 db_config = ActiveRecord::Base.configurations.configs_for(name: "primary").configuration_hash
 
+class Wal::LoggingReplicator
+  def initialize(slot, replicator)
+    @slot = slot
+    @replicator = replicator
+  end
+
+  def replicate_forever(watcher, publications:)
+    replication = @replicator.replicate(watcher, publications:)
+    count = 0
+    start = Time.now
+    loop do
+      case (event = replication.next)
+      when Wal::BeginTransactionEvent
+        start = Time.now
+        count = 0
+        if event.estimated_size > 0
+          Wal.logger&.info("[#{@slot}] Begin transaction=#{event.transaction_id} size=#{event.estimated_size}")
+        end
+      when Wal::CommitTransactionEvent
+        if count > 0
+          elapsed = ((Time.now - start) * 1000.0).round(1)
+          Wal.logger&.info("[#{@slot}] Commit transaction=#{event.transaction_id} elapsed=#{elapsed} events=#{count}")
+        end
+      when Wal::InsertEvent
+        Wal.logger&.debug("[#{@slot}] Insert transaction=#{event.transaction_id} table=#{event.table} primary_key=#{event.primary_key}")
+        count += 1
+      when Wal::UpdateEvent
+        Wal.logger&.debug("[#{@slot}] Update transaction=#{event.transaction_id} table=#{event.table} primary_key=#{event.primary_key}")
+        count += 1
+      when Wal::DeleteEvent
+        Wal.logger&.debug("[#{@slot}] Delete transaction=#{event.transaction_id} table=#{event.table} primary_key=#{event.primary_key}")
+        count += 1
+      else
+        count += 1
+      end
+    end
+  rescue StopIteration
+    nil
+  end
+end
+
 if cli["watch"]
   watcher = cli["--watcher"].constantize.new
   use_temporary_slot = cli["--tmp-slot"] || false
   replication_slot = cli["--slot"]
   replication_slot = replication_slot.presence || "wal_watcher_#{SecureRandom.alphanumeric(4)}" if use_temporary_slot
   publications = cli["--publication"] || []
-  replicator = cli["--replicator"].presence&.constantize || Wal::Replicator
+  replicator_class = cli["--replicator"].presence&.constantize || Wal::Replicator
 
   puts "Watcher started for #{replication_slot} slot (#{publications.join(", ")})"
-  replicator
-    .new(replication_slot:, use_temporary_slot:, db_config:)
-    .replicate_forever(watcher, publications:)
+  replicator = replicator_class.new(replication_slot:, use_temporary_slot:, db_config:)
+  replicator = Wal::LoggingReplicator.new(replication_slot, replicator)
+  replicator.replicate_forever(watcher, publications:)
   puts "Watcher finished for #{replication_slot}"
 
 elsif cli["start"]
@@ -49,17 +90,31 @@ elsif cli["start"]
     watcher = config["watcher"].constantize.new
     temporary = config["temporary"] || false
     publications = config["publications"] || []
-    replicator = config["replicator"].presence&.constantize || Wal::Replicator
+    replicator_class = config["replicator"].presence&.constantize || Wal::Replicator
+    retries = config["retries"]&.to_i || 5
 
     Thread.new(slot, watcher, temporary, publications) do |replication_slot, watcher, use_temporary_slot, publications|
       replication_slot = "#{replication_slot}_#{SecureRandom.alphanumeric(4)}" if use_temporary_slot
       puts "Watcher started for #{replication_slot} slot (#{publications.join(", ")})"
 
-      replicator
-        .new(replication_slot:, use_temporary_slot:, db_config:)
-        .replicate_forever(watcher, publications:)
+      replicator = replicator_class.new(replication_slot:, use_temporary_slot:, db_config:)
+      replicator = Wal::LoggingReplicator.new(replication_slot, replicator)
+
+      begin
+        replicator.replicate_forever(watcher, publications:)
+      rescue StandardError => err
+        if retries > 0
+          Wal.logger&.error("[#{replication_slot}] Error #{err}")
+          retries -= 1
+          sleep 2 ** retries
+          retry
+        end
+        raise
+      end
 
       puts "Watcher finished for #{replication_slot}"
+
+      Process.kill("TERM", Process.pid)
     end
   end
 

data/lib/wal/replicator.rb

@@ -88,6 +88,7 @@ module Wal
 
         in XLogData(lsn:, data: PG::Replication::PGOutput::Message(prefix: "wal_ping"))
           watch_conn.standby_status_update(write_lsn: [watch_conn.last_confirmed_lsn, lsn].compact.max)
+          next
 
         in XLogData(data: PG::Replication::PGOutput::Message(prefix:, content:)) if watcher.valid_context_prefix? prefix
           begin

data/lib/wal/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Wal
-  VERSION = "0.0.16"
+  VERSION = "0.0.19"
 end

data/lib/wal.rb

@@ -12,6 +12,14 @@ require_relative "wal/railtie"
 require_relative "wal/version"
 
 module Wal
+  class << self
+    attr_accessor :logger
+  end
+
+  def self.configure(&block)
+    yield self
+  end
+
   class BeginTransactionEvent < Data.define(:transaction_id, :lsn, :final_lsn, :timestamp)
     def estimated_size
       final_lsn - lsn

data/rbi/wal.rbi

@@ -7,7 +7,15 @@ module Wal
       UpdateEvent,
       DeleteEvent,
     ) }
-  VERSION = "0.0.16"
+  VERSION = "0.0.19"
+
+  class << self
+    sig { returns(T.class_of(Logger)) }
+    attr_accessor :logger
+  end
+
+  sig { params(block: T.proc.params(config: T.class_of(Wal)).void).void }
+  def self.configure(&block); end
 
   class BeginTransactionEvent < T::Struct
     prop :transaction_id, Integer, immutable: true

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: wal
 version: !ruby/object:Gem::Version
-  version: 0.0.16
+  version: 0.0.19
 platform: ruby
 authors:
 - Rodrigo Navarro
 bindir: exe
 cert_chain: []
-date: 2025-09-17 00:00:00.000000000 Z
+date: 2025-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg