wal 0.0.15 → 0.0.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 14c30e66cc9c1425cb894a9cfb558941017659cf4f0ad5a2183782c25df4a5b4
-  data.tar.gz: 7704871035cecbb90ee47cbefff50ad2fe58b4515f7d2c0d439e40adf31bf3f4
+  metadata.gz: d279bf26f64494f4f52cff1122ade2a9435cd3942be615a92a49f7a5dd64b5d6
+  data.tar.gz: 6b899eacde8085da1ab80bd9cb63741f9b0a7357c1fc86f7821d52d07fbbe48f
 SHA512:
-  metadata.gz: e47256ec844eace9aabb20a1364fc0d0ab724e81c335f7b0bbb826871a328d8a0dca806d3e2ec891717339acc2286b2f71c2f424ccfed60c906772ded95f10ef
-  data.tar.gz: 0c584c14ec81977b7a26d66117a42180c73450da3a53ec9fa7dc5fe6ddd11e941833de45024032b3a8c7d31ddab6f7261911eaff7e2d83328031d7b8f4ddf9b2
+  metadata.gz: 8ad343f0b1a3468cf4707e68fcda76ba9859dbba1243c2dfa90c33eef1d0ea50e2791d57ca824163c4f1c7f231ddfeea0c3919400ecba3836c5314947caab6ad
+  data.tar.gz: d5975b490f804064db2e1fba3dd02821d2877927d9333cbc8aa780a347295dd6816eb35a9468163c0d5f301a66d494504b4c195f79f30deb52f56cf48ef2b75a

data/README.md

@@ -1,49 +1,136 @@
 # Wal
 
-Easily hook into Postgres WAL event log from your Rails app.
+Wal is a framework that lets you hook into Postgres WAL events directly from your Rails application.
 
-Proper documentation TBD
+Unlike using database triggers, Wal allows you to keep your logic in your application code while still reacting to persistence events coming from the database.
 
-## Examples
+Also, unlike ActiveRecord callbacks, these events are guaranteed by Postgres to be 100% consistent, ensuring you never miss one.
 
-### Watch for model changes using the RecordWatcher DSL
+# Getting started
+
+## Installation
+
+Add `wal` to your application's Gemfile:
 
 ```ruby
-class ProductAvailabilityWatcher < Wal::RecordWatcher
-  on_save Product, changed: %w[price] do |event|
-    recalculate_inventory_price(event.primary_key, event.new["price"])
-  end
+gem "wal"
+```
+
+And then:
+
+```bash
+$ bundle install
+```
+
+## Getting started
 
-  on_delete Product do |event|
-    clear_product_inventory(event.primary_key)
+The core building block in Wal is a `Watcher`. The easiest way to create one is by extending `Wal::RecordWatcher`, which handles most of the boilerplate for you.
+
+For example, let's create a watcher that denormalizes `Post` and `Category` models into a `DenormalizedPost`.
+
+Create a new file at `app/watchers/denormalize_post_watcher.rb`:
+
+```ruby
+class DenormalizePostWatcher < Wal::RecordWatcher
+  # When a new `Post` is created, we create a new `DenormalizedPost` record
+  on_insert Post do |event|
+    DenormalizedPost.create!(
+      post_id: event.primary_key,
+      title: event.new["title"],
+      body: event.new["body"],
+      category_id: event.new["category_id"],
+      category_name: Category.find_by(id: event.new["category_id"])&.name,
+    )
   end
 
-  on_save Sales, changed: %w[status] do |event|
-    recalculate_inventory_quantity(event.primary_key)
+  # When a `Post` title or body is changed, we update its `DenormalizedPost` record
+  on_update Post, changed: [:title, :body] do |event|
+    DenormalizedPost
+      .where(post_id: event.primary_key)
+      .update_all(
+        title: event.new["title"],
+        body: event.new["body"],
+      )
   end
 
-  def recalculate_inventory_price(product_id, new_price)
-    # ...
+  # When a `Post` category changes, we also update its `DenormalizedPost` record
+  on_update Post, changed: [:category_id] do |event|
+    DenormalizedPost
+      .where(post_id: event.primary_key)
+      .update_all(
+        category_id: event.new["category_id"],
+        category_name: Category.find_by(id: event.new["category_id"])&.name,
+      )
   end
 
-  def clear_product_inventory(product_id)
-    # ...
+  # When a `Category` changes, we update all the `DenormalizedPosts` referencing it
+  on_update Category, changed: [:name] do |event|
+    DenormalizedPost
+      .where(category_id: event.primary_key)
+      .update_all(
+        category_name: event.new["name"],
+      )
   end
 
-  def recalculate_inventory_quantity(sales_id)
-    # ...
+  # Finally when a `Category` is deleted, we clear all the `DenormalizedPosts` referencing it
+  on_delete Category do |event|
+    DenormalizedPost
+      .where(category_id: event.primary_key)
+      .update_all(
+        category_id: nil,
+        category_name: nil,
+      )
   end
 end
 ```
 
-### Basic watcher implementation
+You might wonder: *Why not just use ActiveRecord callbacks for this?*
 
-```ruby
-class LogWatcher
-  include Wal::Watcher
+And while it is hard to justify that for our simple example, ActiveRecord callbacks are not guaranteed to always run. Depending on the methods you use to perform the changes, they can be skipped.
+
+Wal ensures every single change is captured. *Even when updates happen directly in the database and bypass Rails entirely*. That's the main reason to use it: when you need 100% consistency.
+
+Usually one could resort into database triggers when full consistency is required, but running and maintaining application level code on the database tends to be painful. Wal let's you do the same but at the application level.
 
-  def on_event(event)
-    puts "Wal event received #{event}"
+## Configuring the Watcher
+
+Wal relies on [Postgres logical replication](https://www.postgresql.org/docs/current/logical-replication.html) to stream changes to your watchers.
+
+First, create a [Postgres publication](https://www.postgresql.org/docs/current/logical-replication-publication.html) for the tables your watcher uses. Wal provides a generator for this:
+
+```
+$ rails generate wal:migration DenormalizePostWatcher
+```
+
+This will generate a new migration with all the tables that your watcher uses:
+```ruby
+class SetDenormalizePostWatcherPublication < ActiveRecord::Migration
+  def change
+    define_publication :denormalize_post_publication do |p|
+      p.table :posts
+      p.table :categories
+    end
   end
 end
 ```
+
+Next, create a `config/wal.yml` configuration file to link the `Watcher` to its publication:
+
+```yaml
+slots:
+  denormalize_posts:
+    watcher: DenormalizePostWatcher
+    publications:
+      - denormalize_post_publication
+```
+This associates your watcher with the `denormalize_post_publication` and with the `denormalize_posts` [Postgres replication slot](https://www.postgresql.org/docs/9.4/warm-standby.html#STREAMING-REPLICATION-SLOTS).
+
+## Running the Watcher
+
+With everything configured, start the Wal process:
+
+```bash
+bundle exec wal start config/wal.yaml
+```
+
+Wal will now process your replication slot and run the `DenormalizePostWatcher` whenever a change occur.

data/exe/wal

@@ -28,6 +28,47 @@ require "./config/environment"
 
 db_config = ActiveRecord::Base.configurations.configs_for(name: "primary").configuration_hash
 
+class Wal::LoggingReplicator
+  def initialize(slot, replicator)
+    @slot = slot
+    @replicator = replicator
+  end
+
+  def replicate_forever(watcher, publications:)
+    replication = @replicator.replicate(watcher, publications:)
+    count = 0
+    start = Time.now
+    loop do
+      case (event = replication.next)
+      when Wal::BeginTransactionEvent
+        start = Time.now
+        count = 0
+        if event.estimated_size > 0
+          Wal.logger&.info("[#{@slot}] Begin transaction=#{event.transaction_id} size=#{event.estimated_size}")
+        end
+      when Wal::CommitTransactionEvent
+        if count > 0
+          elapsed = ((Time.now - start) * 1000.0).round(1)
+          Wal.logger&.info("[#{@slot}] Commit transaction=#{event.transaction_id} elapsed=#{elapsed} events=#{count}")
+        end
+      when Wal::InsertEvent
+        Wal.logger&.debug("[#{@slot}] Insert transaction=#{event.transaction_id} table=#{event.table} primary_key=#{event.primary_key}")
+        count += 1
+      when Wal::UpdateEvent
+        Wal.logger&.debug("[#{@slot}] Update transaction=#{event.transaction_id} table=#{event.table} primary_key=#{event.primary_key}")
+        count += 1
+      when Wal::DeleteEvent
+        Wal.logger&.debug("[#{@slot}] Delete transaction=#{event.transaction_id} table=#{event.table} primary_key=#{event.primary_key}")
+        count += 1
+      else
+        count += 1
+      end
+    end
+  rescue StopIteration
+    nil
+  end
+end
+
 if cli["watch"]
   watcher = cli["--watcher"].constantize.new
   use_temporary_slot = cli["--tmp-slot"] || false
@@ -37,9 +78,9 @@ if cli["watch"]
   replicator = cli["--replicator"].presence&.constantize || Wal::Replicator
 
   puts "Watcher started for #{replication_slot} slot (#{publications.join(", ")})"
-  replicator
-    .new(replication_slot:, use_temporary_slot:, db_config:)
-    .replicate_forever(watcher, publications:)
+  replicator = replicator.new(replication_slot:, use_temporary_slot:, db_config:)
+  replicator = Wal::LoggingReplicator.new(replication_slot, replicator)
+  replicator.replicate_forever(watcher, publications:)
   puts "Watcher finished for #{replication_slot}"
 
 elsif cli["start"]
@@ -50,16 +91,30 @@ elsif cli["start"]
     temporary = config["temporary"] || false
     publications = config["publications"] || []
     replicator = config["replicator"].presence&.constantize || Wal::Replicator
+    retries = config["retries"]&.to_i || 5
 
     Thread.new(slot, watcher, temporary, publications) do |replication_slot, watcher, use_temporary_slot, publications|
       replication_slot = "#{replication_slot}_#{SecureRandom.alphanumeric(4)}" if use_temporary_slot
       puts "Watcher started for #{replication_slot} slot (#{publications.join(", ")})"
 
-      replicator
-        .new(replication_slot:, use_temporary_slot:, db_config:)
-        .replicate_forever(watcher, publications:)
+      replicator = replicator.new(replication_slot:, use_temporary_slot:, db_config:)
+      replicator = Wal::LoggingReplicator.new(replication_slot, replicator)
+
+      begin
+        replicator.replicate_forever(watcher, publications:)
+      rescue StandardError => err
+        if retries > 0
+          Wal.logger&.error("[#{replication_slot}] Error #{err}")
+          retries -= 1
+          sleep 2 ** retries
+          retry
+        end
+        raise
+      end
 
       puts "Watcher finished for #{replication_slot}"
+
+      Process.kill("TERM", Process.pid)
     end
   end
 

data/lib/wal/generators/migration.rb

@@ -32,7 +32,7 @@ module Wal
       end
 
       def publication_name
-        "#{watcher.underscore}_publication"
+        "#{class_name.gsub("Watcher", "").underscore}_publication"
       end
 
       def class_name

data/lib/wal/replicator.rb

@@ -88,6 +88,7 @@ module Wal
 
         in XLogData(lsn:, data: PG::Replication::PGOutput::Message(prefix: "wal_ping"))
           watch_conn.standby_status_update(write_lsn: [watch_conn.last_confirmed_lsn, lsn].compact.max)
+          next
 
         in XLogData(data: PG::Replication::PGOutput::Message(prefix:, content:)) if watcher.valid_context_prefix? prefix
           begin

data/lib/wal/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Wal
-  VERSION = "0.0.15"
+  VERSION = "0.0.18"
 end

data/lib/wal.rb

@@ -12,6 +12,14 @@ require_relative "wal/railtie"
 require_relative "wal/version"
 
 module Wal
+  class << self
+    attr_accessor :logger
+  end
+
+  def self.configure(&block)
+    yield self
+  end
+
   class BeginTransactionEvent < Data.define(:transaction_id, :lsn, :final_lsn, :timestamp)
     def estimated_size
       final_lsn - lsn

data/rbi/wal.rbi

@@ -7,7 +7,15 @@ module Wal
       UpdateEvent,
       DeleteEvent,
     ) }
-  VERSION = "0.0.15"
+  VERSION = "0.0.18"
+
+  class << self
+    sig { returns(T.class_of(Logger)) }
+    attr_accessor :logger
+  end
+
+  sig { params(block: T.proc.params(config: T.class_of(Wal)).void).void }
+  def self.configure(&block); end
 
   class BeginTransactionEvent < T::Struct
     prop :transaction_id, Integer, immutable: true

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: wal
 version: !ruby/object:Gem::Version
-  version: 0.0.15
+  version: 0.0.18
 platform: ruby
 authors:
 - Rodrigo Navarro
 bindir: exe
 cert_chain: []
-date: 2025-09-16 00:00:00.000000000 Z
+date: 2025-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -104,7 +104,6 @@ files:
 - lib/wal/version.rb
 - lib/wal/watcher.rb
 - rbi/wal.rbi
-- sig/wal.rbs
 licenses:
 - MIT
 metadata:

data/sig/wal.rbs

@@ -1,186 +0,0 @@
-# typed: strong
-module Wal
-end
-
-Wal::Event: untyped
-
-Wal::VERSION: untyped
-
-class Wal::BeginTransactionEvent < T::Struct
-  def estimated_size: () -> Integer
-end
-
-class Wal::CommitTransactionEvent < T::Struct
-end
-
-module Wal::ChangeEvent
-  def diff: () -> ::Hash[String, [ untyped, untyped ]]
-
-  def changed_attribute?: (Symbol | String attribute) -> bool
-
-  def attribute: (Symbol | String attribute) -> untyped
-
-  def attribute_changes: (Symbol | String attribute) -> [ untyped, untyped ]?
-
-  def attribute_was: (Symbol | String attribute) -> untyped
-end
-
-class Wal::InsertEvent < T::Struct
-  include ::Wal::ChangeEvent
-
-  def diff: () -> ::Hash[String, [ untyped, untyped ]]
-end
-
-class Wal::UpdateEvent < T::Struct
-  include ::Wal::ChangeEvent
-
-  def diff: () -> ::Hash[String, [ untyped, untyped ]]
-end
-
-class Wal::DeleteEvent < T::Struct
-  include ::Wal::ChangeEvent
-
-  def diff: () -> ::Hash[String, [ untyped, untyped ]]
-end
-
-module Wal::ActiveRecordContextExtension
-  def set_wal_watcher_context: (untyped context, ?prefix: untyped prefix) -> untyped
-end
-
-class Wal::NoopWatcher
-  include Wal::Watcher
-
-  def on_event: (Event event) -> void
-end
-
-class Wal::RecordWatcher
-  include Wal::Watcher
-
-  extend T::Helpers
-
-  def self.inherited: (untyped subclass) -> untyped
-
-  def self.on_insert: (String | singleton(::ActiveRecord::Base) table) { (InsertEvent event) -> void } -> void
-
-  def self.on_update: (String | singleton(::ActiveRecord::Base) table, ?changed: ::Array[String | Symbol]? changed) { (UpdateEvent event) -> void } -> void
-
-  def self.on_save: (String | singleton(::ActiveRecord::Base) table, ?changed: ::Array[String | Symbol]? changed) { (InsertEvent | UpdateEvent event) -> void } -> void
-
-  def self.on_destroy: (String | singleton(::ActiveRecord::Base) table) { (DeleteEvent event) -> void } -> void
-
-  def on_record_changed: (RecordEvent event) -> void
-
-  def should_watch_table?: (String table) -> bool
-
-  def aggregation_strategy: (BeginTransactionEvent event) -> Symbol
-
-  def on_event: (Event event) -> void
-end
-
-Wal::Wal::RecordWatcher::RecordEvent: untyped
-
-class Wal::Wal::RecordWatcher::MemoryRecordWatcher
-  include Wal::Watcher
-
-  include Wal::Watcher::SeparatedEvents
-
-  extend T::Helpers
-
-  def initialize: (untyped watcher) -> void
-
-  def on_begin: (BeginTransactionEvent event) -> void
-
-  def on_commit: (untyped _event) -> untyped
-
-  def on_insert: (InsertEvent event) -> void
-
-  def on_update: (UpdateEvent event) -> void
-
-  def on_delete: (DeleteEvent event) -> void
-end
-
-Wal::Wal::RecordWatcher::Wal::Wal::RecordWatcher::MemoryRecordWatcher::RecordsStorage: untyped
-
-class Wal::Wal::RecordWatcher::TemporaryTableRecordWatcher
-  include Wal::Watcher
-
-  include Wal::Watcher::SeparatedEvents
-
-  extend T::Helpers
-
-  def initialize: (untyped watcher, ?batch_size: untyped batch_size) -> void
-
-  def on_begin: (BeginTransactionEvent event) -> void
-
-  def on_commit: (untyped _event) -> untyped
-
-  def on_insert: (InsertEvent event) -> void
-
-  def on_update: (UpdateEvent event) -> void
-
-  def on_delete: (DeleteEvent event) -> void
-
-  def base_class: () -> singleton(::ActiveRecord::Base)
-
-  def serialize: (untyped event) -> untyped
-
-  def deserialize: (untyped persisted_event) -> untyped
-end
-
-class Wal::Replicator
-  include PG::Replication::Protocol
-
-  def initialize: (?replication_slot: String replication_slot, ?use_temporary_slot: bool use_temporary_slot, ?db_config: ::Hash[Symbol, untyped] db_config) -> void
-
-  def replicate_forever: (Watcher watcher, ?publications: ::Array[String] publications) -> void
-
-  def replicate: (Watcher watcher, ?publications: ::Array[String] publications) -> ::Enumerator::Lazy[Event]
-end
-
-class Wal::Wal::Replicator::Column < T::Struct
-  def decode: (untyped value) -> untyped
-end
-
-class Wal::Wal::Replicator::Table < T::Struct
-  def primary_key: (untyped decoded_row) -> untyped
-
-  def decode_row: (untyped values) -> untyped
-end
-
-class Wal::StreamingWatcher
-  include Wal::Watcher
-
-  extend T::Helpers
-
-  def on_transaction_events: (::Enumerator[Event] events) -> void
-
-  def queue_size: (BeginTransactionEvent event) -> Integer
-
-  def on_event: (Event event) -> void
-end
-
-module Wal::Watcher
-  include Wal
-
-  extend T::Helpers
-
-  def on_event: (Event event) -> void
-
-  def should_watch_table?: (String table) -> bool
-
-  def valid_context_prefix?: (String prefix) -> bool
-end
-
-module Wal::Wal::Watcher::SeparatedEvents
-  def on_event: (Event event) -> void
-
-  def on_begin: (BeginTransactionEvent event) -> void
-
-  def on_insert: (InsertEvent event) -> void
-
-  def on_update: (UpdateEvent event) -> void
-
-  def on_delete: (DeleteEvent event) -> void
-
-  def on_commit: (CommitTransactionEvent event) -> void
-end