wal 0.0.14 → 0.0.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a10c111df779be284d3a1e20dac362c574d0abed4483d86479ad716c39c868dd
-  data.tar.gz: 18763673175466a85adef9b3d91eb6a188bfc2db1b8cd178678757b81a912eb1
+  metadata.gz: fad10e5a8aea823dba89b5f50ba13e60b771d442e9dbe682b0b03c8c0969b1c1
+  data.tar.gz: b1a2fff4a202dad05abe78d79fed809d18b6396fd97a8d6aef3438e7a30ab3c8
 SHA512:
-  metadata.gz: 762f7c704d2a0dceafd9c1094e1413dbe272b7c71d676a49bcbc652d0d5266e91de357f8dd44cd5b4e26b8ec0ba2bde628c997b83cb64e1df9a218164d1ec686
-  data.tar.gz: ed456c724a0d80a36f7d78bd477a059d89142f68f85293ff9225eb8ad4cc9a1fc3900781cf888c36742bb4406161542d9935fef9e6e3dd3919d4da51fe4f390c
+  metadata.gz: cb9cd1dcdb4afa3611597d6aac89470892d63d0fabe1db2cb1ba6c6e8013cf610cf27e20185a40a8aa0365ee5e5639b44789718aa46cded921dfa4c2c92c7938
+  data.tar.gz: 1655d1ee9a327383ddde0483a1e8a432d9b529601b7d3f700a427a0ac1560c01f786230e8fd93d62fc503d688cd4d02a92c69d7e53d05801c62106b2dfcb00da

data/README.md

@@ -1,49 +1,126 @@
 # Wal
 
-Easily hook into Postgres WAL event log from your Rails app.
+Wal is a framework that lets you hook into Postgres WAL events directly from your Rails application.
 
-Proper documentation TBD
+Unlike using database triggers, Wal allows you to keep your logic in your application code while still reacting to persistence events coming from the database.
 
-## Examples
+Also, unlike ActiveRecord callbacks, these events are guaranteed by Postgre to be 100% consistent, ensuring you never miss one.
 
-### Watch for model changes using the RecordWatcher DSL
+# Getting started
+
+## Installation
+
+Add `wal` to your application's Gemfile:
 
 ```ruby
-class ProductAvailabilityWatcher < Wal::RecordWatcher
-  on_save Product, changed: %w[price] do |event|
-    recalculate_inventory_price(event.primary_key, event.new["price"])
-  end
+gem "wal"
+```
+
+And then:
+
+```bash
+$ bundle install
+```
+
+## Getting started
 
-  on_destroy Product do |event|
-    clear_product_inventory(event.primary_key)
+The core building block in Wal is a `Watcher`. The easiest way to create one is by extending `Wal::RecordWatcher`, which handles most of the boilerplate for you.
+
+For example, let's create a watcher that denormalizes `Post` and `Category` models into a `DenormalizedPost`.
+
+Create a new file at `app/watchers/denormalize_post_watcher.rb`:
+
+```ruby
+class DenormalizePostWatcher < Wal::RecordWatcher
+  # When a new `Post` is created, we create a new `DenormalizedPost` record
+  on_insert Post do |event|
+    DenormalizedPost.create!(
+      post_id: event.primary_key,
+      title: event.new["title"],
+      body: event.new["body"],
+      category_id: event.new["category_id"],
+      category_name: Category.find_by(id: event.new["category_id"])&.name,
+    )
   end
 
-  on_save Sales, changed: %w[status] do |event|
-    recalculate_inventory_quantity(event.primary_key)
+  # When a `Post` title or body is changed, we update its `DenormalizedPost` record
+  on_update Post, changed: [:title, :body] do |event|
+    DenormalizedPost.where(post_id: event.primary_key).update_all(
+      title: event.new["title"],
+      body: event.new["body"],
+    )
   end
 
-  def recalculate_inventory_price(product_id, new_price)
-    # ...
+  # When a `Post` category changes, we also update its `DenormalizedPost` record
+  on_update Post, changed: [:category_id] do |event|
+    DenormalizedPost.where(post_id: event.primary_key).update_all(
+      category_id: event.new["category_id"],
+      category_name: Category.find_by(id: event.new["category_id"])&.name,
+    )
   end
 
-  def clear_product_inventory(product_id)
-    # ...
+  # When a `Category` changes, we update all the `DenormalizedPosts` referencing it
+  on_update Category, changed: [:name] do |event|
+    DenormalizedPost.where(category_id: event.primary_key).update_all(
+      category_name: event.new["name"],
+    )
   end
 
-  def recalculate_inventory_quantity(sales_id)
-    # ...
+  # Finally when a `Category` is deleted, we clear all the `DenormalizedPosts` referencing it
+  on_update Category, changed: [:name] do |event|
+    DenormalizedPost.where(category_id: event.primary_key).update_all(
+      category_id: nil,
+      category_name: nil,
+    )
   end
 end
 ```
 
-### Basic watcher implementation
+You might wonder: *Why not just use ActiveRecord callbacks for this?*
 
-```ruby
-class LogWatcher
-  include Wal::Watcher
+While callbacks seem simpler, they are not guaranteed to always run. Depending on the methods you use to perform the changes, it can be skipped.
+
+Wal ensures every single change is captured. Even if updates happen directly in the database and bypass Rails entirely. That's the main reason to use it: when you need 100% consistency.
+
+## Configuring the Watcher
 
-  def on_event(event)
-    puts "Wal event received #{event}"
+Wal relies on [Postgres logical replication](https://www.postgresql.org/docs/current/logical-replication.html) to stream changes to your watchers.
+
+First, create a [Postgres publication](https://www.postgresql.org/docs/current/logical-replication-publication.html) for the tables your watcher uses. Wal provides a generator for this:
+
+```
+$ rails generate wal:migration DenormalizePostWatcher
+```
+
+This will generate a new migration with all the tables that your watcher uses:
+```ruby
+class SetDenormalizePostWatcherPublication < ActiveRecord::Migration
+  def change
+    define_publication :denormalize_post_publication do |p|
+      p.table :posts
+      p.table :categories
+    end
   end
 end
 ```
+
+Next, create a `config/wal.yml` configuration file to link the `Watcher` to its publication:
+
+```yaml
+slots:
+  denormalize_posts:
+    watcher: DenormalizePostWatcher
+    publications:
+      - denormalize_post_publication
+```
+This associates your watcher with the `denormalize_post_publication` and with the `denormalize_posts` [Postgres replication slot](https://www.postgresql.org/docs/9.4/warm-standby.html#STREAMING-REPLICATION-SLOTS).
+
+## Running the Watcher
+
+With everything configured, start the Wal process:
+
+```bash
+bundle exec wal start config/wal.yaml
+```
+
+Wal will now process your replication slot and run the `DenormalizePostWatcher` whenever a change occur.

data/lib/wal/generators/migration.rb

@@ -32,7 +32,7 @@ module Wal
       end
 
       def publication_name
-        "#{watcher.underscore}_publication"
+        "#{class_name.gsub("Watcher", "").underscore}_publication"
       end
 
       def class_name

data/lib/wal/record_watcher.rb

@@ -27,7 +27,7 @@ module Wal
   #     end
   #   end
   #
-  #   on_destroy OrderItem do |event|
+  #   on_delete OrderItem do |event|
   #     recalculate_inventory_availability(event.attribute(:item_id))
   #   end
   #
@@ -60,7 +60,7 @@ module Wal
       @@change_callbacks[table].push(only: [:create, :update], changed: changed&.map(&:to_s), block: block)
     end
 
-    def self.on_destroy(table, &block)
+    def self.on_delete(table, &block)
       table = table.is_a?(String) ? table : table.table_name
       @@delete_callbacks[table].push(block: block)
     end
@@ -102,7 +102,7 @@ module Wal
       (@@change_callbacks.keys | @@delete_callbacks.keys).include? table
     end
 
-    # `RecordWatcher` supports two processing strategies:
+    # `RecordWatcher` supports three processing strategies:
     #
     # `:memory`: Stores and aggregates records from a single transaction in memory. This has better performance but uses
     #  more memory, as at least one event for each record must be stored in memory until the end of a transaction
@@ -110,6 +110,10 @@ module Wal
     # `:temporary_table`: Offloads the record aggregation to a temporary table on the database. This is useful when you
     # are processing very large transactions that can't fit in memory. The tradeoff is obviously a worse performance.
     #
+    # `:none`: Doesn't aggregate anything at all, so multiple updates on the same record on the same transaction would
+    # be notified. Also, if the same record is deleted on the same transaction it was created, this would end up
+    # triggering both `on_insert` and `on_delete` callbacks. This strategy should usually be avoided.
+    #
     # These strategies can be defined per transaction, and by default it will uses the memory one, and only fallback
     # to the temporary table if the transaction size is roughly 2 gigabytes or more.
     def aggregation_strategy(begin_transaction_event)
@@ -127,6 +131,8 @@ module Wal
           MemoryRecordWatcher.new(self)
         when :temporary_table
           TemporaryTableRecordWatcher.new(self)
+        when :none
+          NoAggregationWatcher.new(self)
         else
           raise "Invalid aggregation strategy: #{strategy}"
         end
@@ -134,6 +140,21 @@ module Wal
       @current_record_watcher.on_event(event)
     end
 
+    class NoAggregationWatcher
+      include Wal::Watcher
+
+      def initialize(watcher)
+        @watcher = watcher
+      end
+
+      def on_event(event)
+        case event
+        when InsertEvent, UpdateEvent, DeleteEvent
+          @watcher.on_record_changed(event)
+        end
+      end
+    end
+
     class MemoryRecordWatcher
       include Wal::Watcher
       include Wal::Watcher::SeparatedEvents

data/lib/wal/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Wal
-  VERSION = "0.0.14"
+  VERSION = "0.0.16"
 end

data/rbi/wal.rbi

@@ -7,7 +7,7 @@ module Wal
       UpdateEvent,
       DeleteEvent,
     ) }
-  VERSION = "0.0.14"
+  VERSION = "0.0.16"
 
   class BeginTransactionEvent < T::Struct
     prop :transaction_id, Integer, immutable: true
@@ -128,7 +128,7 @@ module Wal
     def self.on_save(table, changed: nil, &block); end
 
     sig { params(table: T.any(String, T.class_of(::ActiveRecord::Base)), block: T.proc.bind(T.attached_class).params(event: Wal::DeleteEvent).void).void }
-    def self.on_destroy(table, &block); end
+    def self.on_delete(table, &block); end
 
     sig { params(event: RecordEvent).void }
     def on_record_changed(event); end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: wal
 version: !ruby/object:Gem::Version
-  version: 0.0.14
+  version: 0.0.16
 platform: ruby
 authors:
 - Rodrigo Navarro
 bindir: exe
 cert_chain: []
-date: 2025-09-14 00:00:00.000000000 Z
+date: 2025-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -104,7 +104,6 @@ files:
 - lib/wal/version.rb
 - lib/wal/watcher.rb
 - rbi/wal.rbi
-- sig/wal.rbs
 licenses:
 - MIT
 metadata:

data/sig/wal.rbs

@@ -1,186 +0,0 @@
-# typed: strong
-module Wal
-end
-
-Wal::Event: untyped
-
-Wal::VERSION: untyped
-
-class Wal::BeginTransactionEvent < T::Struct
-  def estimated_size: () -> Integer
-end
-
-class Wal::CommitTransactionEvent < T::Struct
-end
-
-module Wal::ChangeEvent
-  def diff: () -> ::Hash[String, [ untyped, untyped ]]
-
-  def changed_attribute?: (Symbol | String attribute) -> bool
-
-  def attribute: (Symbol | String attribute) -> untyped
-
-  def attribute_changes: (Symbol | String attribute) -> [ untyped, untyped ]?
-
-  def attribute_was: (Symbol | String attribute) -> untyped
-end
-
-class Wal::InsertEvent < T::Struct
-  include ::Wal::ChangeEvent
-
-  def diff: () -> ::Hash[String, [ untyped, untyped ]]
-end
-
-class Wal::UpdateEvent < T::Struct
-  include ::Wal::ChangeEvent
-
-  def diff: () -> ::Hash[String, [ untyped, untyped ]]
-end
-
-class Wal::DeleteEvent < T::Struct
-  include ::Wal::ChangeEvent
-
-  def diff: () -> ::Hash[String, [ untyped, untyped ]]
-end
-
-module Wal::ActiveRecordContextExtension
-  def set_wal_watcher_context: (untyped context, ?prefix: untyped prefix) -> untyped
-end
-
-class Wal::NoopWatcher
-  include Wal::Watcher
-
-  def on_event: (Event event) -> void
-end
-
-class Wal::RecordWatcher
-  include Wal::Watcher
-
-  extend T::Helpers
-
-  def self.inherited: (untyped subclass) -> untyped
-
-  def self.on_insert: (String | singleton(::ActiveRecord::Base) table) { (InsertEvent event) -> void } -> void
-
-  def self.on_update: (String | singleton(::ActiveRecord::Base) table, ?changed: ::Array[String | Symbol]? changed) { (UpdateEvent event) -> void } -> void
-
-  def self.on_save: (String | singleton(::ActiveRecord::Base) table, ?changed: ::Array[String | Symbol]? changed) { (InsertEvent | UpdateEvent event) -> void } -> void
-
-  def self.on_destroy: (String | singleton(::ActiveRecord::Base) table) { (DeleteEvent event) -> void } -> void
-
-  def on_record_changed: (RecordEvent event) -> void
-
-  def should_watch_table?: (String table) -> bool
-
-  def aggregation_strategy: (BeginTransactionEvent event) -> Symbol
-
-  def on_event: (Event event) -> void
-end
-
-Wal::Wal::RecordWatcher::RecordEvent: untyped
-
-class Wal::Wal::RecordWatcher::MemoryRecordWatcher
-  include Wal::Watcher
-
-  include Wal::Watcher::SeparatedEvents
-
-  extend T::Helpers
-
-  def initialize: (untyped watcher) -> void
-
-  def on_begin: (BeginTransactionEvent event) -> void
-
-  def on_commit: (untyped _event) -> untyped
-
-  def on_insert: (InsertEvent event) -> void
-
-  def on_update: (UpdateEvent event) -> void
-
-  def on_delete: (DeleteEvent event) -> void
-end
-
-Wal::Wal::RecordWatcher::Wal::Wal::RecordWatcher::MemoryRecordWatcher::RecordsStorage: untyped
-
-class Wal::Wal::RecordWatcher::TemporaryTableRecordWatcher
-  include Wal::Watcher
-
-  include Wal::Watcher::SeparatedEvents
-
-  extend T::Helpers
-
-  def initialize: (untyped watcher, ?batch_size: untyped batch_size) -> void
-
-  def on_begin: (BeginTransactionEvent event) -> void
-
-  def on_commit: (untyped _event) -> untyped
-
-  def on_insert: (InsertEvent event) -> void
-
-  def on_update: (UpdateEvent event) -> void
-
-  def on_delete: (DeleteEvent event) -> void
-
-  def base_class: () -> singleton(::ActiveRecord::Base)
-
-  def serialize: (untyped event) -> untyped
-
-  def deserialize: (untyped persisted_event) -> untyped
-end
-
-class Wal::Replicator
-  include PG::Replication::Protocol
-
-  def initialize: (?replication_slot: String replication_slot, ?use_temporary_slot: bool use_temporary_slot, ?db_config: ::Hash[Symbol, untyped] db_config) -> void
-
-  def replicate_forever: (Watcher watcher, ?publications: ::Array[String] publications) -> void
-
-  def replicate: (Watcher watcher, ?publications: ::Array[String] publications) -> ::Enumerator::Lazy[Event]
-end
-
-class Wal::Wal::Replicator::Column < T::Struct
-  def decode: (untyped value) -> untyped
-end
-
-class Wal::Wal::Replicator::Table < T::Struct
-  def primary_key: (untyped decoded_row) -> untyped
-
-  def decode_row: (untyped values) -> untyped
-end
-
-class Wal::StreamingWatcher
-  include Wal::Watcher
-
-  extend T::Helpers
-
-  def on_transaction_events: (::Enumerator[Event] events) -> void
-
-  def queue_size: (BeginTransactionEvent event) -> Integer
-
-  def on_event: (Event event) -> void
-end
-
-module Wal::Watcher
-  include Wal
-
-  extend T::Helpers
-
-  def on_event: (Event event) -> void
-
-  def should_watch_table?: (String table) -> bool
-
-  def valid_context_prefix?: (String prefix) -> bool
-end
-
-module Wal::Wal::Watcher::SeparatedEvents
-  def on_event: (Event event) -> void
-
-  def on_begin: (BeginTransactionEvent event) -> void
-
-  def on_insert: (InsertEvent event) -> void
-
-  def on_update: (UpdateEvent event) -> void
-
-  def on_delete: (DeleteEvent event) -> void
-
-  def on_commit: (CommitTransactionEvent event) -> void
-end