waldit 0.0.25 → 0.0.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ba555412a6c079daefae27957b707136b94b086ec4fb351c1243e7efb14470d0
-  data.tar.gz: 189649ec353443111a932cb10fbe6e3beb59efee8550453bb282da148a1d8858
+  metadata.gz: 036035f234a35b9fd2c2c352a361a8826b54d0a0c6a86d93946c0b192523ea9d
+  data.tar.gz: d655ad860384e3e1c2dc5a8a6b3ac84f320c6210f14fcb53510cb016b1bf6e0d
 SHA512:
-  metadata.gz: 2839fd6125953177e69c733438fb042a40364b236712f6e2a8d8c0c2490b4917d82be47d265422de4a19cad27c4ce58189fc922d0d28288a476775e85e119a20
-  data.tar.gz: b49559cc7f60ab7b889fa3e183039214494f40dadc1ca50e00c006e8f62704d7b8ca709719ded76bf6c948835a9705617c7b673988ce97edd1e39fa332206bef
+  metadata.gz: c4dac68e52cb6ba0a72809f09fcffdc6425e6b154fe91d1b5c5f94e4bf3ea349fccf0ea43327b69cd2b94d968b832136c8fea9bfe5dc1a04b21ba50434e8195a
+  data.tar.gz: a10904c3cd2df88460cd83be6604edc6c4bbeb8a66e1e44a47966e40eb00c4222a1de3d1720055847a516ef0513c14109bd879fbd35909244db145f02f5e828e

data/README.md

@@ -1,117 +1,231 @@
 # Waldit
 
-[![Gem Version](https://badge.fury.io/rb/waldit.svg)](https://badge.fury.io/rb/waldit)
+Waldit is a Postgres-based audit trail for Rails.
 
-Waldit is a Ruby gem that provides a simple and extensible way to audit changes to your ActiveRecord models. It leverages PostgreSQL's logical replication capabilities to capture changes directly from your database with 100% consistency.
+It hooks into [Postgres logical replication](https://www.postgresql.org/docs/current/logical-replication.html) via the [`wal`](https://github.com/reu/wal) gem to capture every `insert`, `update`, and `delete` directly from the WAL. Unlike ActiveRecord callbacks, these events are guaranteed by Postgres to be 100% consistent -- even changes that bypass Rails entirely are captured.
 
-## Features
+## Getting started
 
-- **Automatic Auditing:** Automatically track `create`, `update`, and `delete` operations on your models.
-- **Contextual Auditing:** Add custom context to your audit records to understand who made the change and why.
-- **Flexible Configuration:** Configure which tables and columns to watch, and how to store audit information.
-- **High Performance:** Built on top of [`wal`](https://github.com/reu/wal), which uses PostgreSQL's logical replication for minimal overhead.
+### Installation
 
-## Installation
-
-Add this line to your application's Gemfile:
+Add `waldit` to your application's Gemfile:
 
 ```ruby
 gem "waldit"
 ```
 
-And then execute:
+### Database adapter
+
+Waldit ships a custom database adapter that injects audit context into your transactions. Update your `config/database.yml`:
+
+```yaml
+default: &default
+  adapter: waldit
+  # ... rest of your config
+```
+
+### Migrations
+
+Waldit provides migration helpers. First, create the audit table and publication:
+
+```ruby
+class SetupWaldit < ActiveRecord::Migration[7.0]
+  def change
+    create_waldit_table
+    create_waldit_publication
+  end
+end
+```
+
+Then, for each table you want to audit:
+
+```ruby
+class AuditUsers < ActiveRecord::Migration[7.0]
+  def change
+    add_table_to_waldit :users
+  end
+end
+```
+
+This sets `REPLICA IDENTITY FULL` on the table and adds it to the Waldit publication.
 
-    $ bundle
+### Running the watcher
 
-Or install it yourself as:
+Create a `config/waldit.yml`:
 
-    $ gem install waldit
+```yaml
+slots:
+  audit:
+    publications: [waldit_publication]
+    watcher: Waldit::Watcher
+```
 
-## Usage
+Then start the process:
 
-1.  **Configure your database adapter:**
+```bash
+bundle exec wal start config/waldit.yml
+```
 
-    First step is to configure in your `config/database.yml` and change your adapter to `waldit`, which is a special adapter that allows injecting `waldit` contextual information on your transactions:
+That's it. Every change to your audited tables is now being recorded.
 
-    ```yaml
-    default: &default
-      adapter: waldit
-      # ...
-    ```
+## Adding context
 
-2.  **Create an audit table:**
+Wrap your operations with `Waldit.with_context` to record who made the change and why:
 
-    Generate a migration to create the `waldit` table:
+```ruby
+Waldit.with_context(user_id: current_user.id, reason: "Profile update") do
+  user.update(name: "New Name")
+end
+```
 
-    ```bash
-    rails generate migration create_waldit
-    ```
+Context can be nested and updated mid-transaction:
 
-    And then add the following to your migration file:
+```ruby
+Waldit.with_context(user_id: current_user.id) do
+  user.update(name: "New Name")
 
-    ```ruby
-    class CreateWalditTable < ActiveRecord::Migration[7.0]
-      def change
-        create_table :waldit do |t|
-          t.bigint :transaction_id, null: false
-          t.bigint :lsn, null: false
-          t.string :action, null: false
-          t.jsonb :context, default: {}
-          t.string :table_name, null: false
-          t.string :primary_key, null: false
-          t.jsonb :old, default: {}
-          t.jsonb :new, default: {}
-          t.timestamp :commited_at
+  Waldit.with_context(via: "admin_panel") do
+    account.update(plan: "premium")  # context: { user_id: 1, via: "admin_panel" }
+  end
 
-          t.index [:table_name, :primary_key, :transaction_id], unique: true
-        end
-      end
-    end
-    ```
+  Waldit.add_context(batch: true)
+  other_user.update(name: "Other")  # context: { user_id: 1, batch: true }
+end
+```
 
-3.  **Configure Waldit:**
+### Sidekiq integration
 
-    Create an initializer file at `config/initializers/waldit.rb`:
+Waldit can propagate context into background jobs:
 
-    ```ruby
-    Waldit.configure do |config|
-      # A callback that returns true if a table should be watched.
-      config.watched_tables = ->(table) { table != "waldit" }
+```ruby
+# config/initializers/sidekiq.rb
+Sidekiq.configure_client do |config|
+  config.client_middleware do |chain|
+    chain.add Waldit::Sidekiq::SaveContext
+  end
+end
+
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add Waldit::Sidekiq::LoadContext
+  end
+end
+```
 
-      # A callback that returns an array of columns to ignore for a given table.
-      config.ignored_columns = ->(table) { %w[created_at updated_at] }
-    end
-    ```
+## Querying the audit trail
 
-4.  **Add context to your changes:**
+Waldit provides scopes on the audit model:
 
-    Use the `with_context` method to add context to your database operations:
+```ruby
+# All audit records for a specific record
+Waldit.model.for(user)
 
-    ```ruby
-    Waldit.with_context(user_id: 1, reason: "User updated their profile") do
-      user.update(name: "New Name")
-    end
-    ```
+# All audit records for a table
+Waldit.model.from_model(User)
 
-5.  **Start the watcher:**
+# All audit records with a specific context
+Waldit.model.with_context(user_id: 1)
+```
+
+Each audit record exposes:
+
+```ruby
+audit = Waldit.model.for(user).last
+
+audit.action       # "insert", "update", or "delete"
+audit.old          # previous attributes (updates and deletes)
+audit.new          # new attributes (inserts and updates)
+audit.diff         # changed attributes as { "name" => ["old", "new"] }
+audit.context      # the context hash
+audit.committed_at # when the transaction was committed
+audit.primary_key  # the record's primary key
+```
+
+The `old`, `new`, and `diff` accessors are smart -- if you only store `:diff`, calling `.old` or `.new` will compute the values from the diff, and vice versa.
+
+## Configuration
+
+```ruby
+# config/initializers/waldit.rb
+Waldit.configure do |config|
+  # Which tables to watch (default: all except "waldit")
+  config.watched_tables = -> table { table != "waldit" }
+
+  # Columns to exclude from audit records (default: created_at, updated_at)
+  config.ignored_columns = -> table { %w[created_at updated_at] }
+
+  # What to store per table (default: [:old, :new])
+  # Options: :old, :new, :diff (any combination)
+  config.store_changes = [:old, :new]
+
+  # WAL byte threshold for switching to streaming mode (default: 10MB)
+  # Transactions smaller than this are processed in memory for better performance
+  config.large_transaction_threshold = 10_000_000
+end
+```
+
+### Storage policies
+
+By default, Waldit stores both `old` and `new` attributes for every change. You can reduce storage by only keeping what you need:
+
+```ruby
+# Only store diffs for updates (most compact)
+config.store_changes = :diff
+
+# Per-table policies
+config.store_changes = -> table {
+  case table
+  when "events" then [:new]
+  when "logs"   then [:diff]
+  else               [:old, :new]
+  end
+}
+```
+
+### Per-table ignored columns
+
+```ruby
+config.ignored_columns = -> table {
+  case table
+  when "users" then %w[created_at updated_at last_sign_in_at]
+  else              %w[created_at updated_at]
+  end
+}
+```
+
+### Custom audit model
+
+You can provide your own model class if you need custom methods or a different table name:
+
+```ruby
+class AuditRecord < ApplicationRecord
+  include Waldit::Record
+  self.table_name = "waldit"
+end
+
+Waldit.configure do |config|
+  config.model = AuditRecord
+end
+```
 
-    To process the events, you need to start a WAL watcher. The recommended way is to have a config/waldit.yml
+## How it works
 
-    ```yml
-    slots:
-      audit:
-        publications: [waldit_publication]
-        watcher: Waldit::Watcher
-    ```
+Waldit uses Postgres logical replication to stream changes from the WAL (Write-Ahead Log). The flow is:
 
-    And then run:
+1. The custom database adapter sets a `waldit_context` session variable before each write operation
+2. Postgres captures the change and the context in the WAL
+3. `Waldit::Watcher` receives the events via a replication slot
+4. Events are deduplicated per-transaction (multiple updates to the same record produce a single audit entry)
+5. The final audit records are persisted to the `waldit` table
 
-    ```bash
-    bundle exec wal start config/waldit.yml
-    ```
+For small transactions, events are accumulated in memory and persisted in a single batch insert. For large transactions (configurable via `large_transaction_threshold`), events are streamed and persisted individually to avoid memory pressure.
 
-## How it Works
+### Transaction-level deduplication
 
-Waldit uses a custom PostgreSQL adapter to set the `waldit_context` session variable before each transaction. This context is then captured by the logical replication slot and stored in the `waldit` table by the `Waldit::Watcher`.
+Within a single database transaction, Waldit collapses events intelligently:
 
-The `Waldit::Watcher` is a streaming watcher that listens for changes in the logical replication slot and creates audit records in the `waldit` table. It processes events in batches to minimize the number of database transactions.
+- **Insert then update** -- recorded as a single `insert` with the final state
+- **Multiple updates** -- recorded as a single `update` with the original `old` and final `new`
+- **Insert then delete** -- not recorded (the record never existed outside the transaction)
+- **Update then delete** -- recorded as a `delete` with the original `old` values
+- **Update that reverts to original** -- not recorded (no net change)

data/lib/waldit/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Waldit
-  VERSION = "0.0.25"
+  VERSION = "0.0.26"
 end

data/lib/waldit/watcher.rb

@@ -6,46 +6,204 @@ module Waldit
   class Watcher < Wal::StreamingWatcher
     include Wal
 
-    def audit_event(event)
-      return unless event.primary_key
-      primary_key = event.primary_key.to_json
+    def initialize(*)
+      super
+      initialize_connection
+      @retry = false
+    end
 
-      audit = [event.transaction_id, event.lsn, event.table, primary_key, event.context.to_json]
+    def on_transaction_events(events)
+      begin_event = events.next
+      if begin_event.estimated_size < Waldit.large_transaction_threshold
+        process_in_memory(events)
+      else
+        process_streaming(events)
+      end
+    rescue PG::ConnectionBad
+      raise if @retry
+      initialize_connection
+      @retry = true
+      retry
+    end
 
-      case event
-      when InsertEvent
-        new_attributes = clean_attributes(event.table, event.new)
-        @connection.exec_prepared("waldit_insert", audit + [new_attributes.to_json])
-        true
+    def should_watch_table?(table)
+      Waldit.watched_tables.call(table)
+    end
 
-      when UpdateEvent
-        return if event.diff.without(ignored_columns(event.table)).empty?
-        old_attributes = clean_attributes(event.table, event.old)
-        new_attributes = clean_attributes(event.table, event.new)
+    def valid_context_prefix?(prefix)
+      prefix == Waldit.context_prefix
+    end
 
-        @connection.exec_prepared("waldit_update", audit + [old_attributes.to_json, new_attributes.to_json])
-        true
+    def ignored_columns(table)
+      (@ignored_columns_cache ||= {})[table] ||= Waldit.ignored_columns.call(table)
+    end
 
-      when DeleteEvent
-        case @connection.exec_prepared("waldit_delete_cleanup", [event.transaction_id, event.table, primary_key]).values
-        in [["update", previous_old]]
-          @connection.exec_prepared("waldit_delete", audit + [previous_old])
-        in []
-          @connection.exec_prepared("waldit_delete", audit + [clean_attributes(event.table, event.old).to_json])
-        else
-          # Don't need to audit anything on this case
+    def store_changes(table)
+      (@store_changes_cache ||= {})[table] ||= Waldit.store_changes.call(table)
+    end
+
+    def clean_attributes(table, attributes)
+      attributes.without(ignored_columns(table))
+    end
+
+    def record
+      Waldit.model
+    end
+
+    private
+
+    COLUMNS = %w[
+      transaction_id
+      lsn
+      table_name
+      primary_key
+      action
+      context
+      committed_at
+      old
+      new
+      diff
+    ].freeze
+
+    PARAMS_PER_ROW = COLUMNS.size
+    MAX_ROWS_PER_BATCH = 65535 / PARAMS_PER_ROW
+
+    def process_in_memory(events)
+      records = {}
+
+      events.each do |event|
+        case event
+        when InsertEvent
+          next unless event.primary_key
+          key = [event.full_table_name, event.primary_key.to_json]
+          records[key] = event
+
+        when UpdateEvent
+          next unless event.primary_key
+          next if event.diff.without(ignored_columns(event.table)).empty?
+          key = [event.full_table_name, event.primary_key.to_json]
+          records[key] = case (existing_event = records[key])
+          when InsertEvent
+            # A record inserted on this transaction is being updated, which means it should still reflect as a insert
+            # event, we just change the information to reflect the most current data that was just updated.
+            existing_event.with(new: event.new)
+          when UpdateEvent
+            # We are updating again a event that was already updated on this transaction.
+            # Same as the insert, we keep the old data from the previous update and the new data from the new one.
+            existing_event.with(new: event.new)
+          else
+            event
+          end
+
+        when DeleteEvent
+          next unless event.primary_key
+          key = [event.full_table_name, event.primary_key.to_json]
+          records[key] = case (existing_event = records[key])
+          when InsertEvent
+            # We are removing a record that was inserted on this transaction, we should not even report this change, as
+            # this record never existed outside this transaction anyways.
+            nil
+          when UpdateEvent
+            # Deleting a record that was previously updated by this transaction. Just store the previous data while
+            # keeping the record as deleted.
+            event.with(old: existing_event.old)
+          else
+            event
+          end
+
+        when CommitTransactionEvent
+          rows = records.compact.values.filter_map do |evt|
+            table = evt.full_table_name
+            store = store_changes(table)
+
+            rec = {
+              committed_at: event.timestamp,
+              transaction_id: event.transaction_id,
+              lsn: evt.lsn,
+              table_name: table,
+              primary_key: evt.primary_key.to_json,
+              context: evt.context,
+            }
+
+            case evt
+            when InsertEvent
+              { **rec, action: "insert", new: clean_attributes(table, evt.new) }
+            when UpdateEvent
+              rec = {
+                **rec,
+                action: "update",
+                old: evt.old&.then { |attrs| clean_attributes(table, attrs) } || {},
+                new: evt.new&.then { |attrs| clean_attributes(table, attrs) } || {},
+              }
+              next if rec[:old] == rec[:new]
+              rec[:old] = nil unless store.include? :old
+              rec[:new] = nil unless store.include? :new
+              rec[:diff] = clean_attributes(table, evt.diff) if store.include? :diff
+              rec
+            when DeleteEvent
+              { **rec, action: "delete", old: clean_attributes(table, evt.old) }
+            end
+          end
+
+          unless rows.empty?
+            if rows.size <= MAX_ROWS_PER_BATCH
+              insert_batch(rows)
+            else
+              @connection.transaction do
+                rows.each_slice(MAX_ROWS_PER_BATCH) { |batch| insert_batch(batch) }
+              end
+            end
+          end
+
+          @retry = false
         end
-        true
       end
     end
 
-    def initialize(*)
-      super
-      initialize_connection
-      @retry = false
+    def insert_batch(batch)
+      rows = batch.each_with_index.map do |_, i|
+        o = i * PARAMS_PER_ROW
+        row = [
+          "$#{o + 1}",
+          "$#{o + 2}",
+          "$#{o + 3}",
+          "$#{o + 4}",
+          "$#{o + 5}::waldit_action",
+          "$#{o + 6}::jsonb",
+          "$#{o + 7}",
+          "$#{o + 8}::jsonb",
+          "$#{o + 9}::jsonb",
+          "$#{o + 10}::jsonb",
+        ].join(",")
+        "(#{row})"
+      end
+
+      params = batch.flat_map do |r|
+        [
+          r[:transaction_id],
+          r[:lsn],
+          r[:table_name],
+          r[:primary_key],
+          r[:action],
+          r[:context]&.to_json,
+          r[:committed_at],
+          r[:old]&.to_json,
+          r[:new]&.to_json,
+          r[:diff]&.to_json,
+        ]
+      end
+
+      @connection.exec_params(<<~SQL, params)
+        INSERT INTO #{record.table_name} (#{COLUMNS.join(",")})
+        VALUES #{rows.join(",")}
+        ON CONFLICT (table_name, primary_key, transaction_id)
+        DO NOTHING
+      SQL
     end
 
-    def on_transaction_events(events)
+    def process_streaming(events)
+      ensure_streaming_statements_prepared
+
       @connection.transaction do
         tables = Set.new
 
@@ -54,7 +212,7 @@ module Waldit
           when CommitTransactionEvent
             unless tables.empty?
               changes = [:old, :new, :diff]
-                .map { |diff| [diff, tables.filter { |table| Waldit.store_changes.call(table).include? diff }] }
+                .map { |diff| [diff, tables.filter { |table| store_changes(table).include? diff }] }
                 .to_h
 
               log_new = (changes[:new] || []).map { |table| "#{table}" }
@@ -76,7 +234,6 @@ module Waldit
               ])
             end
 
-            # We sucessful retried a connection, let's reset our retry state
             @retry = false
 
           when InsertEvent
@@ -90,44 +247,57 @@ module Waldit
           end
         end
       end
-    rescue PG::ConnectionBad
-      raise if @retry
-      # Let's try to fetch a new connection and reprocess the transaction
-      initialize_connection
-      @retry = true
-      retry
     end
 
-    def should_watch_table?(table)
-      Waldit.watched_tables.call(table)
-    end
+    def audit_event(event)
+      return unless event.primary_key
+      primary_key = event.primary_key.to_json
+      table = event.full_table_name
 
-    def valid_context_prefix?(prefix)
-      prefix == Waldit.context_prefix
-    end
+      audit = [event.transaction_id, event.lsn, table, primary_key, event.context.to_json]
 
-    def ignored_columns(table)
-      Waldit.ignored_columns.call(table)
-    end
+      case event
+      when InsertEvent
+        new_attributes = clean_attributes(table, event.new)
+        @connection.exec_prepared("waldit_insert", audit + [new_attributes.to_json])
+        true
 
-    def clean_attributes(table, attributes)
-      attributes.without(ignored_columns(table))
-    end
+      when UpdateEvent
+        return if event.diff.without(ignored_columns(table)).empty?
+        old_attributes = clean_attributes(table, event.old)
+        new_attributes = clean_attributes(table, event.new)
 
-    def record
-      Waldit.model
-    end
+        @connection.exec_prepared("waldit_update", audit + [old_attributes.to_json, new_attributes.to_json])
+        true
 
-    private
+      when DeleteEvent
+        case @connection.exec_prepared("waldit_delete_cleanup", [event.transaction_id, table, primary_key]).values
+        in [["update", previous_old]]
+          @connection.exec_prepared("waldit_delete", audit + [previous_old])
+        in []
+          @connection.exec_prepared("waldit_delete", audit + [clean_attributes(table, event.old).to_json])
+        else
+          # Don't need to audit anything on this case
+        end
+        true
+      end
+    end
 
     def initialize_connection
       @connection = record.connection_pool.checkout.raw_connection
+      @streaming_statements_prepared = false
+    end
+
+    def ensure_streaming_statements_prepared
+      return if @streaming_statements_prepared
+
       prepare_insert
       prepare_update
       prepare_delete
       prepare_delete_cleanup
       prepare_finish
       prepare_cleanup
+      @streaming_statements_prepared = true
     end
 
     def prepare_insert

data/lib/waldit.rb

@@ -39,6 +39,7 @@ module Waldit
     attr_accessor :ignored_columns
     attr_accessor :model
     attr_accessor :context_prefix
+    attr_accessor :large_transaction_threshold
   end
 
   def self.configure(&block)
@@ -54,6 +55,8 @@ module Waldit
 
     config.ignored_columns = -> table { %w[created_at updated_at] }
 
+    config.large_transaction_threshold = 10_000_000
+
     config.model = Class.new(ActiveRecord::Base) do
       include Waldit::Record
       self.table_name = "waldit"

data/rbi/waldit.rbi

@@ -2,7 +2,7 @@
 module Waldit
   extend T::Sig
   extend Waldit::Context
-  VERSION = "0.0.25"
+  VERSION = "0.0.26"
 
   class << self
     sig { returns(String) }
@@ -19,6 +19,9 @@ module Waldit
 
     sig { returns(T.class_of(ActiveRecord::Base)) }
     attr_accessor :model
+
+    sig { returns(Integer) }
+    attr_accessor :large_transaction_threshold
   end
 
   sig { params(tables: T.any(T::Array[String], T.proc.params(table: String).returns(T::Boolean))).void }

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: waldit
 version: !ruby/object:Gem::Version
-  version: 0.0.25
+  version: 0.0.26
 platform: ruby
 authors:
 - Rodrigo Navarro
 bindir: exe
 cert_chain: []
-date: 2026-02-12 00:00:00.000000000 Z
+date: 2026-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: wal