waldit 0.0.4 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 81634f3ce5d00503e50882a14495a960dfc8fb1072b53c92e3cd2a08441666c9
-  data.tar.gz: b235c7ee87fda30c747db3a678a3b9f6c7b5f5b95861f998b037a63418bee29b
+  metadata.gz: e8e8ecb8b8fc2c3c4b855fb06b204d858b7cfbf24f12dc016b4c3f5ca10849a2
+  data.tar.gz: b72218c4df4b396e497f1aa9d96fd159305fabb7faf0d481392044536da5141a
 SHA512:
-  metadata.gz: ad16ce87f467600879c9358540143cb1ad4eeeb1fe278472647aad21e28db4420a5238b13e6d70c02a65014d72251d54acfdfb0fe6c448191644871a28d4ef45
-  data.tar.gz: e932e94f85304d7b8f756a5fce3a3ba0e2c07ca302766e0f8ff3563b4acac9e14b7943ae0bcc3030ee68567571315e635fa172c02c945e70d68e45c46fba4d41
+  metadata.gz: 5471e11fa41cb7c3576dba2e0c1095ee3b2a7b7fd4eda88cd46e10523cf54ecdcda5a2d3470fdb9e97f3aebc066ce84bba7aa63be8507a0f0e37927b6604520f
+  data.tar.gz: 11dd782e95115cb77e8df0bed87f4dd8422454f5ef6ce84c35b9fe39af931c9855b37b07685cb3833ace964757c7d81b3d64a55a23330b52177ada7b60dd7ea8

data/lib/waldit/record.rb

@@ -2,8 +2,22 @@
 
 module Waldit
   module Record
+    def new
+      return self[:new] if self[:new]
+
+      (self[:diff] || {}).transform_values { |_old, new| new }
+    end
+
+    def old
+      return self[:old] if self[:old]
+
+      (self[:diff] || {}).transform_values { |old, _new| old }
+    end
+
     def diff
-      (old.keys | new.keys).reduce({}.with_indifferent_access) do |diff, key|
+      return self[:diff] if self[:diff]
+
+      (old.keys | new.keys).reduce({}) do |diff, key|
         old[key] != new[key] ? diff.merge(key => [old[key], new[key]]) : diff
       end
     end

data/lib/waldit/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Waldit
-  VERSION = "0.0.4"
+  VERSION = "0.0.6"
 end

data/lib/waldit/watcher.rb

@@ -9,85 +9,86 @@ module Waldit
     def audit_event(event)
       return unless event.primary_key
 
-      audit = {
-        transaction_id: event.transaction_id,
-        lsn: event.lsn,
-        context: event.context,
-        table_name: event.table,
-        primary_key: event.primary_key,
-      }
-
-      unique_by = %i[table_name primary_key transaction_id]
+      audit = [event.transaction_id, event.lsn, event.table, event.primary_key, event.context.to_json]
 
       case event
       when InsertEvent
-        record.upsert(
-          audit.merge(action: "insert", new: event.new),
-          unique_by:,
-          on_duplicate: :update,
-        )
+        @connection.exec_prepared("waldit_insert", audit + [event.new.to_json])
 
       when UpdateEvent
         return if event.diff.without(ignored_columns(event.table)).empty?
-        record.upsert(
-          audit.merge(action: "update", old: event.old, new: event.new),
-          unique_by:,
-          on_duplicate: :update,
-          update_only: %w[new],
-        )
+        @connection.exec_prepared("waldit_update", audit + [event.old.to_json, event.new.to_json])
 
       when DeleteEvent
-        case record.where(audit.slice(*unique_by)).pluck(:action, :old).first
-        in ["insert", _]
-          # We are deleting a record that was inserted on this transaction, which means we don't need to audit anything,
-          # as the record was never commited
-          record.where(audit.slice(*unique_by)).delete_all
-
-        in ["update", old]
-          # We are deleting a record we updated on this transaction. Here we are making sure we keep the correct previous
-          # state, and not the state at the moment of the deletion
-          record.upsert(
-            audit.merge(action: "delete", old:, new: {}),
-            unique_by:,
-            on_duplicate: :update,
-          )
-
-        in ["delete", _]
-          # This should never happend, we wouldn't be able to delete a record that was already deleted on this transaction
-
+        case @connection.exec_prepared("waldit_delete_cleanup", [event.transaction_id, event.table, event.primary_key]).values
+        in [["update", previous_old]]
+          @connection.exec_prepared("waldit_delete", audit + [previous_old])
+        in []
+          @connection.exec_prepared("waldit_delete", audit + [event.old.to_json])
         else
-          # Finally the most common case: just deleting a record not created or updated on this transaction
-          record.upsert(
-            audit.merge(action: "delete", old: event.old),
-            unique_by:,
-            on_duplicate: :update,
-          )
+          # Don't need to audit anything on this case
         end
       end
     end
 
     def on_transaction_events(events)
-      counter = 0
-      catch :finish do
-        loop do
-          record.transaction do
-            events.each do |event|
-              case event
-              when CommitTransactionEvent
-                record
-                  .where(transaction_id: event.transaction_id)
-                  .update_all(commited_at: event.timestamp) if counter > 0
-                # Using throw to break the outside loop and finish the thread gracefully
-                throw :finish
-
-              when InsertEvent, UpdateEvent, DeleteEvent
-                audit_event(event)
-
-                counter += 1
-                # We break here to force a commit, so we don't keep a single big transaction pending
-                break if counter % max_transaction_size == 0
-              end
+      record.transaction do
+        @connection = record.connection.raw_connection
+        tables = Set.new
+        insert_prepared = false
+        update_prepared = false
+        delete_prepared = false
+
+        events.each do |event|
+          case event
+          when CommitTransactionEvent
+            record.where(transaction_id: event.transaction_id).update_all(commited_at: event.timestamp)
+
+            log_new = tables.filter { |table| Waldit.store_changes.call(table).include? :new }
+            log_old = tables.filter { |table| Waldit.store_changes.call(table).include? :old }
+            log_diff = tables.filter { |table| Waldit.store_changes.call(table).include? :diff }
+            record.where(transaction_id: event.transaction_id).update_all(<<~SQL)
+              new = CASE WHEN table_name = ANY (ARRAY[#{log_new.map { |table| "'#{table}'" }.join(",")}]::varchar[]) THEN new ELSE null END,
+              old = CASE WHEN table_name = ANY (ARRAY[#{log_old.map { |table| "'#{table}'" }.join(",")}]::varchar[]) THEN old ELSE null END,
+              diff =
+                CASE WHEN table_name = ANY (ARRAY[#{log_diff.map { |table| "'#{table}'" }.join(",")}]::varchar[]) THEN (
+                  SELECT
+                    jsonb_object_agg(
+                      coalesce(old_kv.key, new_kv.key),
+                      jsonb_build_array(old_kv.value, new_kv.value)
+                    )
+                  FROM jsonb_each(old) AS old_kv
+                  FULL OUTER JOIN jsonb_each(new) AS new_kv ON old_kv.key = new_kv.key
+                  WHERE old_kv.value IS DISTINCT FROM new_kv.value
+                )
+                ELSE null
+                END
+            SQL
+
+          when InsertEvent
+            tables << event.table
+            unless insert_prepared
+              prepare_insert
+              insert_prepared = true
+            end
+            audit_event(event)
+
+          when UpdateEvent
+            tables << event.table
+            unless update_prepared
+              prepare_update
+              update_prepared = true
             end
+            audit_event(event)
+
+          when DeleteEvent
+            tables << event.table
+            unless delete_prepared
+              prepare_delete
+              prepare_delete_cleanup
+              delete_prepared = true
+            end
+            audit_event(event)
           end
         end
       end
@@ -112,5 +113,46 @@ module Waldit
     def record
       Waldit.model
     end
+
+    private
+
+    def prepare_insert
+      @connection.prepare("waldit_insert", <<~SQL)
+        INSERT INTO #{record.table_name} (transaction_id, lsn, table_name, primary_key, action, context, new)
+        VALUES ($1, $2, $3, $4, 'insert'::waldit_action, $5, $6)
+        ON CONFLICT (table_name, primary_key, transaction_id)
+        DO UPDATE SET new = #{record.table_name}.new
+      SQL
+    end
+
+    def prepare_update
+      @connection.prepare("waldit_update", <<~SQL)
+        INSERT INTO #{record.table_name} (transaction_id, lsn, table_name, primary_key, action, context, old, new)
+        VALUES ($1, $2, $3, $4, 'update'::waldit_action, $5, $6, $7)
+        ON CONFLICT (table_name, primary_key, transaction_id)
+        DO UPDATE SET new = excluded.new
+      SQL
+    end
+
+    def prepare_delete
+      @connection.prepare("waldit_delete", <<~SQL)
+        INSERT INTO #{record.table_name} (transaction_id, lsn, table_name, primary_key, action, context, old)
+        VALUES ($1, $2, $3, $4, 'delete'::waldit_action, $5, $6)
+        ON CONFLICT (table_name, primary_key, transaction_id)
+        DO UPDATE SET old = #{record.table_name}.old
+      SQL
+    end
+
+    def prepare_delete_cleanup
+      @connection.prepare("waldit_delete_cleanup", <<~SQL)
+        DELETE FROM #{record.table_name}
+        WHERE
+          transaction_id = $1
+          AND table_name = $2
+          AND primary_key = $3
+          AND action IN ('insert'::waldit_action, 'update'::waldit_action)
+        RETURNING action, old
+      SQL
+    end
   end
 end

data/lib/waldit.rb

@@ -21,6 +21,21 @@ module Waldit
       end
     end
 
+    attr_reader :store_changes
+
+    def store_changes=(changes)
+      case changes
+      when Symbol
+        changes = [changes].to_set
+        @store_changes = -> { changes }
+      when Array
+        changes = changes.map(&:to_sym).to_set
+        @store_changes = -> { changes }
+      else
+        @store_changes = changes
+      end
+    end
+
     attr_accessor :ignored_columns
     attr_accessor :max_transaction_size
     attr_accessor :model
@@ -36,6 +51,8 @@ module Waldit
 
     config.watched_tables = -> table { table != "waldit" }
 
+    config.store_changes = -> table { %i[old new diff] }
+
     config.ignored_columns = -> table { %w[created_at updated_at] }
 
     config.max_transaction_size = 10_000

data/rbi/waldit.rbi

@@ -2,7 +2,7 @@
 module Waldit
   extend T::Sig
   extend Waldit::Context
-  VERSION = "0.0.4"
+  VERSION = "0.0.6"
 
   class << self
     sig { returns(String) }
@@ -11,6 +11,9 @@ module Waldit
     sig { returns(T.proc.params(table: String).returns(T::Boolean)) }
     attr_reader :watched_tables
 
+    sig { returns(T.proc.params(table: String).returns(T::Array[Symbol])) }
+    attr_reader :store_changes
+
     sig { returns(T.proc.params(table: String).returns(T::Array[String])) }
     attr_accessor :ignored_columns
 
@@ -24,6 +27,9 @@ module Waldit
   sig { params(tables: T.any(T::Array[String], T.proc.params(table: String).returns(T::Boolean))).void }
   def self.watched_tables=(tables); end
 
+  sig { params(changes: T.any(Symbol, T::Array[Symbol], T.proc.params(table: String).returns(T::Array[Symbol]))).void }
+  def self.store_changes=(changes); end
+
   sig { params(block: T.proc.params(config: T.class_of(Waldit)).void).void }
   def self.configure(&block); end
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: waldit
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.6
 platform: ruby
 authors:
 - Rodrigo Navarro
 bindir: exe
 cert_chain: []
-date: 2025-08-02 00:00:00.000000000 Z
+date: 2025-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: wal
@@ -71,7 +71,6 @@ files:
 - lib/waldit/version.rb
 - lib/waldit/watcher.rb
 - rbi/waldit.rbi
-- sig/waldit.rbs
 homepage: https://github.com/reu/waldit
 licenses:
 - MIT

data/sig/waldit.rbs

@@ -1,73 +0,0 @@
-# typed: strong
-module Waldit
-  extend Waldit::Context
-
-  def self.watched_tables=: () -> String
-                          | () -> ^(String table) -> bool
-                          | () -> ^(String table) -> ::Array[String]
-                          | () -> Integer
-                          | () -> singleton(ActiveRecord::Base)
-                          | (::Array[String] | ^(String table) -> bool tables) -> void
-
-  def self.configure: () { (singleton(Waldit) config) -> void } -> void
-end
-
-Waldit::VERSION: untyped
-
-module Waldit::Context
-  def with_context: [U] (Context context) { () -> U } -> U
-
-  def context: () -> Context?
-
-  def add_context: (Context added_context) -> void
-
-  def new_context: (?Context context) -> void
-end
-
-Waldit::Waldit::Context::Context: untyped
-
-class Waldit::Railtie < Rails::Railtie
-end
-
-module Waldit::Record
-  extend T::Helpers
-
-  def new: () -> ::Hash[String | Symbol, untyped]
-
-  def old: () -> ::Hash[String | Symbol, untyped]
-
-  def diff: () -> ::Hash[String | Symbol, [ untyped, untyped ]]
-end
-
-module Waldit::Sidekiq
-end
-
-class Waldit::Waldit::Sidekiq::SaveContext
-  include ::Sidekiq::ClientMiddleware
-
-  def call: (untyped job_class, untyped job, untyped queue, untyped redis) -> untyped
-end
-
-class Waldit::Waldit::Sidekiq::LoadContext
-  include ::Sidekiq::ServerMiddleware
-
-  def call: (untyped job_instance, untyped job, untyped queue) { () -> untyped } -> untyped
-
-  def deserialize_context: (untyped job) -> untyped
-end
-
-class Waldit::Watcher < Wal::StreamingWatcher
-  def audit_event: (InsertEvent | UpdateEvent | DeleteEvent event) -> void
-
-  def on_transaction_events: (::Enumerator[Event] events) -> void
-
-  def should_watch_table?: (String table) -> bool
-
-  def valid_context_prefix?: (String prefix) -> bool
-
-  def ignored_columns: (String table) -> ::Array[String]
-
-  def max_transaction_size: () -> Integer
-
-  def record: () -> singleton(ActiveRecord::Base)
-end