wal 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cc5df8a8a67077b8c9a4da905a102e72bf4b1e6e41ad2569a4662038d81d123a
-  data.tar.gz: dc5dd65b5b87f43b0b1b4bc7d7d66b2b97fa92fd04963bbbdb32f2e781f4bcaf
+  metadata.gz: 3df0dd993bf343f90d54d4b60f52504ca2a84d4fd839c5c952e6f866b40a46cd
+  data.tar.gz: 8e8b568f3a26ed687e9010bd9c69ea4a656e6af59be20449a9142b61c61fe4ad
 SHA512:
-  metadata.gz: 0e1b415b21cef101a2493ed922b95e8e70e972575bb6bfc190200aee567a51a4582a26b29f07eca316621ae48ec4b63e995e718cec3a963ffd42fb6774c40ee1
-  data.tar.gz: 940d945bafe3f36dd303a427bbe25524f50a98ac88a76f2ef02e842cfe30b190ba87c047a15256c5b7f80408ee211bda08beb7b6bb37bb93325540fd5ad6e4e7
+  metadata.gz: '09591c7506a7b29d9261574f9a512f120862b58d08ae71d31501147076bbae7f9ab65c9c0ded4fe3c15ddfaa9ec1881ebebf934069c08847d8b486d4fcdfc6f7'
+  data.tar.gz: c09c1417b8021fde2890f9dd38d32cec9e64e9fec6d1341613b2a8c4522edd60c0ae34e78fdee3c45e8257c6be66d480fb8ad5f3d52fcec5fb0955d79c054a63

data/Rakefile

@@ -3,10 +3,3 @@
 require "bundler/gem_tasks"
 
 task(:test) { sh "bundle exec rspec" }
-
-task default: %i[build]
-
-task("sig/wal.rbi") { sh "bundle exec parlour" }
-task("rbi/wal.rbs") { sh "rbs prototype rbi rbi/wal.rbi > sig/wal.rbs" }
-
-Rake::Task["build"].enhance(["sig/wal.rbi", "rbi/wal.rbs"])

data/lib/wal/noop_watcher.rb

@@ -1,12 +1,8 @@
-# typed: strict
-
 module Wal
   # A watcher that does nothing. Just for performance testing in general. Useful in testing aswell.
   class NoopWatcher
-    extend T::Sig
     include Wal::Watcher
 
-    sig { override.params(event: Event).void }
     def on_event(event); end
   end
 end

data/lib/wal/record_watcher.rb

@@ -1,5 +1,3 @@
-# typed: true
-
 module Wal
   # Watcher that process records at the end of a transaction, keeping only its final state.
   #
@@ -39,12 +37,7 @@ module Wal
   # end
   # ```
   class RecordWatcher
-    extend T::Sig
-    extend T::Helpers
     include Wal::Watcher
-    abstract!
-
-    RecordEvent = T.type_alias { T.any(InsertEvent, UpdateEvent, DeleteEvent) }
 
     def self.inherited(subclass)
       super
@@ -52,53 +45,26 @@ module Wal
       @@delete_callbacks = Hash.new { |hash, key| hash[key] = [] }
     end
 
-    sig do
-      params(
-        table: T.any(String, T.class_of(::ActiveRecord::Base)),
-        block: T.proc.bind(T.attached_class).params(event: InsertEvent).void,
-      ).void
-    end
     def self.on_insert(table, &block)
       table = table.is_a?(String) ? table : table.table_name
       @@change_callbacks[table].push(only: [:create], block: block)
     end
 
-    sig do
-      params(
-        table: T.any(String, T.class_of(::ActiveRecord::Base)),
-        changed: T.nilable(T::Array[T.any(String, Symbol)]),
-        block: T.proc.bind(T.attached_class).params(event: UpdateEvent).void,
-      ).void
-    end
     def self.on_update(table, changed: nil, &block)
       table = table.is_a?(String) ? table : table.table_name
       @@change_callbacks[table].push(only: [:update], changed: changed&.map(&:to_s), block: block)
     end
 
-    sig do
-      params(
-        table: T.any(String, T.class_of(::ActiveRecord::Base)),
-        changed: T.nilable(T::Array[T.any(String, Symbol)]),
-        block: T.proc.bind(T.attached_class).params(event: T.any(InsertEvent, UpdateEvent)).void,
-      ).void
-    end
     def self.on_save(table, changed: nil, &block)
       table = table.is_a?(String) ? table : table.table_name
       @@change_callbacks[table].push(only: [:create, :update], changed: changed&.map(&:to_s), block: block)
     end
 
-    sig do
-      params(
-        table: T.any(String, T.class_of(::ActiveRecord::Base)),
-        block: T.proc.bind(T.attached_class).params(event: DeleteEvent).void,
-      ).void
-    end
     def self.on_destroy(table, &block)
       table = table.is_a?(String) ? table : table.table_name
       @@delete_callbacks[table].push(block: block)
     end
 
-    sig { params(event: RecordEvent).void }
     def on_record_changed(event)
       case event
       when InsertEvent
@@ -124,7 +90,6 @@ module Wal
       end
     end
 
-    sig { params(table: String).returns(T::Boolean) }
     def should_watch_table?(table)
       (@@change_callbacks.keys | @@delete_callbacks.keys).include? table
     end
@@ -139,7 +104,6 @@ module Wal
     #
     # These strategies can be defined per transaction, and by default it will uses the memory one, and only fallback
     # to the temporary table if the transaction size is roughly 2 gigabytes or more.
-    sig { params(event: BeginTransactionEvent).returns(Symbol) }
     def aggregation_strategy(event)
       if event.estimated_size > 1024.pow(3) * 2
         :temporary_table
@@ -148,7 +112,6 @@ module Wal
       end
     end
 
-    sig { override.params(event: Event).void }
     def on_event(event)
       if event.is_a? BeginTransactionEvent
         @current_record_watcher = case (strategy = aggregation_strategy(event))
@@ -164,21 +127,15 @@ module Wal
     end
 
     class MemoryRecordWatcher
-      extend T::Sig
-      extend T::Helpers
       include Wal::Watcher
       include Wal::Watcher::SeparatedEvents
 
-      # Records indexed by table and primary key
-      RecordsStorage = T.type_alias { T::Hash[[String, Integer], T.nilable(RecordEvent)] }
-
       def initialize(watcher)
         @watcher = watcher
       end
 
-      sig { params(event: BeginTransactionEvent).void }
       def on_begin(event)
-        @records = T.let({}, T.nilable(RecordsStorage))
+        @records = {}
       end
 
       def on_commit(_event)
@@ -188,7 +145,6 @@ module Wal
           &.each { |event| @watcher.on_record_changed(event) if event }
       end
 
-      sig { params(event: InsertEvent).void }
       def on_insert(event)
         if (id = event.primary_key)
           @records ||= {}
@@ -196,7 +152,6 @@ module Wal
         end
       end
 
-      sig { params(event: UpdateEvent).void }
       def on_update(event)
         if (id = event.primary_key)
           @records ||= {}
@@ -217,7 +172,6 @@ module Wal
         end
       end
 
-      sig { params(event: DeleteEvent).void }
       def on_delete(event)
         if (id = event.primary_key)
           @records ||= {}
@@ -240,8 +194,6 @@ module Wal
     end
 
     class TemporaryTableRecordWatcher
-      extend T::Sig
-      extend T::Helpers
       include Wal::Watcher
       include Wal::Watcher::SeparatedEvents
 
@@ -255,7 +207,6 @@ module Wal
         @batch_size = 5_000
       end
 
-      sig { params(event: BeginTransactionEvent).void }
       def on_begin(event)
         @table = begin
           table_name = "temp_record_watcher_#{SecureRandom.alphanumeric(10).downcase}"
@@ -276,8 +227,7 @@ module Wal
           base_class.connection.add_index table_name, unique_index, unique: true
 
           Class.new(base_class) do
-            # Using cast here since Sorbet bugs when we don't pass a explicit class to `Class.new`
-            T.cast(self, T.class_of(::ActiveRecord::Base)).table_name = table_name
+            self.table_name = table_name
 
             # All this sh$#1t was necessary because AR schema cache doesn't work with temporary tables...
             insert_all_class = Class.new(::ActiveRecord::InsertAll) do
@@ -290,8 +240,8 @@ module Wal
             define_singleton_method(:upsert) do |attributes, update_only: nil|
               insert_all_class
                 .new(
-                  T.cast(self, T.class_of(::ActiveRecord::Base)).none,
-                  T.cast(self, T.class_of(::ActiveRecord::Base)).connection,
+                  none,
+                  connection,
                   [attributes],
                   on_duplicate: :update,
                   unique_by: unique_index,
@@ -316,17 +266,14 @@ module Wal
         base_class.connection.drop_table @table.table_name
       end
 
-      sig { params(event: InsertEvent).void }
       def on_insert(event)
         @table.upsert(serialize(event))
       end
 
-      sig { params(event: UpdateEvent).void }
       def on_update(event)
         @table.upsert(serialize(event), update_only: %w[new])
       end
 
-      sig { params(event: DeleteEvent).void }
       def on_delete(event)
         case @table.where(table_name: event.table, primary_key: event.primary_key).pluck(:action, :old).first
         in ["insert", _]
@@ -342,7 +289,6 @@ module Wal
 
       private
 
-      sig { returns(T.class_of(::ActiveRecord::Base)) }
       def base_class
         self.class.base_active_record_class || ::ActiveRecord::Base
       end

data/lib/wal/replicator.rb

@@ -1,21 +1,11 @@
-# typed: false
-
 require "ostruct"
 
 module Wal
   # Responsible to hook into a Postgres logical replication slot and stream the changes to a specific `Watcher`.
   # Also it supports inject "contexts" into the replication events.
   class Replicator
-    extend T::Sig
     include PG::Replication::Protocol
 
-    sig do
-      params(
-        replication_slot: String,
-        use_temporary_slot: T::Boolean,
-        db_config: T::Hash[Symbol, T.untyped],
-      ).void
-    end
     def initialize(
       replication_slot:,
       use_temporary_slot: false,
@@ -26,7 +16,6 @@ module Wal
       @use_temporary_slot = use_temporary_slot
     end
 
-    sig { params(watcher: Watcher, publications: T::Array[String]).void }
     def replicate_forever(watcher, publications:)
       replication = replicate(watcher, publications:)
       loop { replication.next }
@@ -34,7 +23,6 @@ module Wal
       nil
     end
 
-    sig { params(watcher: Watcher, publications: T::Array[String]).returns(T::Enumerator::Lazy[Event]) }
     def replicate(watcher, publications:)
       watch_conn = PG.connect(
         dbname: @db_config[:database],
@@ -162,20 +150,13 @@ module Wal
       end
     end
 
-    class Column < T::Struct
-      const :name, String
-      const :decoder, T.untyped
-
+    class Column < Data.define(:name, :decoder)
       def decode(value)
         decoder.deserialize(value)
       end
     end
 
-    class Table < T::Struct
-      const :name, String
-      const :primary_key_colums, T::Array[String]
-      const :columns, T::Array[Column]
-
+    class Table < Data.define(:name, :primary_key_colums, :columns)
       def primary_key(decoded_row)
         case primary_key_colums
         in [key]

data/lib/wal/streaming_watcher.rb

@@ -1,5 +1,3 @@
-# typed: true
-
 module Wal
   # A watcher that streams all the events of each WAL transaction on a separate thread.
   #
@@ -12,7 +10,6 @@ module Wal
   #
   # ```ruby
   # class RegisterDeletesWalWatcher < Wal::StreamingWalWatcher
-  #   sig { override.params(events: T::Enumerator[Event]).void }
   #   def on_transaction_events(events)
   #     DeletedApplicationRecord.transaction do
   #       events
@@ -24,20 +21,14 @@ module Wal
   # end
   # ```
   class StreamingWatcher
-    extend T::Sig
-    extend T::Helpers
     include Wal::Watcher
-    abstract!
 
-    sig { abstract.params(events: T::Enumerator[Event]).void }
     def on_transaction_events(events); end
 
-    sig { params(event: BeginTransactionEvent).returns(Integer) }
     def queue_size(event)
       5_000
     end
 
-    sig { override.params(event: Event).void }
     def on_event(event)
       case event
       when BeginTransactionEvent

data/lib/wal/version.rb

@@ -1,6 +1,5 @@
-# typed: strict
 # frozen_string_literal: true
 
 module Wal
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/lib/wal/watcher.rb

@@ -1,5 +1,3 @@
-# typed: strict
-
 module Wal
   # Watcher is the core API used to hook into Postgres WAL log.
   # The only required method on the API is the `on_event`, which will receive a WAL entry of the following events:
@@ -13,22 +11,14 @@ module Wal
   # desired. In practice, it is rarelly useful to implement this module directly for application level business logic,
   # and instead it is more recomended using more specific ones, such as the `RecordWatcher` and `StreamingWalWatcher`.
   module Watcher
-    extend T::Sig
-    extend T::Helpers
-    include Wal
-    abstract!
-
-    sig { abstract.params(event: Event).void }
     def on_event(event); end
 
     # Allows dropping the processing of any table
-    sig { params(table: String).returns(T::Boolean) }
     def should_watch_table?(table)
       true
     end
 
     # Check if the given context prefix should be allowed for this watcher
-    sig { params(prefix: String).returns(T::Boolean) }
     def valid_context_prefix?(prefix)
       true
     end
@@ -42,25 +32,19 @@ module Wal
     #
     # ```ruby
     # class MeasureTransactionTimeWatcher
-    #   extend T::Sig
     #   include Wal::Watcher
     #   include Wal::Watcher::SeparatedEvents
     #
-    #   sig { params(event: BeginTransactionEvent).void }
     #   def on_begin(event)
     #     @start_time = Time.current
     #   end
     #
-    #   sig { params(event: CommitTransactionEvent).void }
     #   def on_commit(event)
     #     puts "Transaction processing time: #{Time.current - @start_time}"
     #   end
     # end
     # ```
     module SeparatedEvents
-      extend T::Sig
-
-      sig { params(event: Event).void }
       def on_event(event)
         case event
         when BeginTransactionEvent
@@ -76,19 +60,10 @@ module Wal
         end
       end
 
-      sig { params(event: BeginTransactionEvent).void }
       def on_begin(event); end
-
-      sig { params(event: InsertEvent).void }
       def on_insert(event); end
-
-      sig { params(event: UpdateEvent).void }
       def on_update(event); end
-
-      sig { params(event: DeleteEvent).void }
       def on_delete(event); end
-
-      sig { params(event: CommitTransactionEvent).void }
       def on_commit(event); end
     end
   end

data/lib/wal.rb

@@ -1,5 +1,3 @@
-# typed: strict
-
 require "pg"
 require "pg/replication"
 require "active_support"
@@ -13,63 +11,34 @@ require_relative "wal/active_record_context_extension"
 require_relative "wal/version"
 
 module Wal
-  Event = T.type_alias do
-    T.any(
-      BeginTransactionEvent,
-      CommitTransactionEvent,
-      InsertEvent,
-      UpdateEvent,
-      DeleteEvent,
-    )
-  end
-
-  class BeginTransactionEvent < T::Struct
-    extend T::Sig
-
-    const :transaction_id, Integer
-    const :lsn, Integer
-    const :final_lsn, Integer
-    const :timestamp, Time
-
-    sig { returns(Integer) }
+  class BeginTransactionEvent < Data.define(:transaction_id, :lsn, :final_lsn, :timestamp)
     def estimated_size
       final_lsn - lsn
     end
   end
 
-  class CommitTransactionEvent < T::Struct
-    const :transaction_id, Integer
-    const :lsn, Integer
-    const :context, T::Hash[String, T.untyped]
-    const :timestamp, Time
+  class CommitTransactionEvent < Data.define(:transaction_id, :lsn, :context, :timestamp)
   end
 
   module ChangeEvent
-    extend T::Sig
-
-    sig { returns(T::Hash[String, [T.untyped, T.untyped]]) }
     def diff
       {}
     end
 
-    sig { params(attribute: T.any(Symbol, String)).returns(T::Boolean) }
     def changed_attribute?(attribute)
       diff.key? attribute.to_s
     end
 
-    sig { params(attribute: T.any(Symbol, String)).returns(T.untyped) }
     def attribute(attribute)
       if (changes = diff[attribute.to_s])
         changes[1]
       end
     end
 
-    sig { params(attribute: T.any(Symbol, String)).returns(T.nilable([T.untyped, T.untyped])) }
     def attribute_changes(attribute)
       diff[attribute.to_s]
     end
 
-    sig { params(attribute: T.any(Symbol, String)).returns(T.untyped) }
     def attribute_was(attribute)
       if (changes = diff[attribute.to_s])
         changes[0]
@@ -77,36 +46,17 @@ module Wal
     end
   end
 
-  class InsertEvent < T::Struct
-    extend T::Sig
+  class InsertEvent < Data.define(:transaction_id, :lsn, :context, :table, :primary_key, :new)
     include ::Wal::ChangeEvent
 
-    const :transaction_id, Integer
-    const :lsn, Integer
-    const :context, T::Hash[String, T.untyped]
-    const :table, String
-    const :primary_key, T.untyped
-    const :new, T::Hash[String, T.untyped]
-
-    sig { returns(T::Hash[String, [T.untyped, T.untyped]]) }
     def diff
       new.transform_values { |val| [nil, val] }
     end
   end
 
-  class UpdateEvent < T::Struct
-    extend T::Sig
+  class UpdateEvent < Data.define(:transaction_id, :lsn, :context, :table, :primary_key, :old, :new)
     include ::Wal::ChangeEvent
 
-    const :transaction_id, Integer
-    const :lsn, Integer
-    const :context, T::Hash[String, T.untyped]
-    const :table, String
-    const :primary_key, T.untyped
-    const :old, T::Hash[String, T.untyped]
-    const :new, T::Hash[String, T.untyped]
-
-    sig { returns(T::Hash[String, [T.untyped, T.untyped]]) }
     def diff
       (old.keys | new.keys).reduce({}) do |diff, key|
         old[key] != new[key] ? diff.merge(key => [old[key], new[key]]) : diff
@@ -114,18 +64,9 @@ module Wal
     end
   end
 
-  class DeleteEvent < T::Struct
-    extend T::Sig
+  class DeleteEvent < Data.define(:transaction_id, :lsn, :context, :table, :primary_key, :old)
     include ::Wal::ChangeEvent
 
-    const :transaction_id, Integer
-    const :lsn, Integer
-    const :context, T::Hash[String, T.untyped]
-    const :table, String
-    const :primary_key, T.untyped
-    const :old, T::Hash[String, T.untyped]
-
-    sig { returns(T::Hash[String, [T.untyped, T.untyped]]) }
     def diff
       old.transform_values { |val| [val, nil] }
     end

data/rbi/wal.rbi

@@ -7,7 +7,7 @@ module Wal
       UpdateEvent,
       DeleteEvent,
     ) }
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 
   class BeginTransactionEvent < T::Struct
     prop :transaction_id, Integer, immutable: true
@@ -141,66 +141,6 @@ module Wal
 
     sig { override.params(event: Event).void }
     def on_event(event); end
-
-    class MemoryRecordWatcher
-      include Wal::Watcher
-      include Wal::Watcher::SeparatedEvents
-      extend T::Sig
-      extend T::Helpers
-      RecordsStorage = T.type_alias { T::Hash[[String, Integer], T.nilable(RecordEvent)] }
-
-      sig { params(watcher: T.untyped).void }
-      def initialize(watcher); end
-
-      sig { params(event: BeginTransactionEvent).void }
-      def on_begin(event); end
-
-      sig { params(_event: T.untyped).returns(T.untyped) }
-      def on_commit(_event); end
-
-      sig { params(event: InsertEvent).void }
-      def on_insert(event); end
-
-      sig { params(event: UpdateEvent).void }
-      def on_update(event); end
-
-      sig { params(event: DeleteEvent).void }
-      def on_delete(event); end
-    end
-
-    class TemporaryTableRecordWatcher
-      include Wal::Watcher
-      include Wal::Watcher::SeparatedEvents
-      extend T::Sig
-      extend T::Helpers
-
-      sig { params(watcher: T.untyped, batch_size: T.untyped).void }
-      def initialize(watcher, batch_size: 5_000); end
-
-      sig { params(event: BeginTransactionEvent).void }
-      def on_begin(event); end
-
-      sig { params(_event: T.untyped).returns(T.untyped) }
-      def on_commit(_event); end
-
-      sig { params(event: InsertEvent).void }
-      def on_insert(event); end
-
-      sig { params(event: UpdateEvent).void }
-      def on_update(event); end
-
-      sig { params(event: DeleteEvent).void }
-      def on_delete(event); end
-
-      sig { returns(T.class_of(::ActiveRecord::Base)) }
-      def base_class; end
-
-      sig { params(event: T.untyped).returns(T.untyped) }
-      def serialize(event); end
-
-      sig { params(persisted_event: T.untyped).returns(T.untyped) }
-      def deserialize(persisted_event); end
-    end
   end
 
   class Replicator
@@ -215,26 +155,6 @@ module Wal
 
     sig { params(watcher: Watcher, publications: T::Array[String]).returns(T::Enumerator::Lazy[Event]) }
     def replicate(watcher, publications:); end
-
-    class Column < T::Struct
-      prop :name, String, immutable: true
-      prop :decoder, T.untyped, immutable: true
-
-      sig { params(value: T.untyped).returns(T.untyped) }
-      def decode(value); end
-    end
-
-    class Table < T::Struct
-      prop :name, String, immutable: true
-      prop :primary_key_colums, T::Array[String], immutable: true
-      prop :columns, T::Array[Column], immutable: true
-
-      sig { params(decoded_row: T.untyped).returns(T.untyped) }
-      def primary_key(decoded_row); end
-
-      sig { params(values: T.untyped).returns(T.untyped) }
-      def decode_row(values); end
-    end
   end
 
   class StreamingWatcher