undertow 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1aa09fe4d1c2e088f4b7ca874175db916486f8a6296d49af88544bec8ec254be
-  data.tar.gz: 30115bb681233cf045251670a9f2ed369e6290340414f5f031f292cd2e26fe23
+  metadata.gz: 8eaf3f2d446514a0b894990d2b5f0d5151a9664782af31533dfb38a1954b60b1
+  data.tar.gz: 9bdd7da4f3e68d80638e79911ce67fe20a56b689ce048835836eb2e6c90f3d69
 SHA512:
-  metadata.gz: a68ec79086258526eb24fbd2690b5df15701975706ee4ae09bbbd800d4e024d1b9c4546d7df03f465de7bbdcd4efc6252b73ac2244252b74d6f93707ea5dff1b
-  data.tar.gz: 9406d057b05d1a76fade09a0d3b3b698d3503a11bccfdd68591f4eead07bce128dfd3b431fbf4e6078eac092bc4f641a51d0dda036bdd200ab928dbec24155af
+  metadata.gz: 3f45df71d291f76c94a4f6550734ef86dbfb8bbccb8d85011e7bb6c144ea78098471e683cf59fc9860858388517ac570e5d65b0eeea215a4180443ab029fd9cb
+  data.tar.gz: 6ecdd5f94d0500e8c16ca9f127c3aed178f517be9006d589b3a28695d144931a4b08ac9e9e31567fe0b2d94fb5b2a5d1fe8ca6a58c71a864b04c6d0727c30e29

data/lib/undertow/drain_job.rb

@@ -2,11 +2,12 @@
 
 module Undertow
   # Drains the per-model Redis buffers and delivers batches of dirty IDs to
-  # each model's configured on_drain handler.
+  # each model's configured sinks.
   #
-  # Publishes two ActiveSupport::Notifications events:
-  #   drain.undertow , after a successful on_drain call ({ model:, upserted_ids:, deleted_ids:, duration_ms: })
-  #   error.undertow , when on_drain raises ({ model:, exception: })
+  # Publishes two ActiveSupport::Notifications events, once per sink (per chunk,
+  # if a sink's max_batch_size is smaller than the popped batch):
+  #   drain.undertow , after a successful sink call ({ model:, sink:, upserted_ids:, deleted_ids:, duration_ms: })
+  #   error.undertow , when a sink raises ({ model:, sink:, exception: })
   class DrainJob < ActiveJob::Base
     queue_as { Undertow.configuration.queue_name }
 
@@ -26,6 +27,10 @@ module Undertow
     def drain_model(model_name)
       max = Undertow.configuration.max_batch
 
+      config = Registry[model_name]
+      raise "No Undertow config registered for #{model_name}" unless config
+      raise "#{model_name} has no undertow_sink configured" if config.sinks.empty?
+
       # Deregister before popping, any concurrent push will re-add the model,
       # preventing the race where srem fires after a concurrent sadd.
       Buffer.deregister_model(model_name)
@@ -37,26 +42,45 @@ module Undertow
       # If the batch was capped, re-register so the next scheduler tick picks up.
       Buffer.reregister_model(model_name) if Buffer.remaining(model_name).positive?
 
-      config = Registry[model_name]
-      raise "No Undertow config registered for #{model_name}" unless config
-      raise "#{model_name} is missing undertow_on_drain" unless config.on_drain
+      current_sink = nil
+      config.sinks.each do |sink_name, sink|
+        current_sink = sink_name
+        chunk_size   = sink[:max_batch_size] || max
 
-      duration_ms = measure_ms { config.on_drain.call(model_name, upserted_ids, deleted_ids) }
+        each_sink_chunk(upserted_ids, deleted_ids, chunk_size) do |chunk_upserted, chunk_deleted|
+          duration_ms = measure_ms { sink[:handler].call(model_name, chunk_upserted, chunk_deleted) }
 
-      ActiveSupport::Notifications.instrument('drain.undertow', {
-        model: model_name,
-        upserted_ids: upserted_ids,
-        deleted_ids: deleted_ids,
-        duration_ms: duration_ms
-      })
+          ActiveSupport::Notifications.instrument('drain.undertow', {
+            model: model_name,
+            sink: sink_name,
+            upserted_ids: chunk_upserted,
+            deleted_ids: chunk_deleted,
+            duration_ms: duration_ms
+          })
+        end
+      end
     rescue StandardError => e
       Buffer.restore_pending(model_name, upserted_ids) if upserted_ids&.any?
       Buffer.restore_deleted(model_name, deleted_ids)  if deleted_ids&.any?
       Buffer.reregister_model(model_name)
-      ActiveSupport::Notifications.instrument('error.undertow', { model: model_name, exception: e })
+      ActiveSupport::Notifications.instrument('error.undertow', { model: model_name, sink: current_sink, exception: e })
       Rails.logger.error("[Undertow::DrainJob] #{model_name}: #{e.message}") if defined?(Rails)
     end
 
+    # Splits upserted_ids/deleted_ids into chunks of at most chunk_size combined
+    # items, yielding each chunk as (chunk_upserted, chunk_deleted). Tags each ID
+    # with which array it came from, concatenates into one list, slices that,
+    # then splits each slice back apart, so a single chunk_size limit governs
+    # the combined total regardless of the upserted/deleted split.
+    def each_sink_chunk(upserted_ids, deleted_ids, chunk_size)
+      tagged = upserted_ids.map { |id| [true, id] } + deleted_ids.map { |id| [false, id] }
+
+      tagged.each_slice(chunk_size) do |slice|
+        chunk_upserted, chunk_deleted = slice.partition { |upserted, _| upserted }
+        yield chunk_upserted.map(&:last), chunk_deleted.map(&:last)
+      end
+    end
+
     def measure_ms
       start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       yield

data/lib/undertow/dsl.rb

@@ -6,8 +6,8 @@ module Undertow
   # gets Trackable behavior wired in at boot, no include needed.
   #
   #   class Post < ApplicationRecord
-  #     undertow_on_drain ->(model_name, upserted_ids, deleted_ids) { PostSyncJob.perform_later(upserted_ids, deleted_ids) }
-  #     undertow_skip     %w[view_count updated_at]
+  #     undertow_sink(:search_index) { |model_name, upserted_ids, deleted_ids| PostSyncJob.perform_later(upserted_ids, deleted_ids) }
+  #     undertow_skip %w[view_count updated_at]
   #
   #     undertow_depends_on :author, foreign_key: :author_id, watched_columns: %w[name bio]
   #     undertow_depends_on :tag,
@@ -16,8 +16,21 @@ module Undertow
   #   end
   #
   module DSL
-    def undertow_on_drain(callable)
-      _undertow_config.on_drain = callable
+    # Registers a named sink. Call once per sink; every sink on a model receives
+    # the same (model_name, upserted_ids, deleted_ids) on each drain, so the
+    # block decides what to do with them, e.g. reindex a search index or publish
+    # to Kafka.
+    #
+    # `max_batch_size:` bounds how many IDs this sink's block receives per call.
+    # Defaults to `Undertow.configuration.max_batch`, the same size as the
+    # popped batch, meaning the block gets called once with everything. Set it
+    # lower when this sink's downstream call has a tighter limit than other
+    # sinks on the same model; DrainJob will call the block multiple times,
+    # chunked to this size, without affecting any other sink's batch size.
+    def undertow_sink(name, max_batch_size: nil, &handler)
+      raise ArgumentError, 'undertow_sink requires a block' unless handler
+
+      _undertow_config.sinks[name.to_sym] = { max_batch_size: max_batch_size, handler: handler }
       _undertow_ensure_trackable!
     end
 

data/lib/undertow/railtie.rb

@@ -3,7 +3,7 @@
 module Undertow
   class Railtie < Rails::Railtie
     # Extend ActiveRecord::Base with the Undertow DSL so any model can call
-    # undertow_on_drain, undertow_skip, and undertow_depends_on in its class body.
+    # undertow_sink, undertow_skip, and undertow_depends_on in its class body.
     initializer 'undertow.extend_active_record' do
       ActiveSupport.on_load(:active_record) { extend Undertow::DSL }
     end

data/lib/undertow/registry.rb

@@ -2,20 +2,20 @@
 
 module Undertow
   # Holds the declared dependency configuration for every tracked model.
-  # Populated at class load time via the DSL (undertow_on_drain, undertow_skip,
+  # Populated at class load time via the DSL (undertow_sink, undertow_skip,
   # undertow_depends_on); consumed by the Railtie and DrainJob.
   module Registry
     MODELS_KEY = 'undertow:pending:models'
 
     class ModelConfig
-      attr_reader   :model_name, :dependencies
-      attr_accessor :on_drain, :skip_columns
+      attr_reader   :model_name, :dependencies, :sinks
+      attr_accessor :skip_columns
 
       def initialize(model_name)
         @model_name   = model_name
         @dependencies = []
         @skip_columns = []
-        @on_drain     = nil
+        @sinks        = {}
       end
     end
 

data/lib/undertow/trackable.rb

@@ -2,7 +2,7 @@
 
 module Undertow
   # ActiveRecord concern mixed in automatically when a model uses the Undertow DSL
-  # (undertow_on_drain, undertow_skip, undertow_depends_on). Never included manually.
+  # (undertow_sink, undertow_skip, undertow_depends_on). Never included manually.
   #
   # Provides class-level callback registration and dependency push handlers, plus
   # instance-level self-tracking handlers. Callbacks are wired at boot by the Railtie

data/lib/undertow/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Undertow
-  VERSION = '0.4.0'
+  VERSION = '0.5.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: undertow
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Nathan Allen