webhookdb 1.4.0 → 1.5.0

data/lib/webhookdb/jobs/deprecated_jobs.rb

@@ -15,6 +15,8 @@ Amigo::DeprecatedJobs.install(
   "Jobs::ConvertKitSubscriberBackfill",
   "Jobs::ConvertKitTagBackfill",
   "Jobs::CustomerCreatedNotifyInternal",
+  "Jobs::OrganizationDatabaseMigrationNotifyStarted",
+  "Jobs::OrganizationDatabaseMigrationNotifyFinished",
   "Jobs::RssBackfillPoller",
   "Jobs::TwilioScheduledBackfill",
 )

data/lib/webhookdb/jobs/emailer.rb

@@ -9,6 +9,7 @@ class Webhookdb::Jobs::Emailer
   splay 5.seconds
 
   def _perform
-    Webhookdb::Message.send_unsent
+    sent = Webhookdb::Message.send_unsent
+    self.set_job_tags(sent_messages: sent.count)
   end
 end

data/lib/webhookdb/jobs/front_signalwire_message_channel_sync_inbound.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+require "webhookdb/jobs"
+
+# See +Webhookdb::Replicator::FrontSignalwireMessageChannelAppV1#alert_async_failed_signalwire_send+
+# for why we need to pull this into an async job.
+class Webhookdb::Jobs::FrontSignalwireMessageChannelSyncInbound
+  extend Webhookdb::Async::Job
+
+  def perform(service_integration_id, kwargs)
+    sint = self.lookup_model(Webhookdb::ServiceIntegration, service_integration_id)
+    sint.replicator.sync_front_inbound_message(**kwargs.symbolize_keys)
+  end
+end

data/lib/webhookdb/jobs/icalendar_delete_stale_cancelled_events.rb

@@ -9,10 +9,15 @@ class Webhookdb::Jobs::IcalendarDeleteStaleCancelledEvents
   cron "37 7 * * *" # Once a day
   splay 120
 
+  ADVISORY_LOCK_ID = 1_236_432_568
+
   def _perform
-    Webhookdb::ServiceIntegration.where(service_name: "icalendar_event_v1").each do |sint|
+    Webhookdb::ServiceIntegration.where(service_name: Webhookdb::Icalendar::EVENT_REPLICATORS).each do |sint|
       self.with_log_tags(sint.log_tags) do
-        sint.replicator.delete_stale_cancelled_events
+        sint.replicator.with_advisory_lock(ADVISORY_LOCK_ID) do
+          deleted_rows = sint.replicator.stale_row_deleter.run
+          self.set_job_tags("#{sint.organization.key}_#{sint.table_name}" => deleted_rows)
+        end
       end
     end
   end

data/lib/webhookdb/jobs/icalendar_enqueue_syncs.rb

@@ -5,27 +5,90 @@ require "webhookdb/jobs"
 
 # For every IcalendarCalendar row needing a sync (across all service integrations),
 # enqueue a +Webhookdb::Jobs::IcalendarSync+ job.
-# Jobs are 'splayed' over 1/4 of the configured calendar sync period (see +Webhookdb::Icalendar+)
-# to avoid a thundering herd.
+#
+# Because icalendars need to be synced periodically,
+# and there can be quite a lot, we have to be clever with how we sync them to both avoid syncing too often,
+# and especially avoid saturating workers with syncs.
+# We also have to handle workers running behind, the app being slow, etc.
+#
+# - This job runs every 30 minutes.
+# - It finds rows never synced,
+#   or haven't been synced in 8 hours (see +Webhookdb::Icalendar.sync_period_hours+
+#   for the actual value, we'll assume 8 hours for this doc).
+# - Enqueue a row sync sync job for between 1 second and 60 minutes from now
+#   (see +Webhookdb::Icalendar.sync_period_splay_hours+ for actual upper bound value).
+# - When the sync job runs, if the row has been synced less than 8 hours ago, the job noops.
+#
+# This design will lead to the same calendar being enqueued for a sync multiple times
+# (for a splay of 1 hour, and running this job every 30 minutes, about half the jobs in the queue will be duplicate).
+# This isn't a problem however, since the first sync will run but the duplicates will noop
+# since the row will be seen as recently synced.
+#
+# Importantly, if there is a thundering herd situation,
+# because there is a massive traunch of rows that need to be synced,
+# and it takes maybe 10 hours to sync rather than one,
+# the herd will be thinned and smeared over time as each row is synced.
+# There isn't much we can do for the initial herd (other than making sure
+# only some number of syncs are processing for a given org at a time)
+# but we won't keep getting thundering herds from the same calendars over time.
 class Webhookdb::Jobs::IcalendarEnqueueSyncs
   extend Webhookdb::Async::ScheduledJob
 
-  cron "*/30 * * * *" # Every 30 minutes
+  # See docs for explanation of why we run this often.
+  cron "*/30 * * * *"
   splay 30
 
   def _perform
-    max_splay = Webhookdb::Icalendar.sync_period_hours.hours.to_i / 4
+    self.advisory_lock.with_lock? do
+      self.__perform
+    end
+  end
+
+  # Just a random big number
+  LOCK_ID = 2_161_457_251_202_716_167
+
+  def advisory_lock
+    return Sequel::AdvisoryLock.new(Webhookdb::Customer.db, LOCK_ID)
+  end
+
+  def __perform
+    max_projected_out_seconds = Webhookdb::Icalendar.sync_period_splay_hours.hours.to_i
+    total_count = 0
+    threadpool = Concurrent::ThreadPoolExecutor.new(
+      name: "ical-precheck",
+      max_threads: Webhookdb::Icalendar.precheck_feed_change_pool_size,
+      min_threads: 1,
+      idletime: 40,
+      max_queue: 0,
+      fallback_policy: :caller_runs,
+      synchronous: false,
+    )
     Webhookdb::ServiceIntegration.dataset.where_each(service_name: "icalendar_calendar_v1") do |sint|
-      sint.replicator.admin_dataset do |ds|
-        sint.replicator.rows_needing_sync(ds).each do |row|
-          calendar_external_id = row.fetch(:external_id)
-          self.with_log_tags(sint.log_tags) do
-            splay = rand(1..max_splay)
-            enqueued_job_id = Webhookdb::Jobs::IcalendarSync.perform_in(splay, sint.id, calendar_external_id)
-            self.logger.debug("enqueued_icalendar_sync", calendar_external_id:, enqueued_job_id:)
+      sint_count = 0
+      self.with_log_tags(sint.log_tags) do
+        repl = sint.replicator
+        repl.admin_dataset do |ds|
+          row_ds = repl.
+            rows_needing_sync(ds).
+            order(:pk).
+            select(:external_id, :ics_url, :last_fetch_context)
+          row_ds.paged_each(rows_per_fetch: 500, cursor_name: "ical_enqueue_#{sint.id}_cursor") do |row|
+            threadpool.post do
+              break unless repl.feed_changed?(row)
+              calendar_external_id = row.fetch(:external_id)
+              perform_in = rand(1..max_projected_out_seconds)
+              enqueued_job_id = Webhookdb::Jobs::IcalendarSync.perform_in(perform_in, sint.id, calendar_external_id)
+              self.logger.debug("enqueued_icalendar_sync", calendar_external_id:, enqueued_job_id:, perform_in:)
+              sint_count += 1
+            end
           end
         end
       end
+      total_count += sint_count
+      self.set_job_tags("#{sint.organization.key}_#{sint.table_name}" => sint_count)
     end
+    threadpool.shutdown
+    threadpool.wait_for_termination
+    self.set_job_tags(total_enqueued: total_count)
   end
 end

data/lib/webhookdb/jobs/icalendar_enqueue_syncs_for_urls.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+
+class Webhookdb::Jobs::IcalendarEnqueueSyncsForUrls
+  extend Webhookdb::Async::Job
+
+  def perform(urls)
+    self.set_job_tags(url_count: urls.length)
+    row_count = 0
+    Webhookdb::ServiceIntegration.where(service_name: "icalendar_calendar_v1").each do |sint|
+      sint.replicator.admin_dataset do |ds|
+        affected_row_ext_ids = ds.where(ics_url: urls).select_map(:external_id)
+        affected_row_ext_ids.each do |ext_id|
+          Webhookdb::Jobs::IcalendarSync.perform_async(sint.id, ext_id)
+        end
+        row_count += affected_row_ext_ids.length
+      end
+    end
+    self.set_job_tags(result: "icalendar_enqueued_syncs", row_count:)
+  end
+end

data/lib/webhookdb/jobs/icalendar_sync.rb

@@ -1,23 +1,35 @@
 # frozen_string_literal: true
 
+require "amigo/semaphore_backoff_job"
+
 require "webhookdb/async/job"
 require "webhookdb/jobs"
 
 class Webhookdb::Jobs::IcalendarSync
   extend Webhookdb::Async::Job
+  include Amigo::SemaphoreBackoffJob
 
-  sidekiq_options retry: false
+  sidekiq_options retry: false, queue: "netout"
 
   def perform(sint_id, calendar_external_id)
     sint = self.lookup_model(Webhookdb::ServiceIntegration, sint_id)
-    self.with_log_tags(sint.log_tags.merge(calendar_external_id:)) do
-      row = sint.replicator.admin_dataset { |ds| ds[external_id: calendar_external_id] }
-      if row.nil?
-        self.logger.warn("icalendar_sync_row_miss", calendar_external_id:)
-        return
-      end
-      self.logger.debug("icalendar_sync_start")
-      sint.replicator.sync_row(row)
+    self.set_job_tags(sint.log_tags.merge(calendar_external_id:))
+    row = sint.replicator.admin_dataset { |ds| ds[external_id: calendar_external_id] }
+    if row.nil?
+      self.set_job_tags(result: "icalendar_sync_row_miss")
+      return
     end
+    self.logger.debug("icalendar_sync_start")
+    sint.replicator.sync_row(row)
+    self.set_job_tags(result: "icalendar_synced")
+  end
+
+  def before_perform(sint_id, *)
+    @sint = self.lookup_model(Webhookdb::ServiceIntegration, sint_id)
   end
+
+  def semaphore_key = "semaphore-icalendarsync-#{@sint.organization_id}"
+  def semaphore_size = @sint.organization.job_semaphore_size
+  def semaphore_expiry = 15.minutes
+  def semaphore_backoff = 60 + (rand * 30)
 end

data/lib/webhookdb/jobs/increase_event_handler.rb

@@ -8,13 +8,14 @@ class Webhookdb::Jobs::IncreaseEventHandler
   on "increase.*"
 
   def _perform(event)
+    self.set_job_tags(increase_event_name: event.name)
     case event.name
       when "increase.oauth_connection.deactivated"
         conn_id = event.payload[0].fetch("associated_object_id")
-        self.logger.info("increase_oauth_disconnect", oauth_connection_id: conn_id)
+        self.set_job_tags(result: "increase_oauth_disconnected", oauth_connection_id: conn_id)
         Webhookdb::Oauth::IncreaseProvider.disconnect_oauth(conn_id)
       else
-        self.logger.info("increase_event_noop")
+        self.set_job_tags(result: "increase_event_noop")
     end
   end
 end

data/lib/webhookdb/jobs/logged_webhooks_replay.rb

@@ -10,8 +10,10 @@ class Webhookdb::Jobs::LoggedWebhooksReplay
 
   def _perform(event)
     lwh = self.lookup_model(Webhookdb::LoggedWebhook, event)
-    self.with_log_tags(service_integration_opaque_id: lwh.service_integration_opaque_id) do
-      lwh.retry_one(truncate_successful: true)
-    end
+    self.set_job_tags(
+      logged_webhook_id: lwh.id,
+      service_integration_opaque_id: lwh.service_integration_opaque_id,
+    )
+    lwh.retry_one(truncate_successful: true)
   end
 end

data/lib/webhookdb/jobs/message_dispatched.rb

@@ -9,6 +9,7 @@ class Webhookdb::Jobs::MessageDispatched
 
   def _perform(event)
     delivery = self.lookup_model(Webhookdb::Message::Delivery, event)
+    self.set_job_tags(delivery_id: delivery.id, to: delivery.to)
     Webhookdb::Idempotency.once_ever.under_key("message-dispatched-#{delivery.id}") do
       delivery.send!
     end

data/lib/webhookdb/jobs/model_event_system_log_tracker.rb

@@ -9,6 +9,7 @@ class Webhookdb::Jobs::ModelEventSystemLogTracker
   on "webhookdb.*"
 
   def _perform(event)
+    self.set_job_tags(event_name: event.name)
     case event.name
         when "webhookdb.customer.created"
           self.alert_customer_created(event)
@@ -18,6 +19,8 @@ class Webhookdb::Jobs::ModelEventSystemLogTracker
           self.alert_sint_created(event)
         when "webhookdb.serviceintegration.destroyed"
           self.alert_sint_destroyed(event)
+      else
+          self.set_job_tags(result: "noop")
       end
   end
 
@@ -43,6 +46,7 @@ class Webhookdb::Jobs::ModelEventSystemLogTracker
       ],
     ).emit
     create_event("Customer Created", customer.email, customer.admin_link)
+    self.set_job_tags(result: "created_customer", email: customer.email)
   end
 
   def alert_org_created(event)
@@ -58,6 +62,7 @@ class Webhookdb::Jobs::ModelEventSystemLogTracker
       ],
     ).emit
     create_event("Organization Created", "#{org.name} (#{org.key})", org.admin_link)
+    self.set_job_tags(result: "created_organization", key: org.key)
   end
 
   def alert_sint_created(event)
@@ -79,6 +84,7 @@ class Webhookdb::Jobs::ModelEventSystemLogTracker
       "#{sint.service_name} (#{sint.opaque_id}) created in #{sint.organization.name}",
       sint.admin_link,
     )
+    self.set_job_tags(result: "created_service_integration", opaque_id: sint.opaque_id, service: sint.service_name)
   end
 
   def alert_sint_destroyed(event)
@@ -101,5 +107,6 @@ class Webhookdb::Jobs::ModelEventSystemLogTracker
       "#{pl[:service_name]} (#{pl[:opaque_id]}) deleted from #{org.name}",
       org.admin_link,
     )
+    self.set_job_tags(result: "destroyed_service_integration", opaque_id: pl[:oparque_id], service: pl[:service_name])
   end
 end

data/lib/webhookdb/jobs/monitor_metrics.rb

@@ -24,6 +24,6 @@ class Webhookdb::Jobs::MonitorMetrics
     end
     opts[:max_size] = max_size
     opts[:max_latency] = max_latency
-    self.logger.info("metrics_monitor_queue", **opts)
+    self.set_job_tags(action: "metrics_monitor_queue", **opts)
   end
 end

data/lib/webhookdb/jobs/organization_database_migration_notify.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+require "webhookdb/messages/org_database_migration_finished"
+require "webhookdb/messages/org_database_migration_started"
+
+class Webhookdb::Jobs::OrganizationDatabaseMigrationNotify
+  extend Webhookdb::Async::Job
+
+  on "webhookdb.organization.databasemigration.updated"
+
+  def _perform(event)
+    dbm = self.lookup_model(Webhookdb::Organization::DatabaseMigration, event)
+    self.set_job_tags(database_migration_id: dbm.id, organization: dbm.organization.key)
+    case event.payload[1]
+      when changed(:started_at, from: nil)
+        Webhookdb::Idempotency.once_ever.under_key("org-dbmigration-start-#{dbm.id}") do
+          msg = Webhookdb::Messages::OrgDatabaseMigrationStarted.new(dbm)
+          dbm.organization.admin_customers.each { |c| msg.dispatch_email(c) }
+        end
+        self.set_job_tags(result: "started_message_sent")
+      when changed(:finished_at, from: nil)
+        Webhookdb::Idempotency.once_ever.under_key("org-dbmigration-finish-#{dbm.id}") do
+          msg = Webhookdb::Messages::OrgDatabaseMigrationFinished.new(dbm)
+          dbm.organization.admin_customers.each { |c| msg.dispatch_email(c) }
+        end
+        self.set_job_tags(result: "finished_message_sent")
+      else
+        self.set_job_tags(result: "noop")
+    end
+  end
+end

data/lib/webhookdb/jobs/organization_database_migration_run.rb

@@ -11,14 +11,12 @@ class Webhookdb::Jobs::OrganizationDatabaseMigrationRun
 
   def _perform(event)
     dbm = self.lookup_model(Webhookdb::Organization::DatabaseMigration, event)
-    self.with_log_tags(
-      organization_id: dbm.organization.id,
-      organization_name: dbm.organization.name,
-      organization_database_migration_id: dbm.id,
-    ) do
+    self.set_job_tags(organization: dbm.organization.key, database_migration_id: dbm.id)
+    begin
       dbm.migrate
+      self.set_job_tags(result: "migration_finished")
     rescue Webhookdb::Organization::DatabaseMigration::MigrationAlreadyFinished
-      self.logger.warn("org_database_migration_already_finished")
+      self.set_job_tags(result: "migration_already_finished")
     end
   end
 end

data/lib/webhookdb/jobs/organization_error_handler_dispatch.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+
+class Webhookdb::Jobs::OrganizationErrorHandlerDispatch
+  extend Webhookdb::Async::Job
+
+  sidekiq_options queue: "netout"
+
+  def perform(error_handler_id, payload)
+    eh = self.lookup_model(Webhookdb::Organization::ErrorHandler, error_handler_id)
+    self.set_job_tags(error_handler_id: eh.id, **eh.organization.log_tags)
+    begin
+      eh.dispatch(payload)
+      self.set_job_tags(result: "success")
+    rescue StandardError => e
+      # Don't bother logging these errors out
+      self.set_job_tags(result: "error")
+      self.logger.debug("organization_error_handler_post_error", error: e)
+      raise Amigo::Retry::OrDie.new(
+        Webhookdb::Organization::Alerting.error_handler_retries,
+        Webhookdb::Organization::Alerting.error_handler_retry_interval,
+      )
+    end
+  end
+end

data/lib/webhookdb/jobs/prepare_database_connections.rb

@@ -12,6 +12,7 @@ class Webhookdb::Jobs::PrepareDatabaseConnections
 
   def _perform(event)
     org = self.lookup_model(Webhookdb::Organization, event)
+    self.set_job_tags(organization: org.key)
     org.db.transaction do
       # If creating the public host fails, we end up with an orphaned database,
       # but that's not a big deal- we can eventually see it's empty/unlinked and drop it.

data/lib/webhookdb/jobs/process_webhook.rb

@@ -23,18 +23,17 @@ class Webhookdb::Jobs::ProcessWebhook
   end
 
   def _perform(event)
-    self.with_log_tags(@sint.log_tags) do
-      kw = event.payload[1].symbolize_keys
-      svc = Webhookdb::Replicator.create(@sint)
-      # kwargs contains: :headers, :body, :request_path, :request_method
-      req = Webhookdb::Replicator::WebhookRequest.new(
-        body: kw.fetch(:body),
-        headers: kw.fetch(:headers),
-        path: kw.fetch(:request_path),
-        method: kw.fetch(:request_method),
-      )
-      svc.upsert_webhook(req)
-    end
+    self.set_job_tags(@sint.log_tags)
+    kw = event.payload[1].symbolize_keys
+    svc = Webhookdb::Replicator.create(@sint)
+    # kwargs contains: :headers, :body, :request_path, :request_method
+    req = Webhookdb::Replicator::WebhookRequest.new(
+      body: kw.fetch(:body),
+      headers: kw.fetch(:headers),
+      path: kw.fetch(:request_path),
+      method: kw.fetch(:request_method),
+    )
+    svc.upsert_webhook(req)
   end
 
   def semaphore_key

data/lib/webhookdb/jobs/renew_watch_channel.rb

@@ -6,22 +6,19 @@ require "webhookdb/jobs"
 
 # Generic helper to renew watch channels, enqueued by replicator-specific jobs
 # like RenewGoogleWatchChannels.
-# Must be emitted with [service integration id, {row_pk:, expirng_before:}]
+# Must be emitted with [service_integration_id, {row_pk:, expirng_before:}]
 # Calls #renew_watch_channel(row_pk:, expiring_before:).
 class Webhookdb::Jobs::RenewWatchChannel
   extend Webhookdb::Async::Job
   include Amigo::QueueBackoffJob
 
-  on "webhookdb.serviceintegration.renewwatchchannel"
   sidekiq_options queue: "netout"
 
-  def _perform(event)
-    sint = self.lookup_model(Webhookdb::ServiceIntegration, event)
-    self.with_log_tags(sint.log_tags) do
-      opts = event.payload[1]
-      row_pk = opts.fetch("row_pk")
-      expiring_before = Time.parse(opts.fetch("expiring_before"))
-      sint.replicator.renew_watch_channel(row_pk:, expiring_before:)
-    end
+  def perform(service_integration_id, renew_watch_criteria)
+    sint = self.lookup_model(Webhookdb::ServiceIntegration, service_integration_id)
+    self.set_job_tags(sint.log_tags)
+    row_pk = renew_watch_criteria.fetch("row_pk")
+    expiring_before = Time.parse(renew_watch_criteria.fetch("expiring_before"))
+    sint.replicator.renew_watch_channel(row_pk:, expiring_before:)
   end
 end

data/lib/webhookdb/jobs/replication_migration.rb

@@ -2,18 +2,21 @@
 
 # See Organization::enqueue_migrate_all_replication_tables for more info.
 class Webhookdb::Jobs::ReplicationMigration
-  include Sidekiq::Worker
+  extend Webhookdb::Async::Job
 
   def perform(org_id, target_release_created_at)
     (org = Webhookdb::Organization[org_id]) or raise "Organization[#{org_id}] does not exist"
     target_rca = Time.parse(target_release_created_at)
     current_rca = Time.parse(Webhookdb::RELEASE_CREATED_AT)
+    self.set_job_tags(organization_id: org_id, target_release_created_at:)
     if target_rca == current_rca
       self.class.migrate_org(org)
+      self.set_job_tags(result: "ran_replication_migration_job")
     elsif target_rca > current_rca
       self.class.perform_in(1, org_id, target_release_created_at)
+      self.set_job_tags(result: "reenqueued_replication_migration_job")
     else
-      self.logger.warn("stale_replication_migration_job", target_release_created_at:)
+      self.set_job_tags(result: "stale_replication_migration_job")
     end
   end
 

data/lib/webhookdb/jobs/reset_code_create_dispatch.rb

@@ -10,13 +10,12 @@ class Webhookdb::Jobs::ResetCodeCreateDispatch
 
   def _perform(event)
     code = self.lookup_model(Webhookdb::Customer::ResetCode, event)
+    self.set_job_tags(code_id: code.id, customer: code.customer.email, transport: code.transport)
     Webhookdb::Idempotency.once_ever.under_key("reset-code-#{code.customer_id}-#{code.id}") do
       msg = Webhookdb::Messages::Verification.new(code)
       case code.transport
         when "email"
           msg.dispatch_email(code.customer)
-      else
-          raise "Unknown transport for #{code.inspect}"
       end
     end
   end

data/lib/webhookdb/jobs/scheduled_backfills.rb

@@ -51,9 +51,9 @@ module Webhookdb::Jobs
         Webhookdb::Github.activity_cron_expression, 30.seconds, false,
       ),
       Spec.new(
-        # I think we can get rid of this once we're more confident webhooks are working reliably.
+        # This incremental sync is a backstop for any missed webhooks.
         "IntercomScheduledBackfill", "intercom_marketplace_root_v1",
-        "46 4 * * *", 0, false, true,
+        "46 4 * * *", 0, true, true,
       ),
       Spec.new(
         "AtomSingleFeedPoller", "atom_single_feed_v1",

data/lib/webhookdb/jobs/send_invite.rb

@@ -9,7 +9,8 @@ class Webhookdb::Jobs::SendInvite
   on "webhookdb.organizationmembership.invite"
 
   def _perform(event)
-    membership = self.lookup_model(Webhookdb::OrganizationMembership, event)
-    Webhookdb::Messages::Invite.new(membership).dispatch(membership.customer)
+    m = self.lookup_model(Webhookdb::OrganizationMembership, event)
+    self.set_job_tags(membership_id: m.id, organization: m.organization.key, customer: m.customer.email)
+    Webhookdb::Messages::Invite.new(m).dispatch(m.customer)
   end
 end

data/lib/webhookdb/jobs/send_test_webhook.rb

@@ -14,9 +14,7 @@ class Webhookdb::Jobs::SendTestWebhook
   # we don't want to retry and randomly send a payload later.
   sidekiq_options retry: false
 
-  def dependent_queues
-    return ["critical"]
-  end
+  def dependent_queues = ["critical"]
 
   def _perform(event)
     webhook_sub = self.lookup_model(Webhookdb::WebhookSubscription, event)

data/lib/webhookdb/jobs/send_webhook.rb

@@ -10,11 +10,10 @@ class Webhookdb::Jobs::SendWebhook
 
   def _perform(event)
     sint = self.lookup_model(Webhookdb::ServiceIntegration, event)
-    self.with_log_tags(sint.log_tags) do
-      sint.all_webhook_subscriptions_dataset.to_notify.each do |sub|
-        payload = {service_name: sint.service_name, table_name: sint.table_name, **event.payload[1]}
-        sub.enqueue_delivery(payload)
-      end
+    self.set_job_tags(sint.log_tags)
+    sint.all_webhook_subscriptions_dataset.to_notify.each do |sub|
+      payload = {service_name: sint.service_name, table_name: sint.table_name, **event.payload[1]}
+      sub.enqueue_delivery(payload)
     end
   end
 end

data/lib/webhookdb/jobs/stale_row_deleter.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+
+# Run the +stale_row_deleter+ for each service integration
+# which match the given where/exclude clauses.
+# This is generally used to delete old stale rows in the backend
+# (by passing initial: true) when a new stale row deleter is deployed.
+class Webhookdb::Jobs::StaleRowDeleter
+  extend Webhookdb::Async::Job
+
+  def perform(opts={})
+    opts = opts.deep_symbolize_keys
+    opts[:where] ||= {}
+    opts[:exclude] ||= {}
+    opts[:initial] ||= false
+    ds = Webhookdb::ServiceIntegration.dataset
+    ds = ds.where(opts[:where]) if opts[:where]
+    ds = ds.exclude(opts[:exclude]) if opts[:exclude]
+    self.set_job_tags(dataset: ds.sql)
+    count = 0
+    ds.each do |sint|
+      self.with_log_tags(sint.log_tags) do
+        d = sint.replicator.stale_row_deleter
+        opts[:initial] ? d.run_initial : d.run
+        count += 1
+      end
+    end
+    self.set_job_tags(run_count: count)
+  end
+end

data/lib/webhookdb/jobs/sync_target_enqueue_scheduled.rb

@@ -9,8 +9,11 @@ class Webhookdb::Jobs::SyncTargetEnqueueScheduled
   splay 0
 
   def _perform
+    count = 0
     Webhookdb::SyncTarget.due_for_sync(as_of: Time.now).select(:id, :period_seconds).each do |st|
+      count += 1
       Webhookdb::Jobs::SyncTargetRunSync.perform_in(st.jitter, st.id)
     end
+    self.set_job_tags(enqueued_count: count)
   end
 end