webhookdb 1.3.1 → 1.5.0

data/lib/webhookdb/errors.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Webhookdb::Errors
+  class << self
+    # Call the given block for the given exception, each cause (see +Exception#cause+),
+    # and each wrapped errors (see +Amigo::Retry::Error#wrapped+).
+    # If the block returns +true+ for any exception, stop walking.
+    def each_cause(ex, &)
+      raise LocalJumpError unless block_given?
+      return true if yield(ex) == true
+      caused_got = ex.cause && each_cause(ex.cause, &)
+      return true if caused_got == true
+      wrapped_got = ex.respond_to?(:wrapped) && ex.wrapped && each_cause(ex.wrapped, &)
+      return true if wrapped_got == true
+      return nil
+    end
+
+    # Run the given block for each cause (see +each_cause),
+    # returning the first exception the block returns +true+ for.
+    def find_cause(ex, &)
+      raise LocalJumpError unless block_given?
+      got = nil
+      each_cause(ex) do |cause|
+        if yield(cause) == true
+          got = cause
+          true
+        else
+          false
+        end
+      end
+      return got
+    end
+  end
+end

data/lib/webhookdb/fixtures/logged_webhooks.rb

@@ -35,6 +35,10 @@ module Webhookdb::Fixtures::LoggedWebhooks
     self.response_status = rand(400..599)
   end
 
+  decorator :truncated do |t=Time.now|
+    self.truncated_at = t
+  end
+
   decorator :with_organization do |org={}|
     org = Webhookdb::Fixtures.organization.create(org) unless org.is_a?(Webhookdb::Organization)
     self.organization = org

data/lib/webhookdb/fixtures/organization_error_handlers.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "faker"
+
+require "webhookdb/fixtures"
+
+module Webhookdb::Fixtures::OrganizationErrorHandlers
+  extend Webhookdb::Fixtures
+
+  fixtured_class Webhookdb::Organization::ErrorHandler
+
+  base :organization_error_handler do
+    self.url ||= Faker::Internet.url
+  end
+
+  before_saving do |instance|
+    instance.organization ||= Webhookdb::Fixtures.organization.create
+    instance
+  end
+end

data/lib/webhookdb/http.rb

@@ -2,6 +2,7 @@
 
 require "appydays/configurable"
 require "appydays/loggable/httparty_formatter"
+require "down/httpx"
 require "httparty"
 
 module Webhookdb::Http
@@ -11,7 +12,7 @@ module Webhookdb::Http
   end
 
   # Error raised when some API has rate limited us.
-  class BaseError < StandardError; end
+  class BaseError < Webhookdb::WebhookdbError; end
 
   class Error < BaseError
     attr_reader :response, :body, :uri, :status, :http_method
@@ -96,25 +97,38 @@ module Webhookdb::Http
     options[:log_level] = self.log_level
   end
 
-  # Convenience wrapper around Down that handles gzip.
+  # Convenience wrapper around Down so we can use our preferred implementation.
+  # See commit history for more info.
   # @return Array<Down::ChunkedIO, IO> Tuple
   def self.chunked_download(request_url, rewindable: false, **down_kw)
-    io = Down::NetHttp.open(request_url, rewindable:, **down_kw)
-    if io.data[:headers].fetch("Content-Encoding", "").include?("gzip")
-      # If the response is gzipped, Down doesn't handle it properly.
-      # Wrap it with gzip reader, and force the encoding to binary
-      # the server may send back a header like Content-Type: text/plain; UTF-8,
-      # so each line Down yields via #gets will have force_encoding('utf-8').
-      # https://github.com/janko/down/issues/87
-      io.instance_variable_set(:@encoding, "binary")
-      io = Zlib::GzipReader.wrap(io)
-    end
+    uri = URI(request_url)
+    raise URI::InvalidURIError, "#{request_url} must be an http/s url" unless ["http", "https"].include?(uri.scheme)
+    down_kw[:headers] ||= {}
+    down_kw[:headers]["User-Agent"] ||= self.user_agent
+    io = Down::Httpx.open(uri, rewindable:, **down_kw)
     return io
   end
+end
+
+class Down::Httpx
+  alias _original_response_error! response_error!
+  def response_error!(response)
+    # For some reason, Down's httpx backend uses TooManyRedirects for every status code...
+    raise Down::NotModified if response.status == 304
+    return self._original_response_error!(response)
+  end
+end
 
-  def self.gzipped?(string)
-    return false if string.length < 3
-    b = string[..2].bytes
-    return b[0] == 0x1f && b[1] == 0x8b
+class HTTPX::Response::Body
+  alias _original_initialize initialize
+  def initialize(*)
+    _original_initialize(*)
+    # If the encoding is an invalid one like 'utf8' vs 'utf-8', modify what's was in the charset.
+    # See https://github.com/HoneyryderChuck/httpx/issues/66
+    return unless @encoding.is_a?(String) && (md = @encoding.match(/^(utf)(\d+)$/))
+    @encoding = "#{md[1]}-#{md[2]}"
   end
 end
+
+# Not sure why, but Down uses this, loads the plugin, but the constant isn't defined.
+require "httpx/plugins/follow_redirects"

data/lib/webhookdb/icalendar.rb

@@ -7,20 +7,41 @@ module Webhookdb::Icalendar
   # If a manual backfill is attempted, direct customer to this url.
   DOCUMENTATION_URL = "https://docs.webhookdb.com/guides/icalendar/"
 
+  EVENT_REPLICATORS = ["icalendar_event_v1", "icalendar_event_v1_partitioned"].freeze
+
   include Appydays::Configurable
 
   configurable(:icalendar) do
-    # Do not store events older then this when syncing recurring events.
+    # Do not store events older than this when syncing recurring events.
     # Many icalendar feeds are misconfigured and this prevents enumerating 2000+ years of recurrence.
-    setting :oldest_recurring_event, "1990-01-01", convert: ->(s) { Date.parse(s) }
-    # Sync icalendar calendars only this often.
-    # Most services only update every day or so. Assume it takes 5s to sync each feed (request, parse, upsert).
-    # If you have 10,000 feeds, that is 50,000 seconds, or almost 14 hours of processing time,
-    # or two threads for 7 hours. The resyncs are spread out across the sync period
-    # (ie, no thundering herd every 8 hours), but it is still a good idea to sync as infrequently as possible.
+    setting :oldest_recurring_event, "2000-01-01", convert: ->(s) { Date.parse(s) }
+    # Calendars feeds are considered 'fresh' if they have been synced this long ago or less.
+    # Most services only update every day or so.
+    # Assume it takes 5s to sync each feed (request, parse, upsert).
+    # If you have 10,000 feeds, that is 50,000 seconds,
+    # or almost 14 hours of processing time, or two threads for 7 hours.
     setting :sync_period_hours, 6
+    # When stale feeds are scheduled for a resync,
+    # 'smear' them along this duration. Using 0 would immediately enqueue syncs of all stale feeds,
+    # which could saturate the job server. The number here means that feeds will be refreshed between every
+    # +sync_period_hours+ and +sync_period_hours+ + +sync_period_splay_hours+.
+    setting :sync_period_splay_hours, 1
+    # Number of threads for the 'precheck' threadpool, used when enqueing icalendar sync jobs.
+    # Since the precheck process uses many threads, but each check is resource-light and not latency-sensitive,
+    # we use a shared threadpool for it.
+    setting :precheck_feed_change_pool_size, 12
+
+    # Cancelled events that were last updated this long ago are deleted from the database.
+    setting :stale_cancelled_event_threshold_days, 20
+    # The stale row deleter job will look for rows this far before the threshold.
+    setting :stale_cancelled_event_lookback_days, 3
 
-    # Cancelled events that were cancelled this long ago are deleted from the database.
-    setting :stale_cancelled_event_threshold_days, 35
+    # The URL of the icalproxy server, if using one.
+    # See https://github.com/webhookdb/icalproxy for more info.
+    # Used to get property HTTP semantics for any icalendar feed, like Etag and HEAD requests.
+    setting :proxy_url, ""
+    # Api key of the icalproxy server, if using one.
+    # See https://github.com/webhookdb/icalproxy
+    setting :proxy_api_key, ""
   end
 end

data/lib/webhookdb/jobs/amigo_test_jobs.rb

@@ -103,7 +103,7 @@ class Webhookdb::Jobs::RetryChecker
       when "die"
         raise Amigo::Retry::Die
     else
-        raise Amigo::Retry::Die.new(attempts, interval)
+        raise Amigo::Retry::OrDie.new(attempts, interval)
     end
   end
 end

data/lib/webhookdb/jobs/backfill.rb

@@ -13,40 +13,36 @@ class Webhookdb::Jobs::Backfill
   on "webhookdb.backfilljob.run"
   sidekiq_options queue: "netout"
 
-  def dependent_queues
-    # This is really the lowest-priority job so always defer to other queues.
-    return super
-  end
+  # This is really the lowest-priority job so always defer to other queues.
 
   def _perform(event)
     begin
       bfjob = self.lookup_model(Webhookdb::BackfillJob, event.payload)
     rescue RuntimeError => e
-      self.logger.info "skipping_missing_backfill_job", error: e
+      self.set_job_tags(result: "skipped_missing_backfill_job", exception: e)
       return
     end
     sint = bfjob.service_integration
     bflock = bfjob.ensure_service_integration_lock
-    self.with_log_tags(sint.log_tags.merge(backfill_job_id: bfjob.opaque_id)) do
-      sint.db.transaction do
-        unless bflock.lock?
-          self.logger.info "skipping_locked_backfill_job"
-          bfjob.update(finished_at: Time.now)
-          break
-        end
-        bfjob.refresh
-        if bfjob.finished?
-          self.logger.info "skipping_finished_backfill_job"
-          break
-        end
-        begin
-          sint.replicator.backfill(bfjob)
-        rescue Webhookdb::Replicator::CredentialsMissing
-          # The credentials could have been cleared out, so just finish this job.
-          self.logger.info "skipping_backfill_job_without_credentials"
-          bfjob.update(finished_at: Time.now)
-          break
-        end
+    self.set_job_tags(sint.log_tags.merge(backfill_job_id: bfjob.opaque_id))
+    sint.db.transaction do
+      unless bflock.lock?
+        self.set_job_tags(result: "skipped_locked_backfill_job")
+        bfjob.update(finished_at: Time.now)
+        break
+      end
+      bfjob.refresh
+      if bfjob.finished?
+        self.set_job_tags(result: "skipped_finished_backfill_job")
+        break
+      end
+      begin
+        sint.replicator.backfill(bfjob)
+      rescue Webhookdb::Replicator::CredentialsMissing
+        # The credentials could have been cleared out, so just finish this job.
+        self.set_job_tags(result: "skipped_backfill_job_without_credentials")
+        bfjob.update(finished_at: Time.now)
+        break
       end
     end
   end

data/lib/webhookdb/jobs/create_mirror_table.rb

@@ -10,9 +10,8 @@ class Webhookdb::Jobs::CreateMirrorTable
 
   def _perform(event)
     sint = self.lookup_model(Webhookdb::ServiceIntegration, event)
-    self.with_log_tags(sint.log_tags) do
-      svc = Webhookdb::Replicator.create(sint)
-      svc.create_table(if_not_exists: true)
-    end
+    self.set_job_tags(sint.log_tags)
+    svc = Webhookdb::Replicator.create(sint)
+    svc.create_table(if_not_exists: true)
   end
 end

data/lib/webhookdb/jobs/deprecated_jobs.rb

@@ -14,6 +14,9 @@ Amigo::DeprecatedJobs.install(
   "Jobs::ConvertKitBroadcastBackfill",
   "Jobs::ConvertKitSubscriberBackfill",
   "Jobs::ConvertKitTagBackfill",
+  "Jobs::CustomerCreatedNotifyInternal",
+  "Jobs::OrganizationDatabaseMigrationNotifyStarted",
+  "Jobs::OrganizationDatabaseMigrationNotifyFinished",
   "Jobs::RssBackfillPoller",
   "Jobs::TwilioScheduledBackfill",
 )

data/lib/webhookdb/jobs/emailer.rb

@@ -9,6 +9,7 @@ class Webhookdb::Jobs::Emailer
   splay 5.seconds
 
   def _perform
-    Webhookdb::Message.send_unsent
+    sent = Webhookdb::Message.send_unsent
+    self.set_job_tags(sent_messages: sent.count)
   end
 end

data/lib/webhookdb/jobs/front_signalwire_message_channel_sync_inbound.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+require "webhookdb/jobs"
+
+# See +Webhookdb::Replicator::FrontSignalwireMessageChannelAppV1#alert_async_failed_signalwire_send+
+# for why we need to pull this into an async job.
+class Webhookdb::Jobs::FrontSignalwireMessageChannelSyncInbound
+  extend Webhookdb::Async::Job
+
+  def perform(service_integration_id, kwargs)
+    sint = self.lookup_model(Webhookdb::ServiceIntegration, service_integration_id)
+    sint.replicator.sync_front_inbound_message(**kwargs.symbolize_keys)
+  end
+end

data/lib/webhookdb/jobs/icalendar_delete_stale_cancelled_events.rb

@@ -9,10 +9,15 @@ class Webhookdb::Jobs::IcalendarDeleteStaleCancelledEvents
   cron "37 7 * * *" # Once a day
   splay 120
 
+  ADVISORY_LOCK_ID = 1_236_432_568
+
   def _perform
-    Webhookdb::ServiceIntegration.where(service_name: "icalendar_event_v1").each do |sint|
+    Webhookdb::ServiceIntegration.where(service_name: Webhookdb::Icalendar::EVENT_REPLICATORS).each do |sint|
       self.with_log_tags(sint.log_tags) do
-        sint.replicator.delete_stale_cancelled_events
+        sint.replicator.with_advisory_lock(ADVISORY_LOCK_ID) do
+          deleted_rows = sint.replicator.stale_row_deleter.run
+          self.set_job_tags("#{sint.organization.key}_#{sint.table_name}" => deleted_rows)
+        end
       end
     end
   end

data/lib/webhookdb/jobs/icalendar_enqueue_syncs.rb

@@ -5,27 +5,90 @@ require "webhookdb/jobs"
 
 # For every IcalendarCalendar row needing a sync (across all service integrations),
 # enqueue a +Webhookdb::Jobs::IcalendarSync+ job.
-# Jobs are 'splayed' over 1/4 of the configured calendar sync period (see +Webhookdb::Icalendar+)
-# to avoid a thundering herd.
+#
+# Because icalendars need to be synced periodically,
+# and there can be quite a lot, we have to be clever with how we sync them to both avoid syncing too often,
+# and especially avoid saturating workers with syncs.
+# We also have to handle workers running behind, the app being slow, etc.
+#
+# - This job runs every 30 minutes.
+# - It finds rows never synced,
+#   or haven't been synced in 8 hours (see +Webhookdb::Icalendar.sync_period_hours+
+#   for the actual value, we'll assume 8 hours for this doc).
+# - Enqueue a row sync sync job for between 1 second and 60 minutes from now
+#   (see +Webhookdb::Icalendar.sync_period_splay_hours+ for actual upper bound value).
+# - When the sync job runs, if the row has been synced less than 8 hours ago, the job noops.
+#
+# This design will lead to the same calendar being enqueued for a sync multiple times
+# (for a splay of 1 hour, and running this job every 30 minutes, about half the jobs in the queue will be duplicate).
+# This isn't a problem however, since the first sync will run but the duplicates will noop
+# since the row will be seen as recently synced.
+#
+# Importantly, if there is a thundering herd situation,
+# because there is a massive traunch of rows that need to be synced,
+# and it takes maybe 10 hours to sync rather than one,
+# the herd will be thinned and smeared over time as each row is synced.
+# There isn't much we can do for the initial herd (other than making sure
+# only some number of syncs are processing for a given org at a time)
+# but we won't keep getting thundering herds from the same calendars over time.
 class Webhookdb::Jobs::IcalendarEnqueueSyncs
   extend Webhookdb::Async::ScheduledJob
 
-  cron "*/30 * * * *" # Every 30 minutes
+  # See docs for explanation of why we run this often.
+  cron "*/30 * * * *"
   splay 30
 
   def _perform
-    max_splay = Webhookdb::Icalendar.sync_period_hours.hours.to_i / 4
+    self.advisory_lock.with_lock? do
+      self.__perform
+    end
+  end
+
+  # Just a random big number
+  LOCK_ID = 2_161_457_251_202_716_167
+
+  def advisory_lock
+    return Sequel::AdvisoryLock.new(Webhookdb::Customer.db, LOCK_ID)
+  end
+
+  def __perform
+    max_projected_out_seconds = Webhookdb::Icalendar.sync_period_splay_hours.hours.to_i
+    total_count = 0
+    threadpool = Concurrent::ThreadPoolExecutor.new(
+      name: "ical-precheck",
+      max_threads: Webhookdb::Icalendar.precheck_feed_change_pool_size,
+      min_threads: 1,
+      idletime: 40,
+      max_queue: 0,
+      fallback_policy: :caller_runs,
+      synchronous: false,
+    )
     Webhookdb::ServiceIntegration.dataset.where_each(service_name: "icalendar_calendar_v1") do |sint|
-      sint.replicator.admin_dataset do |ds|
-        sint.replicator.rows_needing_sync(ds).each do |row|
-          calendar_external_id = row.fetch(:external_id)
-          self.with_log_tags(sint.log_tags) do
-            splay = rand(1..max_splay)
-            enqueued_job_id = Webhookdb::Jobs::IcalendarSync.perform_in(splay, sint.id, calendar_external_id)
-            self.logger.debug("enqueued_icalendar_sync", calendar_external_id:, enqueued_job_id:)
+      sint_count = 0
+      self.with_log_tags(sint.log_tags) do
+        repl = sint.replicator
+        repl.admin_dataset do |ds|
+          row_ds = repl.
+            rows_needing_sync(ds).
+            order(:pk).
+            select(:external_id, :ics_url, :last_fetch_context)
+          row_ds.paged_each(rows_per_fetch: 500, cursor_name: "ical_enqueue_#{sint.id}_cursor") do |row|
+            threadpool.post do
+              break unless repl.feed_changed?(row)
+              calendar_external_id = row.fetch(:external_id)
+              perform_in = rand(1..max_projected_out_seconds)
+              enqueued_job_id = Webhookdb::Jobs::IcalendarSync.perform_in(perform_in, sint.id, calendar_external_id)
+              self.logger.debug("enqueued_icalendar_sync", calendar_external_id:, enqueued_job_id:, perform_in:)
+              sint_count += 1
+            end
           end
         end
       end
+      total_count += sint_count
+      self.set_job_tags("#{sint.organization.key}_#{sint.table_name}" => sint_count)
     end
+    threadpool.shutdown
+    threadpool.wait_for_termination
+    self.set_job_tags(total_enqueued: total_count)
   end
 end

data/lib/webhookdb/jobs/icalendar_enqueue_syncs_for_urls.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+
+class Webhookdb::Jobs::IcalendarEnqueueSyncsForUrls
+  extend Webhookdb::Async::Job
+
+  def perform(urls)
+    self.set_job_tags(url_count: urls.length)
+    row_count = 0
+    Webhookdb::ServiceIntegration.where(service_name: "icalendar_calendar_v1").each do |sint|
+      sint.replicator.admin_dataset do |ds|
+        affected_row_ext_ids = ds.where(ics_url: urls).select_map(:external_id)
+        affected_row_ext_ids.each do |ext_id|
+          Webhookdb::Jobs::IcalendarSync.perform_async(sint.id, ext_id)
+        end
+        row_count += affected_row_ext_ids.length
+      end
+    end
+    self.set_job_tags(result: "icalendar_enqueued_syncs", row_count:)
+  end
+end

data/lib/webhookdb/jobs/icalendar_sync.rb

@@ -1,23 +1,35 @@
 # frozen_string_literal: true
 
+require "amigo/semaphore_backoff_job"
+
 require "webhookdb/async/job"
 require "webhookdb/jobs"
 
 class Webhookdb::Jobs::IcalendarSync
   extend Webhookdb::Async::Job
+  include Amigo::SemaphoreBackoffJob
 
-  sidekiq_options retry: false
+  sidekiq_options retry: false, queue: "netout"
 
   def perform(sint_id, calendar_external_id)
     sint = self.lookup_model(Webhookdb::ServiceIntegration, sint_id)
-    self.with_log_tags(sint.log_tags.merge(calendar_external_id:)) do
-      row = sint.replicator.admin_dataset { |ds| ds[external_id: calendar_external_id] }
-      if row.nil?
-        self.logger.warn("icalendar_sync_row_miss", calendar_external_id:)
-        return
-      end
-      self.logger.debug("icalendar_sync_start")
-      sint.replicator.sync_row(row)
+    self.set_job_tags(sint.log_tags.merge(calendar_external_id:))
+    row = sint.replicator.admin_dataset { |ds| ds[external_id: calendar_external_id] }
+    if row.nil?
+      self.set_job_tags(result: "icalendar_sync_row_miss")
+      return
     end
+    self.logger.debug("icalendar_sync_start")
+    sint.replicator.sync_row(row)
+    self.set_job_tags(result: "icalendar_synced")
+  end
+
+  def before_perform(sint_id, *)
+    @sint = self.lookup_model(Webhookdb::ServiceIntegration, sint_id)
   end
+
+  def semaphore_key = "semaphore-icalendarsync-#{@sint.organization_id}"
+  def semaphore_size = @sint.organization.job_semaphore_size
+  def semaphore_expiry = 15.minutes
+  def semaphore_backoff = 60 + (rand * 30)
 end

data/lib/webhookdb/jobs/increase_event_handler.rb

@@ -8,13 +8,14 @@ class Webhookdb::Jobs::IncreaseEventHandler
   on "increase.*"
 
   def _perform(event)
+    self.set_job_tags(increase_event_name: event.name)
     case event.name
       when "increase.oauth_connection.deactivated"
         conn_id = event.payload[0].fetch("associated_object_id")
-        self.logger.info("increase_oauth_disconnect", oauth_connection_id: conn_id)
+        self.set_job_tags(result: "increase_oauth_disconnected", oauth_connection_id: conn_id)
         Webhookdb::Oauth::IncreaseProvider.disconnect_oauth(conn_id)
       else
-        self.logger.info("increase_event_noop")
+        self.set_job_tags(result: "increase_event_noop")
     end
   end
 end

data/lib/webhookdb/jobs/{logged_webhook_replay.rb → logged_webhooks_replay.rb}

@@ -10,8 +10,10 @@ class Webhookdb::Jobs::LoggedWebhooksReplay
 
   def _perform(event)
     lwh = self.lookup_model(Webhookdb::LoggedWebhook, event)
-    self.with_log_tags(service_integration_opaque_id: lwh.service_integration_opaque_id) do
-      lwh.retry_one(truncate_successful: true)
-    end
+    self.set_job_tags(
+      logged_webhook_id: lwh.id,
+      service_integration_opaque_id: lwh.service_integration_opaque_id,
+    )
+    lwh.retry_one(truncate_successful: true)
   end
 end

data/lib/webhookdb/jobs/message_dispatched.rb

@@ -9,6 +9,7 @@ class Webhookdb::Jobs::MessageDispatched
 
   def _perform(event)
     delivery = self.lookup_model(Webhookdb::Message::Delivery, event)
+    self.set_job_tags(delivery_id: delivery.id, to: delivery.to)
     Webhookdb::Idempotency.once_ever.under_key("message-dispatched-#{delivery.id}") do
       delivery.send!
     end

data/lib/webhookdb/jobs/model_event_system_log_tracker.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+require "webhookdb/messages/invite"
+
+class Webhookdb::Jobs::ModelEventSystemLogTracker
+  extend Webhookdb::Async::Job
+
+  on "webhookdb.*"
+
+  def _perform(event)
+    self.set_job_tags(event_name: event.name)
+    case event.name
+        when "webhookdb.customer.created"
+          self.alert_customer_created(event)
+        when "webhookdb.organization.created"
+          self.alert_org_created(event)
+        when "webhookdb.serviceintegration.created"
+          self.alert_sint_created(event)
+        when "webhookdb.serviceintegration.destroyed"
+          self.alert_sint_destroyed(event)
+      else
+          self.set_job_tags(result: "noop")
+      end
+  end
+
+  def create_event(title, body, link)
+    Webhookdb::SystemLogEvent.create(
+      at: Time.now,
+      title:,
+      body:,
+      link:,
+    )
+  end
+
+  def alert_customer_created(event)
+    customer = self.lookup_model(Webhookdb::Customer, event)
+    Webhookdb::DeveloperAlert.new(
+      subsystem: "Customer Created",
+      emoji: ":hook:",
+      fallback: "New customer created: #{customer.inspect}",
+      fields: [
+        {title: "Id", value: customer.id, short: true},
+        {title: "Email", value: customer.email, short: true},
+        {title: "Link", value: customer.admin_link},
+      ],
+    ).emit
+    create_event("Customer Created", customer.email, customer.admin_link)
+    self.set_job_tags(result: "created_customer", email: customer.email)
+  end
+
+  def alert_org_created(event)
+    org = self.lookup_model(Webhookdb::Organization, event)
+    Webhookdb::DeveloperAlert.new(
+      subsystem: "Organization Created",
+      emoji: ":office:",
+      fallback: "Organization created: #{org.inspect}",
+      fields: [
+        {title: "Id", value: org.id, short: true},
+        {title: "Email", value: org.name, short: true},
+        {title: "Link", value: org.admin_link},
+      ],
+    ).emit
+    create_event("Organization Created", "#{org.name} (#{org.key})", org.admin_link)
+    self.set_job_tags(result: "created_organization", key: org.key)
+  end
+
+  def alert_sint_created(event)
+    sint = self.lookup_model(Webhookdb::ServiceIntegration, event)
+    Webhookdb::DeveloperAlert.new(
+      subsystem: "Integration Created",
+      emoji: ":fax:",
+      fallback: "Service Integration #{sint.service_name} (#{sint.opaque_id}) created",
+      fields: [
+        {title: "Id", value: sint.opaque_id, short: true},
+        {title: "Service", value: sint.service_name, short: true},
+        {title: "Table", value: sint.table_name, short: true},
+        {title: "Org Name", value: sint.organization.name, short: true},
+        {title: "Link", value: sint.admin_link},
+      ],
+    ).emit
+    create_event(
+      "Integration Created",
+      "#{sint.service_name} (#{sint.opaque_id}) created in #{sint.organization.name}",
+      sint.admin_link,
+    )
+    self.set_job_tags(result: "created_service_integration", opaque_id: sint.opaque_id, service: sint.service_name)
+  end
+
+  def alert_sint_destroyed(event)
+    pl = event.payload[1].symbolize_keys
+    org = Webhookdb::Organization[pl[:organization_id]]
+    Webhookdb::DeveloperAlert.new(
+      subsystem: "Integration Deleted",
+      emoji: ":funeral_urn:",
+      fallback: "Service Integration #{pl[:service_name]} (#{pl[:opaque_id]}) deleted",
+      fields: [
+        {title: "Id", value: pl[:opaque_id], short: true},
+        {title: "Service", value: pl[:service_name], short: true},
+        {title: "Table", value: pl[:table_name], short: true},
+        {title: "Org Name", value: org.name, short: true},
+        {title: "Link", value: org.admin_link},
+      ],
+    ).emit
+    create_event(
+      "Integration Deleted",
+      "#{pl[:service_name]} (#{pl[:opaque_id]}) deleted from #{org.name}",
+      org.admin_link,
+    )
+    self.set_job_tags(result: "destroyed_service_integration", opaque_id: pl[:oparque_id], service: pl[:service_name])
+  end
+end