webhookdb 1.3.1 → 1.5.0

data/lib/webhookdb/jobs/monitor_metrics.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/scheduled_job"
+require "webhookdb/jobs"
+
+# Log out some metrics every minute.
+class Webhookdb::Jobs::MonitorMetrics
+  extend Webhookdb::Async::ScheduledJob
+
+  cron "* * * * *" # Every 1 minute
+  splay 0
+
+  def _perform
+    opts = {}
+    max_size = 0
+    max_latency = 0
+    Sidekiq::Queue.all.each do |q|
+      size = q.size
+      latency = q.latency
+      max_size = [max_size, size].max
+      max_latency = [max_latency, latency].max
+      opts["#{q.name}_size"] = size
+      opts["#{q.name}_latency"] = latency
+    end
+    opts[:max_size] = max_size
+    opts[:max_latency] = max_latency
+    self.set_job_tags(action: "metrics_monitor_queue", **opts)
+  end
+end

data/lib/webhookdb/jobs/organization_database_migration_notify.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+require "webhookdb/messages/org_database_migration_finished"
+require "webhookdb/messages/org_database_migration_started"
+
+class Webhookdb::Jobs::OrganizationDatabaseMigrationNotify
+  extend Webhookdb::Async::Job
+
+  on "webhookdb.organization.databasemigration.updated"
+
+  def _perform(event)
+    dbm = self.lookup_model(Webhookdb::Organization::DatabaseMigration, event)
+    self.set_job_tags(database_migration_id: dbm.id, organization: dbm.organization.key)
+    case event.payload[1]
+      when changed(:started_at, from: nil)
+        Webhookdb::Idempotency.once_ever.under_key("org-dbmigration-start-#{dbm.id}") do
+          msg = Webhookdb::Messages::OrgDatabaseMigrationStarted.new(dbm)
+          dbm.organization.admin_customers.each { |c| msg.dispatch_email(c) }
+        end
+        self.set_job_tags(result: "started_message_sent")
+      when changed(:finished_at, from: nil)
+        Webhookdb::Idempotency.once_ever.under_key("org-dbmigration-finish-#{dbm.id}") do
+          msg = Webhookdb::Messages::OrgDatabaseMigrationFinished.new(dbm)
+          dbm.organization.admin_customers.each { |c| msg.dispatch_email(c) }
+        end
+        self.set_job_tags(result: "finished_message_sent")
+      else
+        self.set_job_tags(result: "noop")
+    end
+  end
+end

data/lib/webhookdb/jobs/organization_database_migration_run.rb

@@ -11,14 +11,12 @@ class Webhookdb::Jobs::OrganizationDatabaseMigrationRun
 
   def _perform(event)
     dbm = self.lookup_model(Webhookdb::Organization::DatabaseMigration, event)
-    self.with_log_tags(
-      organization_id: dbm.organization.id,
-      organization_name: dbm.organization.name,
-      organization_database_migration_id: dbm.id,
-    ) do
+    self.set_job_tags(organization: dbm.organization.key, database_migration_id: dbm.id)
+    begin
       dbm.migrate
+      self.set_job_tags(result: "migration_finished")
     rescue Webhookdb::Organization::DatabaseMigration::MigrationAlreadyFinished
-      self.logger.warn("org_database_migration_already_finished")
+      self.set_job_tags(result: "migration_already_finished")
     end
   end
 end

data/lib/webhookdb/jobs/organization_error_handler_dispatch.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+
+class Webhookdb::Jobs::OrganizationErrorHandlerDispatch
+  extend Webhookdb::Async::Job
+
+  sidekiq_options queue: "netout"
+
+  def perform(error_handler_id, payload)
+    eh = self.lookup_model(Webhookdb::Organization::ErrorHandler, error_handler_id)
+    self.set_job_tags(error_handler_id: eh.id, **eh.organization.log_tags)
+    begin
+      eh.dispatch(payload)
+      self.set_job_tags(result: "success")
+    rescue StandardError => e
+      # Don't bother logging these errors out
+      self.set_job_tags(result: "error")
+      self.logger.debug("organization_error_handler_post_error", error: e)
+      raise Amigo::Retry::OrDie.new(
+        Webhookdb::Organization::Alerting.error_handler_retries,
+        Webhookdb::Organization::Alerting.error_handler_retry_interval,
+      )
+    end
+  end
+end

data/lib/webhookdb/jobs/prepare_database_connections.rb

@@ -12,6 +12,7 @@ class Webhookdb::Jobs::PrepareDatabaseConnections
 
   def _perform(event)
     org = self.lookup_model(Webhookdb::Organization, event)
+    self.set_job_tags(organization: org.key)
     org.db.transaction do
       # If creating the public host fails, we end up with an orphaned database,
       # but that's not a big deal- we can eventually see it's empty/unlinked and drop it.

data/lib/webhookdb/jobs/process_webhook.rb

@@ -23,18 +23,17 @@ class Webhookdb::Jobs::ProcessWebhook
   end
 
   def _perform(event)
-    self.with_log_tags(@sint.log_tags) do
-      kw = event.payload[1].symbolize_keys
-      svc = Webhookdb::Replicator.create(@sint)
-      # kwargs contains: :headers, :body, :request_path, :request_method
-      req = Webhookdb::Replicator::WebhookRequest.new(
-        body: kw.fetch(:body),
-        headers: kw.fetch(:headers),
-        path: kw.fetch(:request_path),
-        method: kw.fetch(:request_method),
-      )
-      svc.upsert_webhook(req)
-    end
+    self.set_job_tags(@sint.log_tags)
+    kw = event.payload[1].symbolize_keys
+    svc = Webhookdb::Replicator.create(@sint)
+    # kwargs contains: :headers, :body, :request_path, :request_method
+    req = Webhookdb::Replicator::WebhookRequest.new(
+      body: kw.fetch(:body),
+      headers: kw.fetch(:headers),
+      path: kw.fetch(:request_path),
+      method: kw.fetch(:request_method),
+    )
+    svc.upsert_webhook(req)
   end
 
   def semaphore_key

data/lib/webhookdb/jobs/renew_watch_channel.rb

@@ -1,24 +1,24 @@
 # frozen_string_literal: true
 
+require "amigo/queue_backoff_job"
 require "webhookdb/async/job"
 require "webhookdb/jobs"
 
 # Generic helper to renew watch channels, enqueued by replicator-specific jobs
 # like RenewGoogleWatchChannels.
-# Must be emitted with [service integration id, {row_pk:, expirng_before:}]
+# Must be emitted with [service_integration_id, {row_pk:, expirng_before:}]
 # Calls #renew_watch_channel(row_pk:, expiring_before:).
 class Webhookdb::Jobs::RenewWatchChannel
   extend Webhookdb::Async::Job
+  include Amigo::QueueBackoffJob
 
-  on "webhookdb.serviceintegration.renewwatchchannel"
+  sidekiq_options queue: "netout"
 
-  def _perform(event)
-    sint = self.lookup_model(Webhookdb::ServiceIntegration, event)
-    self.with_log_tags(sint.log_tags) do
-      opts = event.payload[1]
-      row_pk = opts.fetch("row_pk")
-      expiring_before = Time.parse(opts.fetch("expiring_before"))
-      sint.replicator.renew_watch_channel(row_pk:, expiring_before:)
-    end
+  def perform(service_integration_id, renew_watch_criteria)
+    sint = self.lookup_model(Webhookdb::ServiceIntegration, service_integration_id)
+    self.set_job_tags(sint.log_tags)
+    row_pk = renew_watch_criteria.fetch("row_pk")
+    expiring_before = Time.parse(renew_watch_criteria.fetch("expiring_before"))
+    sint.replicator.renew_watch_channel(row_pk:, expiring_before:)
   end
 end

data/lib/webhookdb/jobs/replication_migration.rb

@@ -2,18 +2,21 @@
 
 # See Organization::enqueue_migrate_all_replication_tables for more info.
 class Webhookdb::Jobs::ReplicationMigration
-  include Sidekiq::Worker
+  extend Webhookdb::Async::Job
 
   def perform(org_id, target_release_created_at)
     (org = Webhookdb::Organization[org_id]) or raise "Organization[#{org_id}] does not exist"
     target_rca = Time.parse(target_release_created_at)
     current_rca = Time.parse(Webhookdb::RELEASE_CREATED_AT)
+    self.set_job_tags(organization_id: org_id, target_release_created_at:)
     if target_rca == current_rca
       self.class.migrate_org(org)
+      self.set_job_tags(result: "ran_replication_migration_job")
     elsif target_rca > current_rca
       self.class.perform_in(1, org_id, target_release_created_at)
+      self.set_job_tags(result: "reenqueued_replication_migration_job")
     else
-      self.logger.warn("stale_replication_migration_job", target_release_created_at:)
+      self.set_job_tags(result: "stale_replication_migration_job")
     end
   end
 

data/lib/webhookdb/jobs/reset_code_create_dispatch.rb

@@ -10,13 +10,12 @@ class Webhookdb::Jobs::ResetCodeCreateDispatch
 
   def _perform(event)
     code = self.lookup_model(Webhookdb::Customer::ResetCode, event)
+    self.set_job_tags(code_id: code.id, customer: code.customer.email, transport: code.transport)
     Webhookdb::Idempotency.once_ever.under_key("reset-code-#{code.customer_id}-#{code.id}") do
       msg = Webhookdb::Messages::Verification.new(code)
       case code.transport
         when "email"
           msg.dispatch_email(code.customer)
-      else
-          raise "Unknown transport for #{code.inspect}"
       end
     end
   end

data/lib/webhookdb/jobs/scheduled_backfills.rb

@@ -51,9 +51,9 @@ module Webhookdb::Jobs
         Webhookdb::Github.activity_cron_expression, 30.seconds, false,
       ),
       Spec.new(
-        # I think we can get rid of this once we're more confident webhooks are working reliably.
+        # This incremental sync is a backstop for any missed webhooks.
         "IntercomScheduledBackfill", "intercom_marketplace_root_v1",
-        "46 4 * * *", 0, false, true,
+        "46 4 * * *", 0, true, true,
       ),
       Spec.new(
         "AtomSingleFeedPoller", "atom_single_feed_v1",

data/lib/webhookdb/jobs/send_invite.rb

@@ -9,7 +9,8 @@ class Webhookdb::Jobs::SendInvite
   on "webhookdb.organizationmembership.invite"
 
   def _perform(event)
-    membership = self.lookup_model(Webhookdb::OrganizationMembership, event)
-    Webhookdb::Messages::Invite.new(membership).dispatch(membership.customer)
+    m = self.lookup_model(Webhookdb::OrganizationMembership, event)
+    self.set_job_tags(membership_id: m.id, organization: m.organization.key, customer: m.customer.email)
+    Webhookdb::Messages::Invite.new(m).dispatch(m.customer)
   end
 end

data/lib/webhookdb/jobs/send_test_webhook.rb

@@ -14,9 +14,7 @@ class Webhookdb::Jobs::SendTestWebhook
   # we don't want to retry and randomly send a payload later.
   sidekiq_options retry: false
 
-  def dependent_queues
-    return ["critical"]
-  end
+  def dependent_queues = ["critical"]
 
   def _perform(event)
     webhook_sub = self.lookup_model(Webhookdb::WebhookSubscription, event)

data/lib/webhookdb/jobs/send_webhook.rb

@@ -10,11 +10,10 @@ class Webhookdb::Jobs::SendWebhook
 
   def _perform(event)
     sint = self.lookup_model(Webhookdb::ServiceIntegration, event)
-    self.with_log_tags(sint.log_tags) do
-      sint.all_webhook_subscriptions_dataset.to_notify.each do |sub|
-        payload = {service_name: sint.service_name, table_name: sint.table_name, **event.payload[1]}
-        sub.enqueue_delivery(payload)
-      end
+    self.set_job_tags(sint.log_tags)
+    sint.all_webhook_subscriptions_dataset.to_notify.each do |sub|
+      payload = {service_name: sint.service_name, table_name: sint.table_name, **event.payload[1]}
+      sub.enqueue_delivery(payload)
     end
   end
 end

data/lib/webhookdb/jobs/stale_row_deleter.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+
+# Run the +stale_row_deleter+ for each service integration
+# which match the given where/exclude clauses.
+# This is generally used to delete old stale rows in the backend
+# (by passing initial: true) when a new stale row deleter is deployed.
+class Webhookdb::Jobs::StaleRowDeleter
+  extend Webhookdb::Async::Job
+
+  def perform(opts={})
+    opts = opts.deep_symbolize_keys
+    opts[:where] ||= {}
+    opts[:exclude] ||= {}
+    opts[:initial] ||= false
+    ds = Webhookdb::ServiceIntegration.dataset
+    ds = ds.where(opts[:where]) if opts[:where]
+    ds = ds.exclude(opts[:exclude]) if opts[:exclude]
+    self.set_job_tags(dataset: ds.sql)
+    count = 0
+    ds.each do |sint|
+      self.with_log_tags(sint.log_tags) do
+        d = sint.replicator.stale_row_deleter
+        opts[:initial] ? d.run_initial : d.run
+        count += 1
+      end
+    end
+    self.set_job_tags(run_count: count)
+  end
+end

data/lib/webhookdb/jobs/sync_target_enqueue_scheduled.rb

@@ -9,8 +9,11 @@ class Webhookdb::Jobs::SyncTargetEnqueueScheduled
   splay 0
 
   def _perform
+    count = 0
     Webhookdb::SyncTarget.due_for_sync(as_of: Time.now).select(:id, :period_seconds).each do |st|
+      count += 1
       Webhookdb::Jobs::SyncTargetRunSync.perform_in(st.jitter, st.id)
     end
+    self.set_job_tags(enqueued_count: count)
   end
 end

data/lib/webhookdb/jobs/sync_target_run_sync.rb

@@ -9,9 +9,7 @@ class Webhookdb::Jobs::SyncTargetRunSync
 
   sidekiq_options queue: "netout"
 
-  def dependent_queues
-    return ["critical"]
-  end
+  def dependent_queues = ["critical"]
 
   def perform(sync_target_id)
     stgt = Webhookdb::SyncTarget[sync_target_id]
@@ -19,22 +17,18 @@ class Webhookdb::Jobs::SyncTargetRunSync
       # A sync target may be enqueued, but destroyed before the sync runs.
       # If so, log a warning. We see this on staging a lot,
       # but it does happen on production too, and should be expected.
-      self.logger.info("missing_sync_target", sync_target_id:)
+      self.set_job_tags(result: "missing_sync_target", sync_target_id:)
       return
     end
-    self.with_log_tags(
-      sync_target_id: stgt.id,
-      sync_target_connection_url: stgt.displaysafe_connection_url,
-      sync_target_service_integration_service: stgt.service_integration.service_name,
-      sync_target_service_integration_table: stgt.service_integration.table_name,
-    ) do
-      stgt.run_sync(now: Time.now)
+    self.set_job_tags(stgt.log_tags)
+    begin
+      started = Time.now
+      stgt.run_sync(now: started)
+      self.set_job_tags(result: "sync_target_synced", synced_at_of: started)
     rescue Webhookdb::SyncTarget::SyncInProgress
-      Webhookdb::Idempotency.every(30.seconds).in_memory.under_key("sync_target_in_progress-#{stgt.id}") do
-        self.logger.info("sync_target_already_in_progress")
-      end
+      self.set_job_tags(result: "sync_target_already_in_progress")
     rescue Webhookdb::SyncTarget::Deleted
-      self.logger.info("sync_target_deleted")
+      self.set_job_tags(result: "sync_target_deleted")
     end
   end
 end

data/lib/webhookdb/jobs/{webhook_subscription_delivery_attempt.rb → webhook_subscription_delivery_event.rb}

@@ -12,18 +12,15 @@ class Webhookdb::Jobs::WebhookSubscriptionDeliveryEvent
 
   sidekiq_options queue: "netout"
 
-  def dependent_queues
-    return ["critical"]
-  end
+  def dependent_queues = ["critical"]
 
   def perform(delivery_id)
     delivery = Webhookdb::WebhookSubscription::Delivery[delivery_id]
-    Webhookdb::Async::JobLogger.with_log_tags(
+    Webhookdb::Async::JobLogger.set_job_tags(
       webhook_subscription_delivery_id: delivery.id,
       webhook_subscription_id: delivery.webhook_subscription_id,
-      organization_key: delivery.webhook_subscription.fetch_organization,
-    ) do
-      delivery.attempt_delivery
-    end
+      organization: delivery.webhook_subscription.fetch_organization,
+    )
+    delivery.attempt_delivery
   end
 end

data/lib/webhookdb/liquid/expose.rb

@@ -24,4 +24,4 @@ class Webhookdb::Liquid::Expose < Liquid::Block
   end
 end
 
-Liquid::Template.register_tag("expose", Webhookdb::Liquid::Expose)
+Liquid::Environment.default.register_tag("expose", Webhookdb::Liquid::Expose)

data/lib/webhookdb/liquid/filters.rb

@@ -13,4 +13,4 @@ module Webhookdb::Liquid::Filters
   end
 end
 
-Liquid::Template.register_filter(Webhookdb::Liquid::Filters)
+Liquid::Environment.default.register_filter(Webhookdb::Liquid::Filters)

data/lib/webhookdb/liquid/partial.rb

@@ -6,7 +6,7 @@ require "liquid"
 class Webhookdb::Liquid::Partial < Liquid::Include
   def initialize(tag_name, name, options)
     name = "'partials/#{Regexp.last_match(1)}'" if name =~ /['"]([a-z0-9_]+)['"]/
-    super(tag_name, name, options)
+    super
   end
 end
-Liquid::Template.register_tag("partial", Webhookdb::Liquid::Partial)
+Liquid::Environment.default.register_tag("partial", Webhookdb::Liquid::Partial)

data/lib/webhookdb/logged_webhook/resilient.rb

@@ -12,10 +12,10 @@ class Webhookdb::LoggedWebhook::Resilient
     str_payload = JSON.dump(kwargs)
     self.database_urls.each do |url|
       next unless self.write_to(url, service_integration_opaque_id, str_payload)
-      self.logger.warn "resilient_insert_handled", error: e, **self._dburl_log_kwargs(url)
+      self.logger.warn "resilient_insert_handled", self._dburl_log_kwargs(url), e
       return true
     end
-    self.logger.error "resilient_insert_unhandled", error: e, logged_webhook_kwargs: kwargs
+    self.logger.error "resilient_insert_unhandled", {logged_webhook_kwargs: kwargs}, e
     raise
   end
 
@@ -37,7 +37,7 @@ class Webhookdb::LoggedWebhook::Resilient
     end
     return true
   rescue StandardError => e
-    self.logger.debug "resilient_insert_failure", error: e, **self._dburl_log_kwargs(dburl)
+    self.logger.debug "resilient_insert_failure", self._dburl_log_kwargs(dburl), e
     return false
   end
 

data/lib/webhookdb/logged_webhook.rb

@@ -88,13 +88,26 @@ class Webhookdb::LoggedWebhook < Webhookdb::Postgres::Model(:logged_webhooks)
     unowned = self.where(organization_id: nil)
     successes = owned.where { response_status < 400 }
     failures = owned.where { response_status >= 400 }
+    # NOTE: This code is tightly coupled with indices created in 050_logged_webhooks_indices.rb
+    # We create a separate index for each operation; the indices (5 in total) cover the full combination of:
+    # - rows without an organization (idx 1)
+    # - rows with an organization
+    #   - rows already truncated
+    #     - rows with status < 400 (idx 2)
+    #     - rows with status >= 400 (idx 3)
+    #   - rows not truncated
+    #     - rows with status < 400 (idx 4)
+    #     - rows with status >= 400 (idx 5)
+    # Note that we only delete already-truncated rows so we can keep our indices smaller;
+    # since deletion ages are always older than truncation ages, this should not be a problem.
+
     # Delete old unowned
     unowned.where { inserted_at < now - DELETE_UNOWNED }.delete
     # Delete successes first so they don't have to be truncated
-    successes.where { inserted_at < now - DELETE_SUCCESSES }.delete
+    successes.where { inserted_at < now - DELETE_SUCCESSES }.exclude(truncated_at: nil).delete
     self.truncate_dataset(successes.where { inserted_at < now - TRUNCATE_SUCCESSES })
     # Delete failures
-    failures.where { inserted_at < now - DELETE_FAILURES }.delete
+    failures.where { inserted_at < now - DELETE_FAILURES }.exclude(truncated_at: nil).delete
     self.truncate_dataset(failures.where { inserted_at < now - TRUNCATE_FAILURES })
   end
 
@@ -145,6 +158,7 @@ class Webhookdb::LoggedWebhook < Webhookdb::Postgres::Model(:logged_webhooks)
   end
 
   def self.truncate_dataset(ds)
+    ds = ds.where(truncated_at: nil)
     return ds.update(request_body: "", request_headers: "{}", truncated_at: Time.now)
   end
 

data/lib/webhookdb/message/email_transport.rb

@@ -12,7 +12,7 @@ class Webhookdb::Message::EmailTransport < Webhookdb::Message::Transport
   register_transport(:email)
 
   configurable(:email) do
-    setting :allowlist, ["*@lithic.tech", "*@webhookdb.com"], convert: ->(s) { s.split }
+    setting :allowlist, ["*@lithic.tech", "*@webhookdb.com"], convert: lambda(&:split)
     setting :from, "WebhookDB <hello@webhookdb.com>"
 
     setting :smtp_host, "localhost"

data/lib/webhookdb/message/transport.rb

@@ -5,7 +5,7 @@ require "webhookdb/message"
 class Webhookdb::Message::Transport
   extend Webhookdb::MethodUtilities
 
-  class Error < StandardError; end
+  class Error < Webhookdb::WebhookdbError; end
   class UndeliverableRecipient < Error; end
 
   singleton_attr_reader :transports

data/lib/webhookdb/message.rb

@@ -29,8 +29,8 @@ module Webhookdb::Message
 
   configurable(:messages) do
     after_configured do
-      Liquid::Template.error_mode = :strict
-      Liquid::Template.file_system = Liquid::LocalFileSystem.new(DATA_DIR, "%s.liquid")
+      Liquid::Environment.default.error_mode = :strict
+      Liquid::Environment.default.file_system = Liquid::LocalFileSystem.new(DATA_DIR, "%s.liquid")
     end
   end
 
@@ -73,6 +73,7 @@ module Webhookdb::Message
       "environment" => Webhookdb::Message::EnvironmentDrop.new,
       "app_url" => Webhookdb.app_url,
     )
+    drops = self.unify_drops_encoding(drops)
 
     content_tmpl = Liquid::Template.parse(template_file.read)
     # The 'expose' drop smashes data into the register.
@@ -101,13 +102,63 @@ module Webhookdb::Message
     return Rendering.new(content, lctx.registers)
   end
 
+  # Handle encoding in liquid drop string values that would likely crash message rendering.
+  #
+  # If there is a mixed character encoding of string values in a liquid drop,
+  # such as when handling user-supplied values, force all strings into UTF-8.
+  #
+  # This is needed because the way Ruby does encoding coercion when parsing input
+  # which does not declare an encoding, such as a file or especially an HTTP response.
+  # Ruby will:
+  # - Use ASCII if the values fit into 7 bits
+  # - Use ASCII-8BIT if the values fit into 8 bits (128 to 255)
+  # - Otherwise, use UTF-8.
+  #
+  # The actual rules are more complex, but this is common enough.
+  #
+  # While ASCII encoding can be used as UTF-8, ASCII-8BIT cannot.
+  # So adding `(ascii-8bit string) + (utf-8 string)` will error with an
+  # `Encoding::CompatibilityError`.
+  #
+  # Instead, if we see a series of liquid drop string values
+  # with different encodings, force them all to be UTF-8.
+  # This can result in some unexpected behavior,
+  # but it should be fine, since you'd only see it with unexpected input
+  # (all valid inputs should be UTF-8).
+  #
+  # @param [Hash] drops
+  # @return [Hash]
+  def self.unify_drops_encoding(drops)
+    return drops if drops.empty?
+    seen_enc = nil
+    force_enc = false
+    drops.each_value do |v|
+      next unless v.respond_to?(:encoding)
+      seen_enc ||= v.encoding
+      next if seen_enc == v.encoding
+      force_enc = true
+      break
+    end
+    return drops unless force_enc
+    utf8 = Encoding.find("UTF-8")
+    result = drops.each_with_object({}) do |(k, v), memo|
+      if v.respond_to?(:encoding) && v.encoding != utf8
+        v2 = v.encode(utf8, invalid: :replace, undef: :replace, replace: "?")
+        memo[k] = v2
+      else
+        memo[k] = v
+      end
+    end
+    return result
+  end
+
   def self.send_unsent
     Webhookdb::Message::Delivery.unsent.each(&:send!)
   end
 
-  class InvalidTransportError < StandardError; end
+  class InvalidTransportError < Webhookdb::ProgrammingError; end
 
-  class MissingTemplateError < StandardError; end
+  class MissingTemplateError < Webhookdb::ProgrammingError; end
 
   # Presents a homogeneous interface for a given 'to' value (email vs. customer, for example).
   # .to will always be a plain object, and .customer will be a +Webhookdb::Customer+ if present.

data/lib/webhookdb/messages/error_generic_backfill.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "webhookdb/message/template"
+
+class Webhookdb::Messages::ErrorGenericBackfill < Webhookdb::Message::Template
+  def self.fixtured(_recipient)
+    sint = Webhookdb::Fixtures.service_integration.create
+    return self.new(
+      sint,
+      response_status: 422,
+      request_url: "https://whdbtest.signalwire.com/2010-04-01/Accounts/projid/Messages.json",
+      request_method: "POST",
+      response_body: "Unauthorized",
+    )
+  end
+
+  attr_accessor :service_integration
+
+  def initialize(service_integration, request_url:, request_method:, response_status:, response_body:)
+    @service_integration = service_integration
+    @request_url = request_url
+    @request_method = request_method
+    @response_status = response_status
+    @response_body = response_body
+    super()
+  end
+
+  def signature
+    # Only alert on the backfill once a day
+    return "msg-#{self.full_template_name}-sint:#{@service_integration.id}"
+  end
+
+  def template_folder = "errors"
+  def template_name = "generic_backfill"
+
+  def liquid_drops
+    return super.merge(
+      friendly_name: @service_integration.replicator.descriptor.resource_name_singular,
+      service_name: @service_integration.service_name,
+      opaque_id: @service_integration.opaque_id,
+      request_method: @request_method,
+      request_url: @request_url,
+      response_status: @response_status,
+      response_body: @response_body,
+    )
+  end
+end

data/lib/webhookdb/messages/error_icalendar_fetch.rb

@@ -9,6 +9,8 @@ class Webhookdb::Messages::ErrorIcalendarFetch < Webhookdb::Message::Template
                     response_status: 403, request_url: "/foo", request_method: "GET", response_body: "hi",)
   end
 
+  attr_accessor :service_integration
+
   def initialize(service_integration, external_calendar_id, request_url:, request_method:, response_status:,
     response_body:)
     @service_integration = service_integration
@@ -36,6 +38,9 @@ class Webhookdb::Messages::ErrorIcalendarFetch < Webhookdb::Message::Template
       response_status: @response_status,
       response_body: @response_body,
       external_calendar_id: @external_calendar_id,
+      webhook_endpoint: @service_integration.replicator.webhook_endpoint,
+      org_name: @service_integration.organization.name,
+      org_key: @service_integration.organization.key,
     )
   end
 end