webhookdb 1.3.1 → 1.4.0

data/lib/webhookdb/messages/error_generic_backfill.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "webhookdb/message/template"
+
+class Webhookdb::Messages::ErrorGenericBackfill < Webhookdb::Message::Template
+  def self.fixtured(_recipient)
+    sint = Webhookdb::Fixtures.service_integration.create
+    return self.new(
+      sint,
+      response_status: 422,
+      request_url: "https://whdbtest.signalwire.com/2010-04-01/Accounts/projid/Messages.json",
+      request_method: "POST",
+      response_body: "Unauthorized",
+    )
+  end
+
+  def initialize(service_integration, request_url:, request_method:, response_status:, response_body:)
+    @service_integration = service_integration
+    @request_url = request_url
+    @request_method = request_method
+    @response_status = response_status
+    @response_body = response_body
+    super()
+  end
+
+  def signature
+    # Only alert on the backfill once a day
+    return "msg-#{self.full_template_name}-sint:#{@service_integration.id}"
+  end
+
+  def template_folder = "errors"
+  def template_name = "generic_backfill"
+
+  def liquid_drops
+    return super.merge(
+      friendly_name: @service_integration.replicator.descriptor.resource_name_singular,
+      service_name: @service_integration.service_name,
+      opaque_id: @service_integration.opaque_id,
+      request_method: @request_method,
+      request_url: @request_url,
+      response_status: @response_status,
+      response_body: @response_body,
+    )
+  end
+end

data/lib/webhookdb/messages/error_icalendar_fetch.rb

@@ -36,6 +36,9 @@ class Webhookdb::Messages::ErrorIcalendarFetch < Webhookdb::Message::Template
       response_status: @response_status,
       response_body: @response_body,
       external_calendar_id: @external_calendar_id,
+      webhook_endpoint: @service_integration.replicator.webhook_endpoint,
+      org_name: @service_integration.organization.name,
+      org_key: @service_integration.organization.key,
     )
   end
 end

data/lib/webhookdb/messages/specs.rb

@@ -27,6 +27,22 @@ module Webhookdb::Messages::Testers
     end
   end
 
+  class WithFields < Base
+    # noinspection RubyInstanceVariableNamingConvention
+    def initialize(a: nil, b: nil, c: nil, d: nil, e: nil)
+      @a = a
+      @b = b
+      @c = c
+      @d = d
+      @e = e
+      super()
+    end
+
+    def liquid_drops
+      return super.merge(a: @a, b: @b, c: @c, d: @d, e: @e)
+    end
+  end
+
   class Nonextant < Base
   end
 

data/lib/webhookdb/organization/alerting.rb

@@ -23,7 +23,10 @@ class Webhookdb::Organization::Alerting
 
   # Dispatch the message template to administrators of the org.
   # @param message_template [Webhookdb::Message::Template]
-  def dispatch_alert(message_template)
+  # @param separate_connection [true,false] If true, send the alert on a separate connection.
+  #   See +Webhookdb::Idempotency+. Defaults to true since this is an alert method and we
+  #   don't want it to error accidentally, if the code is called from an unexpected situation.
+  def dispatch_alert(message_template, separate_connection: true)
     unless message_template.respond_to?(:signature)
       raise Webhookdb::InvalidPrecondition,
             "message template #{message_template.template_name} must define a #signature method, " \
@@ -33,8 +36,9 @@ class Webhookdb::Organization::Alerting
     max_alerts_per_customer_per_day = Webhookdb::Organization::Alerting.max_alerts_per_customer_per_day
     yesterday = Time.now - 24.hours
     self.org.admin_customers.each do |c|
-      idemkey = "orgalert-#{signature}-#{c.id}"
-      Webhookdb::Idempotency.every(Webhookdb::Organization::Alerting.interval).under_key(idemkey) do
+      idem = Webhookdb::Idempotency.every(Webhookdb::Organization::Alerting.interval)
+      idem = idem.using_seperate_connection if separate_connection
+      idem.under_key("orgalert-#{signature}-#{c.id}") do
         sent_last_day = Webhookdb::Message::Delivery.
           where(template: message_template.full_template_name, recipient: c).
           where { created_at > yesterday }.

data/lib/webhookdb/organization/database_migration.rb

@@ -4,7 +4,7 @@ class Webhookdb::Organization::DatabaseMigration < Webhookdb::Postgres::Model(:o
   include Webhookdb::Dbutil
 
   class MigrationInProgress < Webhookdb::DatabaseLocked; end
-  class MigrationAlreadyFinished < StandardError; end
+  class MigrationAlreadyFinished < Webhookdb::WebhookdbError; end
 
   plugin :timestamps
   plugin :text_searchable, terms: [:organization, :started_by]

data/lib/webhookdb/organization/db_builder.rb

@@ -22,7 +22,7 @@ class Webhookdb::Organization::DbBuilder
   include Webhookdb::Dbutil
   extend Webhookdb::MethodUtilities
 
-  class IsolatedOperationError < StandardError; end
+  class IsolatedOperationError < Webhookdb::ProgrammingError; end
 
   DATABASE = "database"
   SCHEMA = "schema"

data/lib/webhookdb/organization.rb

@@ -7,7 +7,9 @@ require "webhookdb/stripe"
 require "webhookdb/jobs/replication_migration"
 
 class Webhookdb::Organization < Webhookdb::Postgres::Model(:organizations)
-  class SchemaMigrationError < StandardError; end
+  include Webhookdb::Admin::Linked
+
+  class SchemaMigrationError < Webhookdb::ProgrammingError; end
 
   plugin :timestamps
   plugin :soft_deletes
@@ -454,6 +456,17 @@ class Webhookdb::Organization < Webhookdb::Postgres::Model(:organizations)
     return self.add_all_membership(opts)
   end
 
+  def close(confirm:)
+    raise Webhookdb::InvalidPrecondition, "confirm must be true to close the org" unless confirm
+    unless self.service_integrations_dataset.empty?
+      msg = "Organization[#{self.key} cannot close with active service integrations"
+      raise Webhookdb::InvalidPrecondition, msg
+    end
+    memberships = self.all_memberships_dataset.all.each(&:destroy)
+    self.destroy
+    return [self, memberships]
+  end
+
   # SUBSCRIPTION PERMISSIONS
 
   def active_subscription?

data/lib/webhookdb/postgres/model.rb

@@ -7,6 +7,7 @@ require "sequel"
 require "tsort"
 
 require "webhookdb"
+require "webhookdb/admin"
 require "webhookdb/postgres"
 require "webhookdb/postgres/validations"
 require "webhookdb/postgres/model_utilities"

data/lib/webhookdb/postgres.rb

@@ -13,7 +13,7 @@ module Webhookdb::Postgres
   extend Webhookdb::MethodUtilities
   include Appydays::Loggable
 
-  class InTransaction < StandardError; end
+  class InTransaction < Webhookdb::ProgrammingError; end
 
   singleton_attr_accessor :unsafe_skip_transaction_check
   @unsafe_skip_transaction_check = false
@@ -66,6 +66,7 @@ module Webhookdb::Postgres
     "webhookdb/service_integration",
     "webhookdb/subscription",
     "webhookdb/sync_target",
+    "webhookdb/system_log_event",
     "webhookdb/webhook_subscription",
     "webhookdb/webhook_subscription/delivery",
   ].freeze

data/lib/webhookdb/replicator/base.rb

@@ -651,6 +651,9 @@ for information on how to refresh data.)
   # @param [Webhookdb::Replicator::WebhookRequest] request
   def upsert_webhook(request, **kw)
     return self._upsert_webhook(request, **kw)
+  rescue Amigo::Retry::Error
+    # Do not log this since it's expected/handled by Amigo
+    raise
   rescue StandardError => e
     self.logger.error("upsert_webhook_error", request: request.as_json, error: e)
     raise
@@ -999,42 +1002,18 @@ for information on how to refresh data.)
     job.update(started_at: Time.now)
 
     backfillers = self._backfillers(**job.criteria.symbolize_keys)
-    if self._parallel_backfill && self._parallel_backfill > 1
-      # Create a dedicated threadpool for these backfillers,
-      # with max parallelism determined by the replicator.
-      pool = Concurrent::FixedThreadPool.new(self._parallel_backfill)
-      # Record any errors that occur, since they won't raise otherwise.
-      # Initialize a sized array to avoid any potential race conditions (though GIL should make it not an issue?).
-      errors = Array.new(backfillers.size)
-      backfillers.each_with_index do |bf, idx|
-        pool.post do
-          bf.backfill(last_backfilled)
-        rescue StandardError => e
-          errors[idx] = e
-        end
-      end
-      # We've enqueued all backfillers; do not accept anymore work.
-      pool.shutdown
-      loop do
-        # We want to stop early if we find an error, so check for errors every 10 seconds.
-        completed = pool.wait_for_termination(10)
-        first_error = errors.find { |e| !e.nil? }
-        if first_error.nil?
-          # No error, and wait_for_termination returned true, so all work is done.
-          break if completed
-          # No error, but work is still going on, so loop again.
-          next
-        end
-        # We have an error; don't run any more backfillers.
-        pool.kill
-        # Wait for all ongoing backfills before raising.
-        pool.wait_for_termination
-        raise first_error
+    begin
+      if self._parallel_backfill && self._parallel_backfill > 1
+        _do_parallel_backfill(backfillers, last_backfilled)
+      else
+        _do_serial_backfill(backfillers, last_backfilled)
       end
-    else
-      backfillers.each do |backfiller|
-        backfiller.backfill(last_backfilled)
+    rescue StandardError => e
+      if self.on_backfill_error(e) == true
+        job.update(finished_at: Time.now)
+        return
       end
+      raise e
     end
 
     sint.update(last_backfilled_at: new_last_backfilled) if job.incremental?
@@ -1042,6 +1021,54 @@ for information on how to refresh data.)
     job.enqueue_children
   end
 
+  protected def _do_parallel_backfill(backfillers, last_backfilled)
+    # Create a dedicated threadpool for these backfillers,
+    # with max parallelism determined by the replicator.
+    pool = Concurrent::FixedThreadPool.new(self._parallel_backfill)
+    # Record any errors that occur, since they won't raise otherwise.
+    # Initialize a sized array to avoid any potential race conditions (though GIL should make it not an issue?).
+    errors = Array.new(backfillers.size)
+    backfillers.each_with_index do |bf, idx|
+      pool.post do
+        bf.backfill(last_backfilled)
+      rescue StandardError => e
+        errors[idx] = e
+      end
+    end
+    # We've enqueued all backfillers; do not accept anymore work.
+    pool.shutdown
+    loop do
+      # We want to stop early if we find an error, so check for errors every 10 seconds.
+      completed = pool.wait_for_termination(10)
+      first_error = errors.find { |e| !e.nil? }
+      if first_error.nil?
+        # No error, and wait_for_termination returned true, so all work is done.
+        break if completed
+        # No error, but work is still going on, so loop again.
+        next
+      end
+      # We have an error; don't run any more backfillers.
+      pool.kill
+      # Wait for all ongoing backfills before raising.
+      pool.wait_for_termination
+      raise first_error
+    end
+  end
+
+  protected def _do_serial_backfill(backfillers, last_backfilled)
+    backfillers.each do |backfiller|
+      backfiller.backfill(last_backfilled)
+    end
+  end
+
+  # Called when the #backfill method errors.
+  # This can do something like dispatch a developer alert.
+  # The handler must raise in order to stop the job from processing-
+  # if nothing is raised, the original exception will be raised instead.
+  # By default, this method noops, so the original exception is raised.
+  # @param e [Exception]
+  def on_backfill_error(e) = nil
+
   # If this replicator supports backfilling in parallel (running multiple backfillers at a time),
   # return the degree of paralellism (or nil if not running in parallel).
   # We leave parallelism up to the replicator, not CPU count, since most work
@@ -1096,15 +1123,15 @@ for information on how to refresh data.)
 
     def fetch_backfill_page(pagination_token, last_backfilled:)
       return @svc._fetch_backfill_page(pagination_token, last_backfilled:)
-    rescue ::Timeout::Error, ::SocketError
-      self.__retryordie
+    rescue ::Timeout::Error, ::SocketError => e
+      self.__retryordie(e)
     rescue Webhookdb::Http::Error => e
-      self.__retryordie if e.status >= 500
+      self.__retryordie(e) if e.status >= 500
       raise
     end
 
-    def __retryordie
-      raise Amigo::Retry::OrDie.new(self.server_error_retries, self.server_error_backoff)
+    def __retryordie(e)
+      raise Amigo::Retry::OrDie.new(self.server_error_retries, self.server_error_backoff, e)
     end
   end
 

data/lib/webhookdb/replicator/column.rb

@@ -349,6 +349,8 @@ class Webhookdb::Replicator::Column
 
   # If provided, use this expression as the UPDATE value when adding the column
   # to an existing table.
+  # To explicitly backfill using NULL, use the value +Sequel[nil]+
+  # rather than +nil+.
   # @return [String,Sequel,Sequel::SQL::Expression]
   attr_reader :backfill_expr
 

data/lib/webhookdb/replicator/fake.rb

@@ -409,6 +409,12 @@ class Webhookdb::Replicator::FakeExhaustiveConverter < Webhookdb::Replicator::Fa
         data_key: "my_id",
         backfill_expr: "hi there",
       ),
+      Webhookdb::Replicator::Column.new(
+        :using_null_backfill_expr,
+        TEXT,
+        data_key: "my_id",
+        backfill_expr: Sequel[nil],
+      ),
       Webhookdb::Replicator::Column.new(
         :using_backfill_statement,
         TEXT,

data/lib/webhookdb/replicator/front_signalwire_message_channel_app_v1.rb

@@ -239,35 +239,44 @@ All of this information can be found in the WebhookDB docs, at https://docs.webh
       @signalwire_sint = replicator.service_integration.depends_on
     end
 
-    def handle_item(item)
-      front_id = item.fetch(:front_message_id)
-      sw_id = item.fetch(:signalwire_sid)
+    def handle_item(db_row)
+      front_id = db_row.fetch(:front_message_id)
+      sw_id = db_row.fetch(:signalwire_sid)
+      # This is sort of gross- we get the db row here, and need to re-update it with certain fields
+      # as a result of the signalwire or front sync. To do that, we need to run the upsert on 'data',
+      # but what's in 'data' is incomplete. So we use the db row to form a more fully complete 'data'.
+      upserting_data = db_row.dup
+      # Remove the columns that don't belong in 'data'
+      upserting_data.delete(:pk)
+      upserting_data.delete(:row_updated_at)
+      # Splat the 'data' column into the row so it all gets put back into 'data'
+      upserting_data.merge!(**upserting_data.delete(:data))
       if (front_id && sw_id) || (!front_id && !sw_id)
-        msg = "row should have a front id OR signalwire id, should not have been inserted, or selected: #{item}"
+        msg = "row should have a front id OR signalwire id, should not have been inserted, or selected: #{db_row}"
         raise Webhookdb::InvariantViolation, msg
       end
-      sender = @replicator.format_phone(item.fetch(:sender))
-      recipient = @replicator.format_phone(item.fetch(:recipient))
-      body = item.fetch(:body)
-      idempotency_key = "fsmca-fims-#{item.fetch(:external_id)}"
+      sender = @replicator.format_phone(db_row.fetch(:sender))
+      recipient = @replicator.format_phone(db_row.fetch(:recipient))
+      body = db_row.fetch(:body)
+      idempotency_key = "fsmca-fims-#{db_row.fetch(:external_id)}"
       idempotency = Webhookdb::Idempotency.once_ever.stored.using_seperate_connection.under_key(idempotency_key)
       if front_id.nil?
-        texted_at = Time.parse(item.fetch(:data).fetch("date_created"))
+        texted_at = Time.parse(db_row.fetch(:data).fetch("date_created"))
         if texted_at < Webhookdb::Front.channel_sync_refreshness_cutoff.seconds.ago
           # Do not sync old rows, just mark them synced
-          item[:front_message_id] = "skipped_due_to_age"
+          upserting_data[:front_message_id] = "skipped_due_to_age"
         else
           # sync the message into Front
           front_response_body = idempotency.execute do
-            self._sync_front_inbound(sender:, texted_at:, item:, body:)
+            self._sync_front_inbound(sender:, texted_at:, db_row:, body:)
           end
-          item[:front_message_id] = front_response_body.fetch("message_uid")
+          upserting_data[:front_message_id] = front_response_body.fetch("message_uid")
         end
       else
-        messaged_at = Time.at(item.fetch(:data).fetch("payload").fetch("created_at"))
+        messaged_at = Time.at(db_row.fetch(:data).fetch("payload").fetch("created_at"))
         if messaged_at < Webhookdb::Front.channel_sync_refreshness_cutoff.seconds.ago
           # Do not sync old rows, just mark them synced
-          item[:signalwire_sid] = "skipped_due_to_age"
+          upserting_data[:signalwire_sid] = "skipped_due_to_age"
         else
           # send the SMS via signalwire
           signalwire_resp = _send_sms(
@@ -276,10 +285,10 @@ All of this information can be found in the WebhookDB docs, at https://docs.webh
             to: recipient,
             body:,
           )
-          item[:signalwire_sid] = signalwire_resp.fetch("sid") if signalwire_resp
+          upserting_data[:signalwire_sid] = signalwire_resp.fetch("sid") if signalwire_resp
         end
       end
-      @replicator.upsert_webhook_body(item.deep_stringify_keys)
+      @replicator.upsert_webhook_body(upserting_data.deep_stringify_keys)
     end
 
     def _send_sms(idempotency, from:, to:, body:)
@@ -321,14 +330,14 @@ All of this information can be found in the WebhookDB docs, at https://docs.webh
       return nil
     end
 
-    def _sync_front_inbound(sender:, texted_at:, item:, body:)
+    def _sync_front_inbound(sender:, texted_at:, db_row:, body:)
       body = {
         sender: {handle: sender},
         body: body || "<no body>",
         delivered_at: texted_at.to_i,
         metadata: {
-          external_id: item.fetch(:external_id),
-          external_conversation_id: item.fetch(:external_conversation_id),
+          external_id: db_row.fetch(:external_id),
+          external_conversation_id: db_row.fetch(:external_conversation_id),
         },
       }
       token = JWT.encode(

data/lib/webhookdb/replicator/icalendar_calendar_v1.rb

@@ -74,6 +74,9 @@ The secret to use for signing is:
       col.new(:row_updated_at, TIMESTAMP, index: true, optional: true, defaulter: :now),
       col.new(:last_synced_at, TIMESTAMP, index: true, optional: true),
       col.new(:ics_url, TEXT, converter: col.converter_gsub("^webcal", "https")),
+      col.new(:event_count, INTEGER, optional: true),
+      col.new(:feed_bytes, INTEGER, optional: true),
+      col.new(:last_sync_duration_ms, INTEGER, optional: true),
     ]
   end
 
@@ -166,11 +169,20 @@ The secret to use for signing is:
   def sync_row(row)
     Appydays::Loggable.with_log_tags(icalendar_url: row.fetch(:ics_url)) do
       self.with_advisory_lock(row.fetch(:pk)) do
+        start = Time.now
         now = Time.now
         if (dep = self.find_dependent("icalendar_event_v1"))
-          self._sync_row(row, dep, now:)
+          processor = self._sync_row(row, dep, now:)
+        end
+        self.admin_dataset do |ds|
+          ds.where(pk: row.fetch(:pk)).
+            update(
+              last_synced_at: now,
+              event_count: processor&.upserted_identities&.count,
+              feed_bytes: processor&.read_bytes,
+              last_sync_duration_ms: (Time.now - start).in_milliseconds,
+            )
         end
-        self.admin_dataset { |ds| ds.where(pk: row.fetch(:pk)).update(last_synced_at: now) }
       end
     end
   end
@@ -204,6 +216,7 @@ The secret to use for signing is:
         row_updated_at: now,
       )
     end
+    return processor
   end
 
   # We get all sorts of strange urls, fix up what we can.
@@ -224,7 +237,20 @@ The secret to use for signing is:
         self.logger.info("icalendar_fetch_not_modified", response_status: 304, request_url:, calendar_external_id:)
         return
       when Down::SSLError
-        self._handle_retryable_down_error!(e, request_url:, calendar_external_id:)
+        # Most SSL errors are transient and can be retried, but some are due to a long-term misconfiguration.
+        # Handle these with an alert, like if we had a 404, which indicates a longer-term issue.
+        is_fatal =
+          # There doesn't appear to be a way to allow unsafe legacy content negotiation on a per-request basis,
+          # it is compiled into OpenSSL (may be wrong about this).
+          e.to_s.include?("unsafe legacy renegotiation disabled") ||
+          # Certificate failures are not transient
+          e.to_s.include?("certificate verify failed")
+        if is_fatal
+          response_status = 0
+          response_body = e.to_s
+        else
+          self._handle_retryable_down_error!(e, request_url:, calendar_external_id:)
+        end
       when Down::TimeoutError, Down::ConnectionError, Down::InvalidUrl, URI::InvalidURIError
         response_status = 0
         response_body = e.to_s
@@ -259,8 +285,9 @@ The secret to use for signing is:
         response_status = nil
     end
     raise e if response_status.nil?
+    loggable_body = response_body && response_body[..256]
     self.logger.warn("icalendar_fetch_error",
-                     response_body:, response_status:, request_url:, calendar_external_id:,)
+                     response_body: loggable_body, response_status:, request_url:, calendar_external_id:,)
     message = Webhookdb::Messages::ErrorIcalendarFetch.new(
       self.service_integration,
       calendar_external_id,
@@ -269,7 +296,7 @@ The secret to use for signing is:
       request_url:,
       request_method: "GET",
     )
-    self.service_integration.organization.alerting.dispatch_alert(message)
+    self.service_integration.organization.alerting.dispatch_alert(message, separate_connection: false)
   end
 
   def _retryable_client_error?(e, request_url:)
@@ -299,7 +326,7 @@ The secret to use for signing is:
   end
 
   class EventProcessor
-    attr_reader :upserted_identities
+    attr_reader :upserted_identities, :read_bytes
 
     def initialize(io, upserter)
       @io = io
@@ -316,6 +343,9 @@ The secret to use for signing is:
       # We need to keep track of how many events each UID spawns,
       # so we can delete any with a higher count.
       @max_sequence_num_by_uid = {}
+      # Keep track of the bytes we've read from the file.
+      # Never trust Content-Length headers for ical feeds.
+      @read_bytes = 0
     end
 
     def delete_condition
@@ -474,7 +504,11 @@ The secret to use for signing is:
     def _ical_entry_from_ruby(r, entry, is_date)
       return {"v" => r.strftime("%Y%m%d")} if is_date
       return {"v" => r.strftime("%Y%m%dT%H%M%SZ")} if r.zone == "UTC"
-      return {"v" => r.strftime("%Y%m%dT%H%M%S"), "TZID" => entry.fetch("TZID")}
+      tzid = entry["TZID"]
+      return {"v" => r.strftime("%Y%m%dT%H%M%S"), "TZID" => tzid} if tzid
+      value = entry.fetch("v")
+      return {"v" => value} if value.end_with?("Z")
+      raise "Cannot create ical entry from: #{r}, #{entry}, is_date: #{is_date}"
     end
 
     def _icecube_rule_from_ical(ical)
@@ -483,11 +517,20 @@ The secret to use for signing is:
       # IceCube errors, because `day_of_month` isn't valid on a WeeklyRule.
       # In this case, we need to sanitize the string to remove the offending rule piece.
       # There are probably many other offending formats, but we'll add them here as needed.
+      unambiguous_ical = nil
       if ical.include?("FREQ=WEEKLY") && ical.include?("BYMONTHDAY=")
-        ical = ical.gsub(/BYMONTHDAY=[\d,]+/, "")
-        ical.delete_prefix! ";"
-        ical.delete_suffix! ";"
-        ical.squeeze!(";")
+        unambiguous_ical = ical.gsub(/BYMONTHDAY=[\d,]+/, "")
+      elsif ical.include?("FREQ=MONTHLY") && ical.include?("BYYEARDAY=") && ical.include?("BYMONTHDAY=")
+        # Another rule: FREQ=MONTHLY;INTERVAL=3;BYYEARDAY=14;BYMONTHDAY=14
+        # Apple interprets this as monthly on the 14th; rrule.js interprets this as never happening.
+        # 'day_of_year' isn't valid on a MonthlyRule, so delete the BYYEARDAY component.
+        unambiguous_ical = ical.gsub(/BYYEARDAY=[\d,]+/, "")
+      end
+      if unambiguous_ical
+        unambiguous_ical.delete_prefix! ";"
+        unambiguous_ical.delete_suffix! ";"
+        unambiguous_ical.squeeze!(";")
+        ical = unambiguous_ical
       end
       return IceCube::IcalParser.rule_from_ical(ical)
     end
@@ -507,6 +550,7 @@ The secret to use for signing is:
       vevent_lines = []
       in_vevent = false
       while (line = @io.gets)
+        @read_bytes += line.size
         begin
           line.rstrip!
         rescue Encoding::CompatibilityError

data/lib/webhookdb/replicator/intercom_v1_mixin.rb

@@ -22,10 +22,9 @@ module Webhookdb::Replicator::IntercomV1Mixin
   # webhook verification, which means that webhooks actually don't require any setup on the integration level. Thus,
   # `supports_webhooks` is false.
   def find_auth_integration
-    # rubocop:disable Naming/MemoizedInstanceVariableName
-    return @auth ||= Webhookdb::Replicator.find_at_root!(self.service_integration,
-                                                         service_name: "intercom_marketplace_root_v1",)
-    # rubocop:enable Naming/MemoizedInstanceVariableName
+    return @find_auth_integration ||= Webhookdb::Replicator.find_at_root!(
+      self.service_integration, service_name: "intercom_marketplace_root_v1",
+    )
   end
 
   def intercom_auth_headers
@@ -93,6 +92,28 @@ module Webhookdb::Replicator::IntercomV1Mixin
         timeout: Webhookdb::Intercom.http_timeout,
       )
     rescue Webhookdb::Http::Error => e
+      is_token_suspended = e.status == 401 &&
+        e.response["errors"].present? &&
+        e.response["errors"].any? { |er| er["code"] == "token_suspended" }
+      if is_token_suspended
+        root_sint = self.find_auth_integration
+        message = "Organization has closed their Intercom workspace and this integration should be deleted. " \
+                  "From a console, run: " \
+                  "Webhookdb::ServiceIntegration[#{root_sint.id}].destroy_self_and_all_dependents"
+        Webhookdb::DeveloperAlert.new(
+          subsystem: "Intercom Workspace Closed Error",
+          emoji: ":hook:",
+          fallback: message,
+          fields: [
+            {title: "Organization", value: root_sint.organization.name, short: true},
+            {title: "Integration ID", value: root_sint.id.to_s, short: true},
+            {title: "Instructions", value: message},
+          ],
+        ).emit
+        # Noop here since there's nothing to do, the developer alert takes care of notifying
+        # so no need to error or log.
+        return [], nil
+      end
       #  We are looking to catch the "api plan restricted" error. This is always a 403 and every
       # 403 will be an "api plan restricted" error according to the API documentation. Because we
       # specify the API version in our headers we can expect that this won't change.

data/lib/webhookdb/replicator/signalwire_message_v1.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
+require "webhookdb/errors"
 require "webhookdb/signalwire"
+require "webhookdb/messages/error_generic_backfill"
 
 class Webhookdb::Replicator::SignalwireMessageV1 < Webhookdb::Replicator::Base
   include Appydays::Loggable
@@ -180,4 +182,33 @@ Press 'Show' next to the newly-created API token, and copy it.)
 
     return messages, data["next_page_uri"]
   end
+
+  def on_backfill_error(be)
+    e = Webhookdb::Errors.find_cause(be) do |ex|
+      next true if ex.is_a?(Webhookdb::Http::Error) && ex.status == 401
+      next true if ex.is_a?(::SocketError)
+    end
+    return unless e
+    if e.is_a?(::SocketError)
+      response_status = 0
+      response_body = e.message
+      request_url = "<unknown>"
+      request_method = "<unknown>"
+    else
+      response_status = e.status
+      response_body = e.body
+      request_url = e.uri.to_s
+      request_method = e.http_method
+    end
+    self.logger.warn("signalwire_backfill_error", response_body:, response_status:, request_url:)
+    message = Webhookdb::Messages::ErrorGenericBackfill.new(
+      self.service_integration,
+      response_status:,
+      response_body:,
+      request_url:,
+      request_method:,
+    )
+    self.service_integration.organization.alerting.dispatch_alert(message)
+    return true
+  end
 end