webhookdb 1.3.1 → 1.5.0

data/lib/webhookdb/replicator/icalendar_calendar_v1.rb

@@ -74,6 +74,10 @@ The secret to use for signing is:
       col.new(:row_updated_at, TIMESTAMP, index: true, optional: true, defaulter: :now),
       col.new(:last_synced_at, TIMESTAMP, index: true, optional: true),
       col.new(:ics_url, TEXT, converter: col.converter_gsub("^webcal", "https")),
+      col.new(:event_count, INTEGER, optional: true),
+      col.new(:feed_bytes, INTEGER, optional: true),
+      col.new(:last_sync_duration_ms, INTEGER, optional: true),
+      col.new(:last_fetch_context, OBJECT, optional: true),
     ]
   end
 
@@ -108,7 +112,7 @@ The secret to use for signing is:
     external_id = request.body.fetch("external_id")
     case request_type
       when "SYNC"
-        super(request)
+        super
         Webhookdb::Jobs::IcalendarSync.perform_async(self.service_integration.id, external_id)
         return
       when "DELETE"
@@ -118,14 +122,12 @@ The secret to use for signing is:
         unless Webhookdb::RACK_ENV == "test"
           raise "someone tried to use the special unit test google event type outside of unit tests"
         end
-        return super(request)
+        return super
       else
         raise ArgumentError, "Unknown request type: #{request_type}"
     end
   end
 
-  CLEANUP_SERVICE_NAMES = ["icalendar_event_v1"].freeze
-
   def rows_needing_sync(dataset, now: Time.now)
     cutoff = now - Webhookdb::Icalendar.sync_period_hours.hours
     return dataset.where(Sequel[last_synced_at: nil] | Sequel.expr { last_synced_at < cutoff })
@@ -133,7 +135,7 @@ The secret to use for signing is:
 
   def delete_data_for_external_id(external_id)
     relevant_integrations = self.service_integration.recursive_dependents.
-      filter { |d| CLEANUP_SERVICE_NAMES.include?(d.service_name) }
+      filter { |d| Webhookdb::Icalendar::EVENT_REPLICATORS.include?(d.service_name) }
     self.admin_dataset do |ds|
       ds.db.transaction do
         ds.where(external_id:).delete
@@ -154,7 +156,7 @@ The secret to use for signing is:
       @now = now
     end
 
-    def upsert_page_size = 500
+    def upsert_page_size = 2000
     def conditional_upsert? = true
 
     def prepare_body(body)
@@ -163,14 +165,50 @@ The secret to use for signing is:
     end
   end
 
-  def sync_row(row)
+  def sync_row(row, force: false, now: Time.now)
     Appydays::Loggable.with_log_tags(icalendar_url: row.fetch(:ics_url)) do
+      last_synced_at = row.fetch(:last_synced_at)
+      should_sync = force ||
+        last_synced_at.nil? ||
+        # If a proxy is configured, we always want to try to sync,
+        # since this could have come from a webhook, but also the proxy feed refresh TTL
+        # is likely much lower than ICALENDAR_SYNC_PERIOD_HOURS so it's good to check on it.
+        # The check is very fast (should 304) so is safe to do relatively often.
+        Webhookdb::Icalendar.proxy_url.present? ||
+        last_synced_at < (now - Webhookdb::Icalendar.sync_period_hours.hours)
+      unless should_sync
+        self.logger.info("skip_sync_recently_synced", last_synced_at:)
+        return
+      end
       self.with_advisory_lock(row.fetch(:pk)) do
-        now = Time.now
-        if (dep = self.find_dependent("icalendar_event_v1"))
-          self._sync_row(row, dep, now:)
+        start = Time.now
+        if (dep = self.find_dependent(Webhookdb::Icalendar::EVENT_REPLICATORS))
+          if dep.replicator.avoid_writes?
+            # Check if this table is being vacuumed/etc. We use this instead of a semaphore job,
+            # since it's a better fit for icalendar, which is pre-scheduled, rather than reactive.
+            # That is, when we receive webhooks, a semaphore job gives us a more predictable rate;
+            # but icalendar rate is negotiated in advance (when enqueing jobs),
+            # and we can be more 'helpful' to something like a vacuum by not running any jobs at all.
+            self.logger.info("skip_sync_table_locked")
+            raise Amigo::Retry::Retry, 60.seconds + (rand * 10.seconds)
+          end
+          processor = self._sync_row(row, dep, now:)
+        end
+        self.admin_dataset do |ds|
+          ds.where(pk: row.fetch(:pk)).
+            update(
+              last_synced_at: now,
+              event_count: processor&.upserted_identities&.count,
+              feed_bytes: processor&.read_bytes,
+              last_sync_duration_ms: (Time.now - start).in_milliseconds,
+              last_fetch_context: {
+                "hash" => processor&.feed_hash,
+                "content_type" => processor&.headers&.fetch("Content-Type", nil),
+                "content_length" => processor&.headers&.fetch("Content-Length", nil),
+                "etag" => processor&.headers&.fetch("Etag", nil),
+              }.to_json,
+            )
         end
-        self.admin_dataset { |ds| ds.where(pk: row.fetch(:pk)).update(last_synced_at: now) }
       end
     end
   end
@@ -179,14 +217,19 @@ The secret to use for signing is:
     calendar_external_id = row.fetch(:external_id)
     begin
       request_url = self._clean_ics_url(row.fetch(:ics_url))
-      io = Webhookdb::Http.chunked_download(request_url, rewindable: false)
-    rescue Down::Error, URI::InvalidURIError => e
+      io = self._make_ics_request(request_url, row.fetch(:last_fetch_context))
+    rescue Down::Error,
+           URI::InvalidURIError,
+           HTTPX::NativeResolveError,
+           HTTPX::InsecureRedirectError,
+           HTTPX::Connection::HTTP2::Error,
+           EOFError => e
       self._handle_down_error(e, request_url:, calendar_external_id:)
       return
     end
 
     upserter = Upserter.new(dep.replicator, calendar_external_id, now:)
-    processor = EventProcessor.new(io, upserter)
+    processor = EventProcessor.new(io:, upserter:, headers: io.data[:headers])
     processor.process
     # Delete all the extra replicator rows, and cancel all the rows that weren't upserted.
     dep.replicator.admin_dataset do |ds|
@@ -204,6 +247,25 @@ The secret to use for signing is:
         row_updated_at: now,
       )
     end
+    return processor
+  end
+
+  def _make_ics_request(request_url, last_fetch_context)
+    # Some servers require a VERY explicit accept header,
+    # so tell them we prefer icalendar here.
+    # Using Httpx, Accept-Encoding is gzip,deflate
+    # which seems fine (server should use identity as worst case).
+    headers = {
+      "Accept" => "text/calendar,*/*",
+    }
+    headers["If-None-Match"] = last_fetch_context["etag"] if last_fetch_context & ["etag"]
+    if (proxy_url = Webhookdb::Icalendar.proxy_url).present?
+      request_url = "#{proxy_url.delete_suffix('/')}/?url=#{URI.encode_www_form_component(request_url)}"
+      headers["Authorization"] = "Apikey #{Webhookdb::Icalendar.proxy_api_key}" if
+        Webhookdb::Icalendar.proxy_api_key.present?
+    end
+    resp = Webhookdb::Http.chunked_download(request_url, rewindable: false, headers:)
+    return resp
   end
 
   # We get all sorts of strange urls, fix up what we can.
@@ -224,13 +286,31 @@ The secret to use for signing is:
         self.logger.info("icalendar_fetch_not_modified", response_status: 304, request_url:, calendar_external_id:)
         return
       when Down::SSLError
-        self._handle_retryable_down_error!(e, request_url:, calendar_external_id:)
-      when Down::TimeoutError, Down::ConnectionError, Down::InvalidUrl, URI::InvalidURIError
+        # Most SSL errors are transient and can be retried, but some are due to a long-term misconfiguration.
+        # Handle these with an alert, like if we had a 404, which indicates a longer-term issue.
+        is_fatal =
+          # There doesn't appear to be a way to allow unsafe legacy content negotiation on a per-request basis,
+          # it is compiled into OpenSSL (may be wrong about this).
+          e.to_s.include?("unsafe legacy renegotiation disabled") ||
+          # Certificate failures are not transient
+          e.to_s.include?("certificate verify failed")
+        if is_fatal
+          response_status = 0
+          response_body = e.to_s
+        else
+          self._handle_retryable_down_error!(e, request_url:, calendar_external_id:)
+        end
+      when Down::TimeoutError, Down::ConnectionError, Down::InvalidUrl,
+        Errno::ECONNRESET,
+        URI::InvalidURIError,
+        HTTPX::NativeResolveError, HTTPX::InsecureRedirectError,
+        HTTPX::Connection::HTTP2::Error,
+        EOFError
         response_status = 0
         response_body = e.to_s
       when Down::ClientError
         raise e if e.response.nil?
-        response_status = e.response.code.to_i
+        response_status = e.response.status.to_i
         self._handle_retryable_down_error!(e, request_url:, calendar_external_id:) if
           self._retryable_client_error?(e, request_url:)
         # These are all the errors we've seen, we can't do anything about.
@@ -242,25 +322,32 @@ The secret to use for signing is:
           404, 405, # Fundamental issues with the URL given
           409, 410, # More access problems
           417, # If someone uses an Outlook HTML calendar, fetch gives us a 417
+          422, # Sometimes used instead of 404
           429, # Usually 429s are retried (as above), but in some cases they're not.
+          500, 503, 504, # Intermittent server issues, usually
+          599, # Represents a timeout in icalproxy
         ]
         # For most client errors, we can't do anything about it. For example,
         # and 'unshared' URL could result in a 401, 403, 404, or even a 405.
         # For now, other client errors, we can raise on,
         # in case it's something we can fix/work around.
         # For example, it's possible something like a 415 is a WebhookDB issue.
+        if response_status == 421 && (origin_err = e.response.headers["Ical-Proxy-Origin-Error"])
+          response_status = origin_err.to_i
+        end
         raise e unless expected_errors.include?(response_status)
-        response_body = e.response.body.to_s
+        response_body = self._safe_read_body(e)
       when Down::ServerError
-        response_status = e.response.code.to_i
-        response_body = e.response.body.to_s
+        response_status = e.response.status.to_i
+        response_body = self._safe_read_body(e)
       else
         response_body = nil
         response_status = nil
     end
     raise e if response_status.nil?
+    loggable_body = response_body && response_body[..256]
     self.logger.warn("icalendar_fetch_error",
-                     response_body:, response_status:, request_url:, calendar_external_id:,)
+                     response_body: loggable_body, response_status:, request_url:, calendar_external_id:,)
     message = Webhookdb::Messages::ErrorIcalendarFetch.new(
       self.service_integration,
       calendar_external_id,
@@ -269,11 +356,19 @@ The secret to use for signing is:
       request_url:,
       request_method: "GET",
     )
-    self.service_integration.organization.alerting.dispatch_alert(message)
+    self.service_integration.organization.alerting.dispatch_alert(message, separate_connection: false)
+  end
+
+  # We can hit an error while reading the error body, since it was opened as a stream.
+  # Ignore those errors.
+  def _safe_read_body(e)
+    return e.response.body.to_s
+  rescue OpenSSL::SSL::SSLError, HTTPX::Error
+    return "<error reading body>"
   end
 
   def _retryable_client_error?(e, request_url:)
-    code = e.response.code.to_i
+    code = e.response.status.to_i
     # This is a bad domain that returns 429 for most requests.
     # Tell the org admins it won't sync.
     return false if code == 429 && request_url.start_with?("https://ical.schedulestar.com")
@@ -290,7 +385,7 @@ The secret to use for signing is:
     retry_in = rand(4..60).minutes
     self.logger.debug(
       "icalendar_fetch_error_retry",
-      response_status: e.respond_to?(:response) ? e.response&.code : 0,
+      response_status: e.respond_to?(:response) ? e.response&.status : 0,
       request_url:,
       calendar_external_id:,
       retry_at: Time.now + retry_in,
@@ -299,11 +394,12 @@ The secret to use for signing is:
   end
 
   class EventProcessor
-    attr_reader :upserted_identities
+    attr_reader :upserted_identities, :read_bytes, :headers
 
-    def initialize(io, upserter)
+    def initialize(io:, upserter:, headers:)
       @io = io
       @upserter = upserter
+      @headers = headers
       # Keep track of everything we upsert. For any rows we aren't upserting,
       # delete them if they're recurring, or cancel them if they're not recurring.
       # If doing it this way is slow, we could invert this (pull down all IDs and pop from the set).
@@ -316,8 +412,14 @@ The secret to use for signing is:
       # We need to keep track of how many events each UID spawns,
       # so we can delete any with a higher count.
       @max_sequence_num_by_uid = {}
+      # Keep track of the bytes we've read from the file.
+      # Never trust Content-Length headers for ical feeds.
+      @read_bytes = 0
+      @feed_md5 = Digest::MD5.new
     end
 
+    def feed_hash = @feed_md5.hexdigest
+
     def delete_condition
       return nil if @max_sequence_num_by_uid.empty?
       return @max_sequence_num_by_uid.map do |uid, n|
@@ -456,7 +558,14 @@ The secret to use for signing is:
           # The new UID has the sequence number.
           e["UID"] = {"v" => "#{uid}-#{idx}"}
           e["DTSTART"] = self._ical_entry_from_ruby(occ.start_time, start_entry, is_date)
-          e["DTEND"] = self._ical_entry_from_ruby(occ.end_time, end_entry, is_date) if has_end_time
+          if has_end_time
+            if !is_date && end_entry["VALUE"] == "DATE"
+              # It's possible that DTSTART is a time, but DTEND is a date. This makes no sense,
+              # so skip setting an end date. It will be in the :data column at least.
+            else
+              e["DTEND"] = self._ical_entry_from_ruby(occ.end_time, end_entry, is_date)
+            end
+          end
           yield e
           final_sequence = idx
           break if occ.start_time > dont_project_after
@@ -474,7 +583,15 @@ The secret to use for signing is:
     def _ical_entry_from_ruby(r, entry, is_date)
       return {"v" => r.strftime("%Y%m%d")} if is_date
       return {"v" => r.strftime("%Y%m%dT%H%M%SZ")} if r.zone == "UTC"
-      return {"v" => r.strftime("%Y%m%dT%H%M%S"), "TZID" => entry.fetch("TZID")}
+      tzid = entry["TZID"]
+      return {"v" => r.strftime("%Y%m%dT%H%M%S"), "TZID" => tzid} if tzid
+      value = entry.fetch("v")
+      return {"v" => value} if value.end_with?("Z")
+      if /^\d{8}T\d{6}$/.match?(value)
+        @upserter.upserting_replicator.logger.warn "ical_assuming_utc_time", ical_entry: entry, ruby_time: r
+        return {"v" => "#{value}Z"}
+      end
+      raise "Cannot create ical entry from: '#{r}', #{entry}"
     end
 
     def _icecube_rule_from_ical(ical)
@@ -483,11 +600,20 @@ The secret to use for signing is:
       # IceCube errors, because `day_of_month` isn't valid on a WeeklyRule.
       # In this case, we need to sanitize the string to remove the offending rule piece.
       # There are probably many other offending formats, but we'll add them here as needed.
+      unambiguous_ical = nil
       if ical.include?("FREQ=WEEKLY") && ical.include?("BYMONTHDAY=")
-        ical = ical.gsub(/BYMONTHDAY=[\d,]+/, "")
-        ical.delete_prefix! ";"
-        ical.delete_suffix! ";"
-        ical.squeeze!(";")
+        unambiguous_ical = ical.gsub(/BYMONTHDAY=[\d,]+/, "")
+      elsif ical.include?("FREQ=MONTHLY") && ical.include?("BYYEARDAY=") && ical.include?("BYMONTHDAY=")
+        # Another rule: FREQ=MONTHLY;INTERVAL=3;BYYEARDAY=14;BYMONTHDAY=14
+        # Apple interprets this as monthly on the 14th; rrule.js interprets this as never happening.
+        # 'day_of_year' isn't valid on a MonthlyRule, so delete the BYYEARDAY component.
+        unambiguous_ical = ical.gsub(/BYYEARDAY=[\d,]+/, "")
+      end
+      if unambiguous_ical
+        unambiguous_ical.delete_prefix! ";"
+        unambiguous_ical.delete_suffix! ";"
+        unambiguous_ical.squeeze!(";")
+        ical = unambiguous_ical
       end
       return IceCube::IcalParser.rule_from_ical(ical)
     end
@@ -507,6 +633,8 @@ The secret to use for signing is:
       vevent_lines = []
       in_vevent = false
       while (line = @io.gets)
+        @read_bytes += line.size
+        @feed_md5.update(line)
         begin
           line.rstrip!
         rescue Encoding::CompatibilityError
@@ -545,4 +673,41 @@ The secret to use for signing is:
       @upserter.upserting_replicator.logger.warn("invalid_vevent_hash", vevent_uids: bad_event_uids.sort)
     end
   end
+
+  # Return true if the data in the feed has changed from what was last synced,
+  # or false if it has not so the sync can be skipped.
+  # This operation is meant to be resource-light (most of the work is the HTTP request),
+  # so should be done in a threadpool.
+  #
+  # - If we have no previous fetch context, we sync.
+  # - If the fetch errors, sync, because we want the normal error handler to figure it out
+  #   (alert admins, etc).
+  # - If the last fetch's content type and length is different from the current, we sync.
+  # - Download the bytes. If the hash of the bytes is different from what was last processed,
+  #   sync. Since this involves reading the streaming body, we must return a copy of the body (a StringIO).
+  def feed_changed?(row)
+    last_fetch = row.fetch(:last_fetch_context)
+    return true if last_fetch.nil? || last_fetch.empty?
+
+    begin
+      url = self._clean_ics_url(row.fetch(:ics_url))
+      resp = self._make_ics_request(url, last_fetch)
+    rescue Down::NotModified
+      return false
+    rescue StandardError
+      return true
+    end
+    headers = resp.data[:headers] || {}
+    content_type_match = headers["Content-Type"] == last_fetch["content_type"] &&
+      headers["Content-Length"] == last_fetch["content_length"]
+    return true unless content_type_match
+    last_hash = last_fetch["hash"]
+    return true if last_hash.nil?
+
+    hash = Digest::MD5.new
+    while (line = resp.gets)
+      hash.update(line)
+    end
+    return hash.hexdigest != last_hash
+  end
 end

data/lib/webhookdb/replicator/icalendar_event_v1.rb

@@ -2,6 +2,7 @@
 
 require "webhookdb/icalendar"
 require "webhookdb/windows_tz"
+require "webhookdb/replicator/base_stale_row_deleter"
 
 class Webhookdb::Replicator::IcalendarEventV1 < Webhookdb::Replicator::Base
   include Appydays::Loggable
@@ -115,7 +116,6 @@ class Webhookdb::Replicator::IcalendarEventV1 < Webhookdb::Replicator::Base
       :compound_identity,
       TEXT,
       data_key: "<compound key, see converter>",
-      index: true,
       converter: CONV_REMOTE_KEY,
       optional: true, # This is done via the converter, data_key never exists
     )
@@ -166,6 +166,7 @@ class Webhookdb::Replicator::IcalendarEventV1 < Webhookdb::Replicator::Base
     data.delete("calendar_external_id")
     data.delete("recurring_event_id")
     data.delete("recurring_event_sequence")
+    data.delete("row_updated_at")
     return data
   end
 
@@ -215,11 +216,21 @@ class Webhookdb::Replicator::IcalendarEventV1 < Webhookdb::Replicator::Base
         columns: [:calendar_external_id, :start_date, :end_date],
         where: Sequel[:status].is_distinct_from("CANCELLED") & (Sequel[:start_date] !~ nil),
       ),
+      Webhookdb::Replicator::IndexSpec.new(
+        columns: [:row_updated_at],
+        where: Sequel[status: "CANCELLED"],
+        identifier: "cancelled_row_updated_at",
+      ),
     ]
   end
 
   def _update_where_expr
-    return self.qualified_table_sequel_identifier[:last_modified_at] < Sequel[:excluded][:last_modified_at]
+    # Compare against data to avoid the constant writes. JSONB != operations are very fast,
+    # so this should not be any real performance issue.
+    # last_modified_at is unreliable because LAST-MODIFIED is unreliable,
+    # even in feeds it is set. There are cases, such as adding an EXDATE to an RRULE,
+    # that do not trigger LAST-MODIFIED changes.
+    return self.qualified_table_sequel_identifier[:data] !~ Sequel[:excluded][:data]
   end
 
   # @param [Array<String>] lines
@@ -369,50 +380,15 @@ class Webhookdb::Replicator::IcalendarEventV1 < Webhookdb::Replicator::Base
   # +stale_at+ to +age_cutoff+. This avoids endlessly adding to the icalendar events table
   # due to feeds that change UIDs each fetch- events with changed UIDs will become CANCELLED,
   # and then deleted over time.
-  # @param stale_at [Time] When an event is considered 'stale'.
-  #   If stale events are a big problem, this can be shortened to just a few days.
-  # @param age_cutoff [Time] Where to stop searching for old events.
-  #   This is important to avoid a full table scale when deleting events,
-  #   since otherwise it is like 'row_updated_at < 35.days.ago'.
-  #   Since this routine should run regularly, we should rarely have events more than 35 or 36 days old,
-  #   for example.
-  #   Use +nil+ to use no limit (a full table scan) which may be necessary when running this feature
-  #   for the first time.
-  # @param chunk_size [Integer] The row delete is done in chunks to avoid long locks.
-  #   The default seems safe, but it's exposed as a parameter if you need to play around with it,
-  #   and can be done via configuration if needed at some point.
-  def delete_stale_cancelled_events(
-    stale_at: Webhookdb::Icalendar.stale_cancelled_event_threshold_days.days.ago,
-    age_cutoff: (Webhookdb::Icalendar.stale_cancelled_event_threshold_days + 10).days.ago,
-    chunk_size: 10_000
-  )
-    # Delete in chunks, like:
-    #   DELETE from "public"."icalendar_event_v1_aaaa"
-    #   WHERE pk IN (
-    #     SELECT pk FROM "public"."icalendar_event_v1_aaaa"
-    #     WHERE row_updated_at < (now() - '35 days'::interval)
-    #     LIMIT 10000
-    #   )
-    age = age_cutoff..stale_at
-    self.admin_dataset do |ds|
-      chunk_ds = ds.where(row_updated_at: age, status: "CANCELLED").select(:pk).limit(chunk_size)
-      loop do
-        # Due to conflicts where a feed is being inserted while the delete is happening,
-        # this may raise an error like:
-        #   deadlock detected
-        #   DETAIL:  Process 18352 waits for ShareLock on transaction 435085606; blocked by process 24191.
-        #   Process 24191 waits for ShareLock on transaction 435085589; blocked by process 18352.
-        #   HINT:  See server log for query details.
-        #   CONTEXT:  while deleting tuple (2119119,3) in relation "icalendar_event_v1_aaaa"
-        # Unit testing this is very difficult though, and in practice it is rare,
-        # and normal Sidekiq job retries should be sufficient to handle this.
-        # So we don't explicitly handle deadlocks, but could if it becomes an issue.
-        deleted = ds.where(pk: chunk_ds).delete
-        break if deleted != chunk_size
-      end
-    end
+  class StaleRowDeleter < Webhookdb::Replicator::BaseStaleRowDeleter
+    def stale_at = Webhookdb::Icalendar.stale_cancelled_event_threshold_days.days
+    def lookback_window = Webhookdb::Icalendar.stale_cancelled_event_lookback_days.days
+    def updated_at_column = :row_updated_at
+    def stale_condition = {status: "CANCELLED"}
   end
 
+  def stale_row_deleter = StaleRowDeleter.new(self)
+
   def calculate_webhook_state_machine
     if (step = self.calculate_dependency_state_machine_step(dependency_help: ""))
       return step

data/lib/webhookdb/replicator/icalendar_event_v1_partitioned.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "webhookdb/replicator/icalendar_event_v1"
+require "webhookdb/replicator/partitionable_mixin"
+
+class Webhookdb::Replicator::IcalendarEventV1Partitioned < Webhookdb::Replicator::IcalendarEventV1
+  include Webhookdb::Replicator::PartitionableMixin
+
+  # @return [Webhookdb::Replicator::Descriptor]
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "icalendar_event_v1_partitioned",
+      ctor: ->(sint) { self.new(sint) },
+      dependency_descriptor: Webhookdb::Replicator::IcalendarCalendarV1.descriptor,
+      feature_roles: ["partitioning_beta"],
+      resource_name_singular: "iCalendar Event",
+      supports_webhooks: true,
+      description: "Individual events in an icalendar, using partitioned tables rather than one big table. " \
+                   "See icalendar_calendar_v1.",
+      api_docs_url: "https://icalendar.org/",
+    )
+  end
+
+  def _denormalized_columns
+    d = super
+    d << Webhookdb::Replicator::Column.new(:calendar_external_hash, INTEGER, optional: true)
+    return d
+  end
+
+  def partition_method = Webhookdb::DBAdapter::Partitioning::HASH
+  def partition_column_name = :calendar_external_hash
+  def partition_value(resource) = self._str2inthash(resource.fetch("calendar_external_id"))
+end

data/lib/webhookdb/replicator/intercom_contact_v1.rb

@@ -43,6 +43,7 @@ class Webhookdb::Replicator::IntercomContactV1 < Webhookdb::Replicator::Base
   end
 
   def _mixin_backfill_url = "https://api.intercom.io/contacts"
+  def _mixin_backfill_hashkey = "data"
 
   def _resource_and_event(request)
     resource, event = super

data/lib/webhookdb/replicator/intercom_conversation_v1.rb

@@ -40,6 +40,7 @@ class Webhookdb::Replicator::IntercomConversationV1 < Webhookdb::Replicator::Bas
   end
 
   def _mixin_backfill_url = "https://api.intercom.io/conversations"
+  def _mixin_backfill_hashkey = "conversations"
 
   def _resource_and_event(request)
     resource, event = super

data/lib/webhookdb/replicator/intercom_v1_mixin.rb

@@ -22,10 +22,9 @@ module Webhookdb::Replicator::IntercomV1Mixin
   # webhook verification, which means that webhooks actually don't require any setup on the integration level. Thus,
   # `supports_webhooks` is false.
   def find_auth_integration
-    # rubocop:disable Naming/MemoizedInstanceVariableName
-    return @auth ||= Webhookdb::Replicator.find_at_root!(self.service_integration,
-                                                         service_name: "intercom_marketplace_root_v1",)
-    # rubocop:enable Naming/MemoizedInstanceVariableName
+    return @find_auth_integration ||= Webhookdb::Replicator.find_at_root!(
+      self.service_integration, service_name: "intercom_marketplace_root_v1",
+    )
   end
 
   def intercom_auth_headers
@@ -74,8 +73,9 @@ module Webhookdb::Replicator::IntercomV1Mixin
   end
 
   def _mixin_backfill_url = raise NotImplementedError
+  def _mixin_backfill_hashkey = raise NotImplementedError
 
-  def _fetch_backfill_page(pagination_token, **_kwargs)
+  def _fetch_backfill_page(pagination_token, last_backfilled:)
     unless self.auth_credentials?
       raise Webhookdb::Replicator::CredentialsMissing,
             "This integration requires that the Intercom Auth integration has a valid Auth Token"
@@ -93,6 +93,28 @@ module Webhookdb::Replicator::IntercomV1Mixin
         timeout: Webhookdb::Intercom.http_timeout,
       )
     rescue Webhookdb::Http::Error => e
+      is_token_suspended = e.status == 401 &&
+        e.response["errors"].present? &&
+        e.response["errors"].any? { |er| er["code"] == "token_suspended" }
+      if is_token_suspended
+        root_sint = self.find_auth_integration
+        message = "Organization has closed their Intercom workspace and this integration should be deleted. " \
+                  "From a console, run: " \
+                  "Webhookdb::ServiceIntegration[#{root_sint.id}].destroy_self_and_all_dependents"
+        Webhookdb::DeveloperAlert.new(
+          subsystem: "Intercom Workspace Closed Error",
+          emoji: ":hook:",
+          fallback: message,
+          fields: [
+            {title: "Organization", value: root_sint.organization.name, short: true},
+            {title: "Integration ID", value: root_sint.id.to_s, short: true},
+            {title: "Instructions", value: message},
+          ],
+        ).emit
+        # Noop here since there's nothing to do, the developer alert takes care of notifying
+        # so no need to error or log.
+        return [], nil
+      end
       #  We are looking to catch the "api plan restricted" error. This is always a 403 and every
       # 403 will be an "api plan restricted" error according to the API documentation. Because we
       # specify the API version in our headers we can expect that this won't change.
@@ -102,8 +124,29 @@ module Webhookdb::Replicator::IntercomV1Mixin
       # a TypeError in the backfiller.
       return [], nil
     end
-    data = response.parsed_response.fetch("data", [])
+    data = response.parsed_response.fetch(self._mixin_backfill_hashkey)
     starting_after = response.parsed_response.dig("pages", "next", "starting_after")
+    # Intercom pagination sorts by updated_at newest. So if we are doing an incremental sync (last_backfilled set),
+    # and we last backfilled after the latest updated_at, we can stop paginating.
+    if last_backfilled && data.last && data.last["updated_at"]
+      oldest_update = Time.at(data.last["updated_at"])
+      starting_after = nil if oldest_update < last_backfilled
+    end
     return data, starting_after
   end
+
+  def _backfillers
+    return [Backfiller.new(self)]
+  end
+
+  class Backfiller < Webhookdb::Replicator::Base::ServiceBackfiller
+    include Webhookdb::Backfiller::Bulk
+
+    # Upsert for each API call
+    def upsert_page_size = Webhookdb::Intercom.page_size
+    def prepare_body(_body) = nil
+    def upserting_replicator = self.svc
+    # We don't want to override newer items from webhooks, so use conditional upsert.
+    def conditional_upsert? = true
+  end
 end