webhookdb 1.4.0 → 1.5.0

data/lib/webhookdb/replicator/icalendar_calendar_v1.rb

@@ -77,6 +77,7 @@ The secret to use for signing is:
       col.new(:event_count, INTEGER, optional: true),
       col.new(:feed_bytes, INTEGER, optional: true),
       col.new(:last_sync_duration_ms, INTEGER, optional: true),
+      col.new(:last_fetch_context, OBJECT, optional: true),
     ]
   end
 
@@ -111,7 +112,7 @@ The secret to use for signing is:
     external_id = request.body.fetch("external_id")
     case request_type
       when "SYNC"
-        super(request)
+        super
         Webhookdb::Jobs::IcalendarSync.perform_async(self.service_integration.id, external_id)
         return
       when "DELETE"
@@ -121,14 +122,12 @@ The secret to use for signing is:
         unless Webhookdb::RACK_ENV == "test"
           raise "someone tried to use the special unit test google event type outside of unit tests"
         end
-        return super(request)
+        return super
       else
         raise ArgumentError, "Unknown request type: #{request_type}"
     end
   end
 
-  CLEANUP_SERVICE_NAMES = ["icalendar_event_v1"].freeze
-
   def rows_needing_sync(dataset, now: Time.now)
     cutoff = now - Webhookdb::Icalendar.sync_period_hours.hours
     return dataset.where(Sequel[last_synced_at: nil] | Sequel.expr { last_synced_at < cutoff })
@@ -136,7 +135,7 @@ The secret to use for signing is:
 
   def delete_data_for_external_id(external_id)
     relevant_integrations = self.service_integration.recursive_dependents.
-      filter { |d| CLEANUP_SERVICE_NAMES.include?(d.service_name) }
+      filter { |d| Webhookdb::Icalendar::EVENT_REPLICATORS.include?(d.service_name) }
     self.admin_dataset do |ds|
       ds.db.transaction do
         ds.where(external_id:).delete
@@ -157,7 +156,7 @@ The secret to use for signing is:
       @now = now
     end
 
-    def upsert_page_size = 500
+    def upsert_page_size = 2000
     def conditional_upsert? = true
 
     def prepare_body(body)
@@ -166,12 +165,33 @@ The secret to use for signing is:
     end
   end
 
-  def sync_row(row)
+  def sync_row(row, force: false, now: Time.now)
     Appydays::Loggable.with_log_tags(icalendar_url: row.fetch(:ics_url)) do
+      last_synced_at = row.fetch(:last_synced_at)
+      should_sync = force ||
+        last_synced_at.nil? ||
+        # If a proxy is configured, we always want to try to sync,
+        # since this could have come from a webhook, but also the proxy feed refresh TTL
+        # is likely much lower than ICALENDAR_SYNC_PERIOD_HOURS so it's good to check on it.
+        # The check is very fast (should 304) so is safe to do relatively often.
+        Webhookdb::Icalendar.proxy_url.present? ||
+        last_synced_at < (now - Webhookdb::Icalendar.sync_period_hours.hours)
+      unless should_sync
+        self.logger.info("skip_sync_recently_synced", last_synced_at:)
+        return
+      end
       self.with_advisory_lock(row.fetch(:pk)) do
         start = Time.now
-        now = Time.now
-        if (dep = self.find_dependent("icalendar_event_v1"))
+        if (dep = self.find_dependent(Webhookdb::Icalendar::EVENT_REPLICATORS))
+          if dep.replicator.avoid_writes?
+            # Check if this table is being vacuumed/etc. We use this instead of a semaphore job,
+            # since it's a better fit for icalendar, which is pre-scheduled, rather than reactive.
+            # That is, when we receive webhooks, a semaphore job gives us a more predictable rate;
+            # but icalendar rate is negotiated in advance (when enqueing jobs),
+            # and we can be more 'helpful' to something like a vacuum by not running any jobs at all.
+            self.logger.info("skip_sync_table_locked")
+            raise Amigo::Retry::Retry, 60.seconds + (rand * 10.seconds)
+          end
           processor = self._sync_row(row, dep, now:)
         end
         self.admin_dataset do |ds|
@@ -181,6 +201,12 @@ The secret to use for signing is:
               event_count: processor&.upserted_identities&.count,
               feed_bytes: processor&.read_bytes,
               last_sync_duration_ms: (Time.now - start).in_milliseconds,
+              last_fetch_context: {
+                "hash" => processor&.feed_hash,
+                "content_type" => processor&.headers&.fetch("Content-Type", nil),
+                "content_length" => processor&.headers&.fetch("Content-Length", nil),
+                "etag" => processor&.headers&.fetch("Etag", nil),
+              }.to_json,
             )
         end
       end
@@ -191,14 +217,19 @@ The secret to use for signing is:
     calendar_external_id = row.fetch(:external_id)
     begin
       request_url = self._clean_ics_url(row.fetch(:ics_url))
-      io = Webhookdb::Http.chunked_download(request_url, rewindable: false)
-    rescue Down::Error, URI::InvalidURIError => e
+      io = self._make_ics_request(request_url, row.fetch(:last_fetch_context))
+    rescue Down::Error,
+           URI::InvalidURIError,
+           HTTPX::NativeResolveError,
+           HTTPX::InsecureRedirectError,
+           HTTPX::Connection::HTTP2::Error,
+           EOFError => e
       self._handle_down_error(e, request_url:, calendar_external_id:)
       return
     end
 
     upserter = Upserter.new(dep.replicator, calendar_external_id, now:)
-    processor = EventProcessor.new(io, upserter)
+    processor = EventProcessor.new(io:, upserter:, headers: io.data[:headers])
     processor.process
     # Delete all the extra replicator rows, and cancel all the rows that weren't upserted.
     dep.replicator.admin_dataset do |ds|
@@ -219,6 +250,24 @@ The secret to use for signing is:
     return processor
   end
 
+  def _make_ics_request(request_url, last_fetch_context)
+    # Some servers require a VERY explicit accept header,
+    # so tell them we prefer icalendar here.
+    # Using Httpx, Accept-Encoding is gzip,deflate
+    # which seems fine (server should use identity as worst case).
+    headers = {
+      "Accept" => "text/calendar,*/*",
+    }
+    headers["If-None-Match"] = last_fetch_context["etag"] if last_fetch_context & ["etag"]
+    if (proxy_url = Webhookdb::Icalendar.proxy_url).present?
+      request_url = "#{proxy_url.delete_suffix('/')}/?url=#{URI.encode_www_form_component(request_url)}"
+      headers["Authorization"] = "Apikey #{Webhookdb::Icalendar.proxy_api_key}" if
+        Webhookdb::Icalendar.proxy_api_key.present?
+    end
+    resp = Webhookdb::Http.chunked_download(request_url, rewindable: false, headers:)
+    return resp
+  end
+
   # We get all sorts of strange urls, fix up what we can.
   def _clean_ics_url(url)
     u = URI(url)
@@ -251,12 +300,17 @@ The secret to use for signing is:
         else
           self._handle_retryable_down_error!(e, request_url:, calendar_external_id:)
         end
-      when Down::TimeoutError, Down::ConnectionError, Down::InvalidUrl, URI::InvalidURIError
+      when Down::TimeoutError, Down::ConnectionError, Down::InvalidUrl,
+        Errno::ECONNRESET,
+        URI::InvalidURIError,
+        HTTPX::NativeResolveError, HTTPX::InsecureRedirectError,
+        HTTPX::Connection::HTTP2::Error,
+        EOFError
         response_status = 0
         response_body = e.to_s
       when Down::ClientError
         raise e if e.response.nil?
-        response_status = e.response.code.to_i
+        response_status = e.response.status.to_i
         self._handle_retryable_down_error!(e, request_url:, calendar_external_id:) if
           self._retryable_client_error?(e, request_url:)
         # These are all the errors we've seen, we can't do anything about.
@@ -268,18 +322,24 @@ The secret to use for signing is:
           404, 405, # Fundamental issues with the URL given
           409, 410, # More access problems
           417, # If someone uses an Outlook HTML calendar, fetch gives us a 417
+          422, # Sometimes used instead of 404
           429, # Usually 429s are retried (as above), but in some cases they're not.
+          500, 503, 504, # Intermittent server issues, usually
+          599, # Represents a timeout in icalproxy
         ]
         # For most client errors, we can't do anything about it. For example,
         # and 'unshared' URL could result in a 401, 403, 404, or even a 405.
         # For now, other client errors, we can raise on,
         # in case it's something we can fix/work around.
         # For example, it's possible something like a 415 is a WebhookDB issue.
+        if response_status == 421 && (origin_err = e.response.headers["Ical-Proxy-Origin-Error"])
+          response_status = origin_err.to_i
+        end
         raise e unless expected_errors.include?(response_status)
-        response_body = e.response.body.to_s
+        response_body = self._safe_read_body(e)
       when Down::ServerError
-        response_status = e.response.code.to_i
-        response_body = e.response.body.to_s
+        response_status = e.response.status.to_i
+        response_body = self._safe_read_body(e)
       else
         response_body = nil
         response_status = nil
@@ -299,8 +359,16 @@ The secret to use for signing is:
     self.service_integration.organization.alerting.dispatch_alert(message, separate_connection: false)
   end
 
+  # We can hit an error while reading the error body, since it was opened as a stream.
+  # Ignore those errors.
+  def _safe_read_body(e)
+    return e.response.body.to_s
+  rescue OpenSSL::SSL::SSLError, HTTPX::Error
+    return "<error reading body>"
+  end
+
   def _retryable_client_error?(e, request_url:)
-    code = e.response.code.to_i
+    code = e.response.status.to_i
     # This is a bad domain that returns 429 for most requests.
     # Tell the org admins it won't sync.
     return false if code == 429 && request_url.start_with?("https://ical.schedulestar.com")
@@ -317,7 +385,7 @@ The secret to use for signing is:
     retry_in = rand(4..60).minutes
     self.logger.debug(
       "icalendar_fetch_error_retry",
-      response_status: e.respond_to?(:response) ? e.response&.code : 0,
+      response_status: e.respond_to?(:response) ? e.response&.status : 0,
       request_url:,
       calendar_external_id:,
       retry_at: Time.now + retry_in,
@@ -326,11 +394,12 @@ The secret to use for signing is:
   end
 
   class EventProcessor
-    attr_reader :upserted_identities, :read_bytes
+    attr_reader :upserted_identities, :read_bytes, :headers
 
-    def initialize(io, upserter)
+    def initialize(io:, upserter:, headers:)
       @io = io
       @upserter = upserter
+      @headers = headers
       # Keep track of everything we upsert. For any rows we aren't upserting,
       # delete them if they're recurring, or cancel them if they're not recurring.
       # If doing it this way is slow, we could invert this (pull down all IDs and pop from the set).
@@ -346,8 +415,11 @@ The secret to use for signing is:
       # Keep track of the bytes we've read from the file.
       # Never trust Content-Length headers for ical feeds.
       @read_bytes = 0
+      @feed_md5 = Digest::MD5.new
     end
 
+    def feed_hash = @feed_md5.hexdigest
+
     def delete_condition
       return nil if @max_sequence_num_by_uid.empty?
       return @max_sequence_num_by_uid.map do |uid, n|
@@ -486,7 +558,14 @@ The secret to use for signing is:
           # The new UID has the sequence number.
           e["UID"] = {"v" => "#{uid}-#{idx}"}
           e["DTSTART"] = self._ical_entry_from_ruby(occ.start_time, start_entry, is_date)
-          e["DTEND"] = self._ical_entry_from_ruby(occ.end_time, end_entry, is_date) if has_end_time
+          if has_end_time
+            if !is_date && end_entry["VALUE"] == "DATE"
+              # It's possible that DTSTART is a time, but DTEND is a date. This makes no sense,
+              # so skip setting an end date. It will be in the :data column at least.
+            else
+              e["DTEND"] = self._ical_entry_from_ruby(occ.end_time, end_entry, is_date)
+            end
+          end
           yield e
           final_sequence = idx
           break if occ.start_time > dont_project_after
@@ -508,7 +587,11 @@ The secret to use for signing is:
       return {"v" => r.strftime("%Y%m%dT%H%M%S"), "TZID" => tzid} if tzid
       value = entry.fetch("v")
       return {"v" => value} if value.end_with?("Z")
-      raise "Cannot create ical entry from: #{r}, #{entry}, is_date: #{is_date}"
+      if /^\d{8}T\d{6}$/.match?(value)
+        @upserter.upserting_replicator.logger.warn "ical_assuming_utc_time", ical_entry: entry, ruby_time: r
+        return {"v" => "#{value}Z"}
+      end
+      raise "Cannot create ical entry from: '#{r}', #{entry}"
     end
 
     def _icecube_rule_from_ical(ical)
@@ -551,6 +634,7 @@ The secret to use for signing is:
       in_vevent = false
       while (line = @io.gets)
         @read_bytes += line.size
+        @feed_md5.update(line)
         begin
           line.rstrip!
         rescue Encoding::CompatibilityError
@@ -589,4 +673,41 @@ The secret to use for signing is:
       @upserter.upserting_replicator.logger.warn("invalid_vevent_hash", vevent_uids: bad_event_uids.sort)
     end
   end
+
+  # Return true if the data in the feed has changed from what was last synced,
+  # or false if it has not so the sync can be skipped.
+  # This operation is meant to be resource-light (most of the work is the HTTP request),
+  # so should be done in a threadpool.
+  #
+  # - If we have no previous fetch context, we sync.
+  # - If the fetch errors, sync, because we want the normal error handler to figure it out
+  #   (alert admins, etc).
+  # - If the last fetch's content type and length is different from the current, we sync.
+  # - Download the bytes. If the hash of the bytes is different from what was last processed,
+  #   sync. Since this involves reading the streaming body, we must return a copy of the body (a StringIO).
+  def feed_changed?(row)
+    last_fetch = row.fetch(:last_fetch_context)
+    return true if last_fetch.nil? || last_fetch.empty?
+
+    begin
+      url = self._clean_ics_url(row.fetch(:ics_url))
+      resp = self._make_ics_request(url, last_fetch)
+    rescue Down::NotModified
+      return false
+    rescue StandardError
+      return true
+    end
+    headers = resp.data[:headers] || {}
+    content_type_match = headers["Content-Type"] == last_fetch["content_type"] &&
+      headers["Content-Length"] == last_fetch["content_length"]
+    return true unless content_type_match
+    last_hash = last_fetch["hash"]
+    return true if last_hash.nil?
+
+    hash = Digest::MD5.new
+    while (line = resp.gets)
+      hash.update(line)
+    end
+    return hash.hexdigest != last_hash
+  end
 end

data/lib/webhookdb/replicator/icalendar_event_v1.rb

@@ -2,6 +2,7 @@
 
 require "webhookdb/icalendar"
 require "webhookdb/windows_tz"
+require "webhookdb/replicator/base_stale_row_deleter"
 
 class Webhookdb::Replicator::IcalendarEventV1 < Webhookdb::Replicator::Base
   include Appydays::Loggable
@@ -115,7 +116,6 @@ class Webhookdb::Replicator::IcalendarEventV1 < Webhookdb::Replicator::Base
       :compound_identity,
       TEXT,
       data_key: "<compound key, see converter>",
-      index: true,
       converter: CONV_REMOTE_KEY,
       optional: true, # This is done via the converter, data_key never exists
     )
@@ -166,6 +166,7 @@ class Webhookdb::Replicator::IcalendarEventV1 < Webhookdb::Replicator::Base
     data.delete("calendar_external_id")
     data.delete("recurring_event_id")
     data.delete("recurring_event_sequence")
+    data.delete("row_updated_at")
     return data
   end
 
@@ -215,11 +216,21 @@ class Webhookdb::Replicator::IcalendarEventV1 < Webhookdb::Replicator::Base
         columns: [:calendar_external_id, :start_date, :end_date],
         where: Sequel[:status].is_distinct_from("CANCELLED") & (Sequel[:start_date] !~ nil),
       ),
+      Webhookdb::Replicator::IndexSpec.new(
+        columns: [:row_updated_at],
+        where: Sequel[status: "CANCELLED"],
+        identifier: "cancelled_row_updated_at",
+      ),
     ]
   end
 
   def _update_where_expr
-    return self.qualified_table_sequel_identifier[:last_modified_at] < Sequel[:excluded][:last_modified_at]
+    # Compare against data to avoid the constant writes. JSONB != operations are very fast,
+    # so this should not be any real performance issue.
+    # last_modified_at is unreliable because LAST-MODIFIED is unreliable,
+    # even in feeds it is set. There are cases, such as adding an EXDATE to an RRULE,
+    # that do not trigger LAST-MODIFIED changes.
+    return self.qualified_table_sequel_identifier[:data] !~ Sequel[:excluded][:data]
   end
 
   # @param [Array<String>] lines
@@ -369,50 +380,15 @@ class Webhookdb::Replicator::IcalendarEventV1 < Webhookdb::Replicator::Base
   # +stale_at+ to +age_cutoff+. This avoids endlessly adding to the icalendar events table
   # due to feeds that change UIDs each fetch- events with changed UIDs will become CANCELLED,
   # and then deleted over time.
-  # @param stale_at [Time] When an event is considered 'stale'.
-  #   If stale events are a big problem, this can be shortened to just a few days.
-  # @param age_cutoff [Time] Where to stop searching for old events.
-  #   This is important to avoid a full table scale when deleting events,
-  #   since otherwise it is like 'row_updated_at < 35.days.ago'.
-  #   Since this routine should run regularly, we should rarely have events more than 35 or 36 days old,
-  #   for example.
-  #   Use +nil+ to use no limit (a full table scan) which may be necessary when running this feature
-  #   for the first time.
-  # @param chunk_size [Integer] The row delete is done in chunks to avoid long locks.
-  #   The default seems safe, but it's exposed as a parameter if you need to play around with it,
-  #   and can be done via configuration if needed at some point.
-  def delete_stale_cancelled_events(
-    stale_at: Webhookdb::Icalendar.stale_cancelled_event_threshold_days.days.ago,
-    age_cutoff: (Webhookdb::Icalendar.stale_cancelled_event_threshold_days + 10).days.ago,
-    chunk_size: 10_000
-  )
-    # Delete in chunks, like:
-    #   DELETE from "public"."icalendar_event_v1_aaaa"
-    #   WHERE pk IN (
-    #     SELECT pk FROM "public"."icalendar_event_v1_aaaa"
-    #     WHERE row_updated_at < (now() - '35 days'::interval)
-    #     LIMIT 10000
-    #   )
-    age = age_cutoff..stale_at
-    self.admin_dataset do |ds|
-      chunk_ds = ds.where(row_updated_at: age, status: "CANCELLED").select(:pk).limit(chunk_size)
-      loop do
-        # Due to conflicts where a feed is being inserted while the delete is happening,
-        # this may raise an error like:
-        #   deadlock detected
-        #   DETAIL:  Process 18352 waits for ShareLock on transaction 435085606; blocked by process 24191.
-        #   Process 24191 waits for ShareLock on transaction 435085589; blocked by process 18352.
-        #   HINT:  See server log for query details.
-        #   CONTEXT:  while deleting tuple (2119119,3) in relation "icalendar_event_v1_aaaa"
-        # Unit testing this is very difficult though, and in practice it is rare,
-        # and normal Sidekiq job retries should be sufficient to handle this.
-        # So we don't explicitly handle deadlocks, but could if it becomes an issue.
-        deleted = ds.where(pk: chunk_ds).delete
-        break if deleted != chunk_size
-      end
-    end
+  class StaleRowDeleter < Webhookdb::Replicator::BaseStaleRowDeleter
+    def stale_at = Webhookdb::Icalendar.stale_cancelled_event_threshold_days.days
+    def lookback_window = Webhookdb::Icalendar.stale_cancelled_event_lookback_days.days
+    def updated_at_column = :row_updated_at
+    def stale_condition = {status: "CANCELLED"}
   end
 
+  def stale_row_deleter = StaleRowDeleter.new(self)
+
   def calculate_webhook_state_machine
     if (step = self.calculate_dependency_state_machine_step(dependency_help: ""))
       return step

data/lib/webhookdb/replicator/icalendar_event_v1_partitioned.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "webhookdb/replicator/icalendar_event_v1"
+require "webhookdb/replicator/partitionable_mixin"
+
+class Webhookdb::Replicator::IcalendarEventV1Partitioned < Webhookdb::Replicator::IcalendarEventV1
+  include Webhookdb::Replicator::PartitionableMixin
+
+  # @return [Webhookdb::Replicator::Descriptor]
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "icalendar_event_v1_partitioned",
+      ctor: ->(sint) { self.new(sint) },
+      dependency_descriptor: Webhookdb::Replicator::IcalendarCalendarV1.descriptor,
+      feature_roles: ["partitioning_beta"],
+      resource_name_singular: "iCalendar Event",
+      supports_webhooks: true,
+      description: "Individual events in an icalendar, using partitioned tables rather than one big table. " \
+                   "See icalendar_calendar_v1.",
+      api_docs_url: "https://icalendar.org/",
+    )
+  end
+
+  def _denormalized_columns
+    d = super
+    d << Webhookdb::Replicator::Column.new(:calendar_external_hash, INTEGER, optional: true)
+    return d
+  end
+
+  def partition_method = Webhookdb::DBAdapter::Partitioning::HASH
+  def partition_column_name = :calendar_external_hash
+  def partition_value(resource) = self._str2inthash(resource.fetch("calendar_external_id"))
+end

data/lib/webhookdb/replicator/intercom_contact_v1.rb

@@ -43,6 +43,7 @@ class Webhookdb::Replicator::IntercomContactV1 < Webhookdb::Replicator::Base
   end
 
   def _mixin_backfill_url = "https://api.intercom.io/contacts"
+  def _mixin_backfill_hashkey = "data"
 
   def _resource_and_event(request)
     resource, event = super

data/lib/webhookdb/replicator/intercom_conversation_v1.rb

@@ -40,6 +40,7 @@ class Webhookdb::Replicator::IntercomConversationV1 < Webhookdb::Replicator::Bas
   end
 
   def _mixin_backfill_url = "https://api.intercom.io/conversations"
+  def _mixin_backfill_hashkey = "conversations"
 
   def _resource_and_event(request)
     resource, event = super

data/lib/webhookdb/replicator/intercom_v1_mixin.rb

@@ -73,8 +73,9 @@ module Webhookdb::Replicator::IntercomV1Mixin
   end
 
   def _mixin_backfill_url = raise NotImplementedError
+  def _mixin_backfill_hashkey = raise NotImplementedError
 
-  def _fetch_backfill_page(pagination_token, **_kwargs)
+  def _fetch_backfill_page(pagination_token, last_backfilled:)
     unless self.auth_credentials?
       raise Webhookdb::Replicator::CredentialsMissing,
             "This integration requires that the Intercom Auth integration has a valid Auth Token"
@@ -123,8 +124,29 @@ module Webhookdb::Replicator::IntercomV1Mixin
       # a TypeError in the backfiller.
       return [], nil
     end
-    data = response.parsed_response.fetch("data", [])
+    data = response.parsed_response.fetch(self._mixin_backfill_hashkey)
     starting_after = response.parsed_response.dig("pages", "next", "starting_after")
+    # Intercom pagination sorts by updated_at newest. So if we are doing an incremental sync (last_backfilled set),
+    # and we last backfilled after the latest updated_at, we can stop paginating.
+    if last_backfilled && data.last && data.last["updated_at"]
+      oldest_update = Time.at(data.last["updated_at"])
+      starting_after = nil if oldest_update < last_backfilled
+    end
     return data, starting_after
   end
+
+  def _backfillers
+    return [Backfiller.new(self)]
+  end
+
+  class Backfiller < Webhookdb::Replicator::Base::ServiceBackfiller
+    include Webhookdb::Backfiller::Bulk
+
+    # Upsert for each API call
+    def upsert_page_size = Webhookdb::Intercom.page_size
+    def prepare_body(_body) = nil
+    def upserting_replicator = self.svc
+    # We don't want to override newer items from webhooks, so use conditional upsert.
+    def conditional_upsert? = true
+  end
 end

data/lib/webhookdb/replicator/partitionable_mixin.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+# Mixin for replicators that support partitioning.
+# Partitioning is currently in beta,
+# with the following limitations/context:
+#
+# - They cannot be created from the CLI.
+#   Because the partitions must be created during the CREATE TABLE call,
+#   the partition_value must be set immediately on creation,
+#   or CREATE TABLE must be deferred.
+# - CLI support would also require making sure this field isn't edited.
+#   This is an annoying change, so we're putting it off for now.
+# - Instead, partitioned replicators must be created in the console.
+# - The number of HASH partitions cannot be changed;
+#   there is no good way to handle this in Postgres so we don't bother here.
+# - RANGE partitions are not supported.
+#   We need to support creating the partition when the INSERT fails.
+#   But creating the partitioned table definition itself does work/has a shared behavior at least.
+# - Existing replicators cannot be converted to partitioned.
+#   This is theoretically possible, but it seems easier to just start over
+#   with a new replicator.
+# - Instead:
+#   - If this is a 'child' replicator, then create a new parent and this child,
+#     then copy over the parent data, either directly (for icalendar)
+#     or using HTTP requests (like with Plaid or Google) where more logic is required.
+#   - Otherwise, it'll depend on the replicator.
+#   - Then to switch clients using the old replicator, to the new replicator, you can:
+#     - Then turn off all workers.
+#     - Rename the new table to the old, and old table to the new.
+#     - Update the service integrations, so the old one points to the new table name and opaque id,
+#       and the new one points to the old table name and opaque id.
+#
+module Webhookdb::Replicator::PartitionableMixin
+  # The partition method, like Webhookdb::DBAdapter::Partitioning::HASH
+  def partition_method = raise NotImplementedError
+  # The partition column name.
+  # Must be present in +_denormalized_columns+.
+  # @return [Symbol]
+  def partition_column_name = raise NotImplementedError
+  # The value for the denormalized column. For HASH partitioning this would be an integer,
+  # for RANGE partitioning this could be a timestamp, etc.
+  # Takes the resource and returns the value.
+  def partition_value(_resource) = raise NotImplementedError
+
+  def partition? = true
+
+  def partitioning
+    return Webhookdb::DBAdapter::Partitioning.new(by: self.partition_method, column: self.partition_column_name)
+  end
+
+  def _prepare_for_insert(resource, event, request, enrichment)
+    h = super
+    h[self.partition_column_name] = self.partition_value(resource)
+    return h
+  end
+
+  def _upsert_conflict_target
+    return [self.partition_column_name, self._remote_key_column.name]
+  end
+
+  # Convert the given string into a stable MD5-derived hash
+  # that can be stored in a (signed, 4 bit) INTEGER column.
+  def _str2inthash(s)
+    # MD5 is 128 bits/16 bytes/32 hex chars (2 chars per byte).
+    # Integers are 32 bits/4 bytes/8 hex chars.
+    # Grab the first 8 chars and convert it to an integer.
+    unsigned_md5int = Digest::MD5.hexdigest(s)[..8].to_i(16)
+    # Then AND it with a 32 bit bitmask to make sure it fits in 32 bits
+    # (though I'm not entirely sure why the above doesn't result in 32 bits always).
+    unsigned_int32 = unsigned_md5int & 0xFFFFFFFF
+    # Convert it from unsigned (0 to 4.2B) to signed (-2.1B to 2.1B) by subtracting 2.1B
+    # (the max 2 byte integer), as opposed to a 4 byte integer which we're dealing with here.
+    signed_md5int = unsigned_int32 - MAX_16BIT_INT
+    return signed_md5int
+  end
+
+  MAX_16BIT_INT = 2**31
+
+  # Return the partitions belonging to the table.
+  # @param db The organization connection.
+  # @return [Array<Webhookdb::DBAdapter::Partition>]
+  def existing_partitions(db)
+    # SELECT inhrelid::regclass AS child
+    # FROM   pg_catalog.pg_inherits
+    # WHERE  inhparent = 'my_schema.foo'::regclass;
+    parent = self.schema_and_table_symbols.map(&:to_s).join(".")
+    partnames = db[Sequel[:pg_catalog][:pg_inherits]].
+      where(inhparent: Sequel[parent].cast(:regclass)).
+      select_map(Sequel[:inhrelid].cast(:regclass))
+    parent_table = self.dbadapter_table
+    result = partnames.map do |part|
+      suffix = self.partition_suffix(part)
+      Webhookdb::DBAdapter::Partition.new(parent_table:, partition_name: part.to_sym, suffix:)
+    end
+    return result
+  end
+
+  def partition_suffix(partname)
+    return partname[/_[a-zA-Z\d]+$/].to_sym
+  end
+
+  def partition_align_name
+    tblname = self.service_integration.table_name
+    self.service_integration.organization.admin_connection do |db|
+      partitions = self.existing_partitions(db)
+      db.transaction do
+        partitions.each do |partition|
+          next if partition.partition_name.to_s.start_with?(tblname)
+          schema = partition.parent_table.schema.name
+          new_partname = "#{tblname}#{partition.suffix}"
+          db << "ALTER TABLE #{schema}.#{partition.partition_name} RENAME TO #{new_partname}"
+        end
+      end
+    end
+  end
+end

data/lib/webhookdb/replicator/shopify_v1_mixin.rb

@@ -45,7 +45,7 @@ module Webhookdb::Replicator::ShopifyV1Mixin
       field = "api_url"
       value = "https://#{value}.myshopify.com"
     end
-    return super(field, value)
+    return super
   end
 
   def calculate_webhook_state_machine

data/lib/webhookdb/replicator/signalwire_message_v1.rb

@@ -52,7 +52,7 @@ class Webhookdb::Replicator::SignalwireMessageV1 < Webhookdb::Replicator::Base
       h = u.host.gsub(/\.signalwire\.com$/, "")
       value = h
     end
-    return super(field, value, attr:)
+    return super
   end
 
   def calculate_backfill_state_machine
@@ -200,7 +200,6 @@ Press 'Show' next to the newly-created API token, and copy it.)
       request_url = e.uri.to_s
       request_method = e.http_method
     end
-    self.logger.warn("signalwire_backfill_error", response_body:, response_status:, request_url:)
     message = Webhookdb::Messages::ErrorGenericBackfill.new(
       self.service_integration,
       response_status:,

data/lib/webhookdb/replicator/sponsy_v1_mixin.rb

@@ -99,7 +99,7 @@ module Webhookdb::Replicator::SponsyV1Mixin
       )
     rescue Webhookdb::Http::Error => e
       raise e unless e.status == 404
-      self.logger.warn("sponsy_404", error: e)
+      self.logger.warn("sponsy_404", e)
       return [], nil
     end