webhookdb 1.2.2 → 1.3.1

data/lib/webhookdb/oauth/session.rb

@@ -22,3 +22,23 @@ class Webhookdb::Oauth::Session < Webhookdb::Postgres::Model(:oauth_sessions)
     }
   end
 end
+
+# Table: oauth_sessions
+# -------------------------------------------------------------------------------------------------------
+# Columns:
+#  id                 | integer                  | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY
+#  created_at         | timestamp with time zone | NOT NULL DEFAULT now()
+#  customer_id        | integer                  |
+#  organization_id    | integer                  |
+#  user_agent         | text                     | NOT NULL
+#  peer_ip            | inet                     | NOT NULL
+#  oauth_state        | text                     | NOT NULL
+#  authorization_code | text                     |
+#  used_at            | timestamp with time zone |
+# Indexes:
+#  oauth_sessions_pkey              | PRIMARY KEY btree (id)
+#  oauth_sessions_customer_id_index | btree (customer_id)
+# Foreign key constraints:
+#  oauth_sessions_customer_id_fkey     | (customer_id) REFERENCES customers(id) ON DELETE CASCADE
+#  oauth_sessions_organization_id_fkey | (organization_id) REFERENCES organizations(id) ON DELETE CASCADE
+# -------------------------------------------------------------------------------------------------------

data/lib/webhookdb/oauth.rb

@@ -11,7 +11,6 @@ module Webhookdb::Oauth
     end
   end
 
-  # rubocop:disable Lint/UnusedMethodArgument
   class Provider
     # @return [String] Unique key to identify the provider.
     def key = raise NotImplementedError
@@ -19,13 +18,6 @@ module Webhookdb::Oauth
     # @return [String] Name of the app to present to users.
     def app_name = raise NotImplementedError
 
-    # True if auth with this provider requires the user auth in WebhookDB,
-    # false if we can get their email from the Oauth process.
-    # If the access token can be used to get the 'me' user,
-    # we can usually use their email for the customer,
-    # but this may not be possible for some integrations.
-    def requires_webhookdb_auth? = raise NotImplementedError
-
     # This is similar to `supports_webhooks` in the Replicator descriptors,
     # except that this is used to make the success page dynamic.
     # True if this provider's integrations support webhooks
@@ -39,22 +31,12 @@ module Webhookdb::Oauth
     # @return [Webhookdb::Oauth::Tokens]
     def exchange_authorization_code(code:) = raise NotImplementedError
 
-    # @param tokens [Webhookdb::Oauth::Tokens]
-    # @param scope [Hash] Used to store data needed in later calls, like when building integrations.
-    # @return [Array{TrueClass, FalseClass, Webhookdb::Customer}]
-    def find_or_create_customer(tokens:, scope:)
-      raise RuntimeError("should not be called") if self.requires_webhookdb_auth?
-      raise NotImplementedError
-    end
-
     # Create the actual service integrations for the given org.
     # @param organization [Webhookdb::Organization]
     # @param tokens [Webhookdb::Oauth::Tokens]
-    # # @param scope [Hash]
-    def build_marketplace_integrations(organization:, tokens:, scope:) = raise NotImplementedError
-  end
-  # rubocop:enable Lint/UnusedMethodArgument
-
+    # @return [Webhookdb::ServiceIntegration]
+    def build_marketplace_integrations(organization:, tokens:) = raise NotImplementedError
+end
   class << self
     def register(cls)
       key = cls.new.key
@@ -74,8 +56,12 @@ module Webhookdb::Oauth
   end
 end
 
+require "webhookdb/oauth/fake_provider"
+Webhookdb::Oauth.register(Webhookdb::Oauth::FakeProvider)
 require "webhookdb/oauth/front_provider"
 Webhookdb::Oauth.register(Webhookdb::Oauth::FrontProvider)
 Webhookdb::Oauth.register(Webhookdb::Oauth::FrontSignalwireChannelProvider)
+require "webhookdb/oauth/increase_provider"
+Webhookdb::Oauth.register(Webhookdb::Oauth::IncreaseProvider)
 require "webhookdb/oauth/intercom_provider"
 Webhookdb::Oauth.register(Webhookdb::Oauth::IntercomProvider)

data/lib/webhookdb/organization/alerting.rb

@@ -31,11 +31,13 @@ class Webhookdb::Organization::Alerting
     end
     signature = message_template.signature
     max_alerts_per_customer_per_day = Webhookdb::Organization::Alerting.max_alerts_per_customer_per_day
+    yesterday = Time.now - 24.hours
     self.org.admin_customers.each do |c|
       idemkey = "orgalert-#{signature}-#{c.id}"
       Webhookdb::Idempotency.every(Webhookdb::Organization::Alerting.interval).under_key(idemkey) do
         sent_last_day = Webhookdb::Message::Delivery.
           where(template: message_template.full_template_name, recipient: c).
+          where { created_at > yesterday }.
           limit(max_alerts_per_customer_per_day).
           count
         next unless sent_last_day < max_alerts_per_customer_per_day

data/lib/webhookdb/organization/database_migration.rb

@@ -7,6 +7,7 @@ class Webhookdb::Organization::DatabaseMigration < Webhookdb::Postgres::Model(:o
   class MigrationAlreadyFinished < StandardError; end
 
   plugin :timestamps
+  plugin :text_searchable, terms: [:organization, :started_by]
   plugin :column_encryption do |enc|
     enc.column :source_admin_connection_url
     enc.column :destination_admin_connection_url
@@ -14,6 +15,7 @@ class Webhookdb::Organization::DatabaseMigration < Webhookdb::Postgres::Model(:o
 
   many_to_one :started_by, class: "Webhookdb::Customer"
   many_to_one :organization, class: "Webhookdb::Organization"
+  many_to_one :last_migrated_service_integration, class: "Webhookdb::ServiceIntegration"
 
   dataset_module do
     def ongoing
@@ -141,6 +143,7 @@ end
 #  organization_schema                  | text                     |
 #  last_migrated_service_integration_id | integer                  | NOT NULL DEFAULT 0
 #  last_migrated_timestamp              | timestamp with time zone |
+#  text_search                          | tsvector                 |
 # Indexes:
 #  organization_database_migrations_pkey                  | PRIMARY KEY btree (id)
 #  one_inprogress_migration_per_org                       | UNIQUE btree (organization_id) WHERE finished_at IS NULL

data/lib/webhookdb/organization.rb

@@ -11,6 +11,7 @@ class Webhookdb::Organization < Webhookdb::Postgres::Model(:organizations)
 
   plugin :timestamps
   plugin :soft_deletes
+  plugin :text_searchable, terms: [:name, :key, :billing_email]
   plugin :column_encryption do |enc|
     enc.column :readonly_connection_url_raw
     enc.column :admin_connection_url_raw
@@ -35,6 +36,7 @@ class Webhookdb::Organization < Webhookdb::Postgres::Model(:organizations)
               order: :id
   one_to_many :service_integrations, class: "Webhookdb::ServiceIntegration", order: :id
   one_to_many :saved_queries, class: "Webhookdb::SavedQuery", order: :id
+  one_to_many :saved_views, class: "Webhookdb::SavedView", order: :id
   one_to_many :webhook_subscriptions, class: "Webhookdb::WebhookSubscription", order: :id
   many_to_many :feature_roles, class: "Webhookdb::Role", join_table: :feature_roles_organizations, right_key: :role_id
   one_to_many :all_webhook_subscriptions,
@@ -145,6 +147,33 @@ class Webhookdb::Organization < Webhookdb::Postgres::Model(:organizations)
     end
   end
 
+  # Run the given SQL inside the org, and use special error handling if it fails.
+  # @return [Array<Webhookdb::Organization::QueryResult,String,nil>] Tuple of query result, and optional message.
+  #   On query success, return <QueryResult, nil>.
+  #   On DatabaseError, return <nil, message>.
+  #   On other types of errors, raise.
+  def execute_readonly_query_with_help(sql)
+    result = self.execute_readonly_query(sql)
+    return result, nil
+  rescue Sequel::DatabaseError => e
+    self.logger.error("db_query_database_error", error: e)
+    # We want to handle InsufficientPrivileges and UndefinedTable explicitly
+    # since we can hint the user at what to do.
+    # Otherwise, we should just return the Postgres exception.
+    msg = ""
+    case e.wrapped_exception
+      when PG::UndefinedTable
+        missing_table = e.wrapped_exception.message.match(/relation (.+) does not/)&.captures&.first
+        msg = "The table #{missing_table} does not exist. Run `webhookdb db tables` to see available tables." if
+          missing_table
+      when PG::InsufficientPrivilege
+        msg = "You do not have permission to perform this query. Queries must be read-only."
+      else
+        msg = e.wrapped_exception.message
+    end
+    return [nil, msg]
+  end
+
   class QueryResult
     attr_accessor :rows, :columns, :max_rows_reached
   end
@@ -355,10 +384,7 @@ class Webhookdb::Organization < Webhookdb::Postgres::Model(:organizations)
   end
 
   def migrate_replication_schema(schema)
-    unless Webhookdb::DBAdapter::VALID_IDENTIFIER.match?(schema)
-      msg = "Sorry, this is not a valid schema name. " + Webhookdb::DBAdapter::INVALID_IDENTIFIER_MESSAGE
-      raise SchemaMigrationError, msg
-    end
+    Webhookdb::DBAdapter.validate_identifier!(schema, type: "schema")
     Webhookdb::Organization::DatabaseMigration.guard_ongoing!(self)
     raise SchemaMigrationError, "destination and target schema are the same" if schema == self.replication_schema
     builder = Webhookdb::Organization::DbBuilder.new(self)
@@ -446,7 +472,7 @@ class Webhookdb::Organization < Webhookdb::Postgres::Model(:organizations)
     return Webhookdb::ServiceIntegration.where(organization: self).count < limit
   end
 
-  def available_replicator_names
+  def available_replicators
     available = Webhookdb::Replicator.registry.values.filter do |desc|
       # The org must have any of the flags required for the service. In other words,
       # the intersection of desc[:feature_roles] & org.feature_roles must
@@ -456,7 +482,7 @@ class Webhookdb::Organization < Webhookdb::Postgres::Model(:organizations)
       org_has_access = (self.feature_roles.map(&:name) & desc.feature_roles).present?
       org_has_access
     end
-    return available.map(&:name)
+    return available
   end
 
   #
@@ -498,6 +524,8 @@ require "webhookdb/organization/db_builder"
 #  minimum_sync_seconds        | integer                  | NOT NULL
 #  sync_target_timeout         | integer                  | NOT NULL DEFAULT 30
 #  max_query_rows              | integer                  |
+#  priority_backfill           | boolean                  | NOT NULL DEFAULT false
+#  text_search                 | tsvector                 |
 # Indexes:
 #  organizations_pkey     | PRIMARY KEY btree (id)
 #  organizations_key_key  | UNIQUE btree (key)
@@ -505,8 +533,11 @@ require "webhookdb/organization/db_builder"
 # Referenced By:
 #  feature_roles_organizations      | feature_roles_organizations_organization_id_fkey      | (organization_id) REFERENCES organizations(id)
 #  logged_webhooks                  | logged_webhooks_organization_id_fkey                  | (organization_id) REFERENCES organizations(id) ON DELETE CASCADE
+#  oauth_sessions                   | oauth_sessions_organization_id_fkey                   | (organization_id) REFERENCES organizations(id) ON DELETE CASCADE
 #  organization_database_migrations | organization_database_migrations_organization_id_fkey | (organization_id) REFERENCES organizations(id) ON DELETE CASCADE
 #  organization_memberships         | organization_memberships_organization_id_fkey         | (organization_id) REFERENCES organizations(id)
+#  saved_queries                    | saved_queries_organization_id_fkey                    | (organization_id) REFERENCES organizations(id) ON DELETE CASCADE
+#  saved_views                      | saved_views_organization_id_fkey                      | (organization_id) REFERENCES organizations(id) ON DELETE CASCADE
 #  service_integrations             | service_integrations_organization_id_fkey             | (organization_id) REFERENCES organizations(id)
 #  webhook_subscriptions            | webhook_subscriptions_organization_id_fkey            | (organization_id) REFERENCES organizations(id)
 # ------------------------------------------------------------------------------------------------------------------------------------------------------------

data/lib/webhookdb/organization_membership.rb

@@ -5,10 +5,16 @@ require "webhookdb/postgres/model"
 class Webhookdb::OrganizationMembership < Webhookdb::Postgres::Model(:organization_memberships)
   VALID_ROLE_NAMES = ["admin", "member"].freeze
 
+  plugin :text_searchable, terms: [:customer, :organization, :membership_role]
+
   many_to_one :organization, class: "Webhookdb::Organization"
   many_to_one :customer, class: "Webhookdb::Customer"
   many_to_one :membership_role, class: "Webhookdb::Role"
 
+  dataset_module do
+    def admin = self.where(membership_role: Webhookdb::Role.admin_role)
+  end
+
   def verified?
     return self.verified
   end
@@ -38,13 +44,14 @@ end
 # Table: organization_memberships
 # ------------------------------------------------------------------------------------------------------------------------------
 # Columns:
-#  id                 | integer | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY
-#  customer_id        | integer | NOT NULL
-#  organization_id    | integer | NOT NULL
-#  verified           | boolean | NOT NULL
-#  invitation_code    | text    | NOT NULL DEFAULT ''::text
-#  membership_role_id | integer | NOT NULL
-#  is_default         | boolean | NOT NULL DEFAULT false
+#  id                 | integer  | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY
+#  customer_id        | integer  | NOT NULL
+#  organization_id    | integer  | NOT NULL
+#  verified           | boolean  | NOT NULL
+#  invitation_code    | text     | NOT NULL DEFAULT ''::text
+#  membership_role_id | integer  | NOT NULL
+#  is_default         | boolean  | NOT NULL DEFAULT false
+#  text_search        | tsvector |
 # Indexes:
 #  organization_memberships_pkey | PRIMARY KEY btree (id)
 #  one_default_per_customer      | UNIQUE btree (customer_id, organization_id) WHERE is_default IS TRUE

data/lib/webhookdb/postgres.rb

@@ -48,6 +48,7 @@ module Webhookdb::Postgres
   # Require paths for all Sequel models used by the app.
   MODELS = [
     "webhookdb/backfill_job",
+    "webhookdb/backfill_job/service_integration_lock",
     "webhookdb/customer",
     "webhookdb/customer/reset_code",
     "webhookdb/database_document",
@@ -61,6 +62,7 @@ module Webhookdb::Postgres
     "webhookdb/organization_membership",
     "webhookdb/role",
     "webhookdb/saved_query",
+    "webhookdb/saved_view",
     "webhookdb/service_integration",
     "webhookdb/subscription",
     "webhookdb/sync_target",

data/lib/webhookdb/replicator/base.rb

@@ -159,6 +159,7 @@ class Webhookdb::Replicator::Base
   def process_state_change(field, value, attr: nil)
     attr ||= field
     desc = self.descriptor
+    value = value.strip if value.respond_to?(:strip)
     case field
       when *self._webhook_state_change_fields
         # If we don't support webhooks, then the backfill state machine may be using it.

data/lib/webhookdb/replicator/docgen.rb

@@ -55,13 +55,21 @@ class Webhookdb::Replicator::Docgen
 
   def _intro
     lines << "# #{desc.resource_name_singular} (`#{desc.name}`)"
+    if desc.enterprise?
+      lines << ""
+      lines << "{% include enterprise_integration_list.md %}"
+      lines << ""
+    end
     if desc.description.present?
       lines << ""
       lines << desc.description
     end
+    lines << ""
+    lines << "To get set up, run this code from the [WebhookDB CLI](https://webhookdb.com/terminal):"
+    lines << "```\nwebhookdb integrations create #{desc.name}\n```"
     if desc.api_docs_url.present?
       lines << ""
-      lines << "Docs for this API: [#{desc.api_docs_url}](#{desc.api_docs_url})"
+      lines << "Source documentation for this API: [#{desc.api_docs_url}](#{desc.api_docs_url})"
     end
     lines << ""
   end

data/lib/webhookdb/replicator/fake.rb

@@ -109,9 +109,8 @@ class Webhookdb::Replicator::Fake < Webhookdb::Replicator::Base
   end
 
   def _resource_and_event(request)
-    body = request.body
-    return self.class.resource_and_event_hook.call(body) if self.class.resource_and_event_hook
-    return body, nil
+    return self.class.resource_and_event_hook.call(request) if self.class.resource_and_event_hook
+    return request.body, nil
   end
 
   def _update_where_expr

data/lib/webhookdb/replicator/front_signalwire_message_channel_app_v1.rb

@@ -2,6 +2,7 @@
 
 require "jwt"
 
+require "webhookdb/messages/error_signalwire_send_sms"
 require "webhookdb/replicator/front_v1_mixin"
 
 # Front has a system of 'channels' but it is a challenge to use.
@@ -148,7 +149,7 @@ All of this information can be found in the WebhookDB docs, at https://docs.webh
         self.service_integration.save_changes
         return {type: "success", webhook_url: "#{Webhookdb.api_url}/v1/install/front_signalwire/channel"}.to_json
       when "delete"
-        self.service_integration.destroy
+        self.service_integration.destroy_self_and_all_dependents
         return "{}"
       when "message", "message_autoreply"
         return {
@@ -269,27 +270,61 @@ All of this information can be found in the WebhookDB docs, at https://docs.webh
           item[:signalwire_sid] = "skipped_due_to_age"
         else
           # send the SMS via signalwire
-          signalwire_resp = idempotency.execute do
-            Webhookdb::Signalwire.send_sms(
-              from: sender,
-              to: recipient,
-              body:,
-              space_url: @signalwire_sint.api_url,
-              project_id: @signalwire_sint.backfill_key,
-              api_key: @signalwire_sint.backfill_secret,
-              logger: @replicator.logger,
-            )
-          end
-          item[:signalwire_sid] = signalwire_resp.fetch("sid")
+          signalwire_resp = _send_sms(
+            idempotency,
+            from: sender,
+            to: recipient,
+            body:,
+          )
+          item[:signalwire_sid] = signalwire_resp.fetch("sid") if signalwire_resp
         end
       end
       @replicator.upsert_webhook_body(item.deep_stringify_keys)
     end
 
+    def _send_sms(idempotency, from:, to:, body:)
+      return idempotency.execute do
+        Webhookdb::Signalwire.send_sms(
+          from:,
+          to:,
+          body:,
+          space_url: @signalwire_sint.api_url,
+          project_id: @signalwire_sint.backfill_key,
+          api_key: @signalwire_sint.backfill_secret,
+          logger: @replicator.logger,
+        )
+      end
+    rescue Webhookdb::Http::Error => e
+      response_body = e.body
+      response_status = e.status
+      request_url = e.uri.to_s
+      @replicator.logger.warn("signalwire_send_sms_error",
+                              response_body:, response_status:, request_url:, sms_from: from, sms_to: to,)
+      code = begin
+        # If this fails for whatever reason, or there is no 'code', re-raise the original error
+        e.response.parsed_response["code"]
+      rescue StandardError
+        nil
+      end
+      # All known codes are for the integrator, not on the webhookdb code side.
+      # https://developer.signalwire.com/guides/how-to-troubleshoot-common-messaging-issues
+      raise e if code.nil?
+
+      message = Webhookdb::Messages::ErrorSignalwireSendSms.new(
+        @replicator.service_integration,
+        response_status:,
+        response_body:,
+        request_url:,
+        request_method: "POST",
+      )
+      @replicator.service_integration.organization.alerting.dispatch_alert(message)
+      return nil
+    end
+
     def _sync_front_inbound(sender:, texted_at:, item:, body:)
       body = {
         sender: {handle: sender},
-        body:,
+        body: body || "<no body>",
         delivered_at: texted_at.to_i,
         metadata: {
           external_id: item.fetch(:external_id),

data/lib/webhookdb/replicator/icalendar_calendar_v1.rb

@@ -146,11 +146,12 @@ The secret to use for signing is:
 
   class Upserter
     include Webhookdb::Backfiller::Bulk
-    attr_reader :upserting_replicator, :calendar_external_id
+    attr_reader :upserting_replicator, :calendar_external_id, :now
 
-    def initialize(replicator, calendar_external_id)
+    def initialize(replicator, calendar_external_id, now:)
       @upserting_replicator = replicator
       @calendar_external_id = calendar_external_id
+      @now = now
     end
 
     def upsert_page_size = 500
@@ -158,31 +159,33 @@ The secret to use for signing is:
 
     def prepare_body(body)
       body["calendar_external_id"] = @calendar_external_id
+      body["row_updated_at"] = @now
     end
   end
 
   def sync_row(row)
     Appydays::Loggable.with_log_tags(icalendar_url: row.fetch(:ics_url)) do
       self.with_advisory_lock(row.fetch(:pk)) do
+        now = Time.now
         if (dep = self.find_dependent("icalendar_event_v1"))
-          self._sync_row(row, dep)
+          self._sync_row(row, dep, now:)
         end
-        self.admin_dataset { |ds| ds.where(pk: row.fetch(:pk)).update(last_synced_at: Time.now) }
+        self.admin_dataset { |ds| ds.where(pk: row.fetch(:pk)).update(last_synced_at: now) }
       end
     end
   end
 
-  def _sync_row(row, dep)
+  def _sync_row(row, dep, now:)
     calendar_external_id = row.fetch(:external_id)
-    request_url = row.fetch(:ics_url)
     begin
+      request_url = self._clean_ics_url(row.fetch(:ics_url))
       io = Webhookdb::Http.chunked_download(request_url, rewindable: false)
-    rescue Down::Error => e
+    rescue Down::Error, URI::InvalidURIError => e
       self._handle_down_error(e, request_url:, calendar_external_id:)
       return
     end
 
-    upserter = Upserter.new(dep.replicator, calendar_external_id)
+    upserter = Upserter.new(dep.replicator, calendar_external_id, now:)
     processor = EventProcessor.new(io, upserter)
     processor.process
     # Delete all the extra replicator rows, and cancel all the rows that weren't upserted.
@@ -192,12 +195,23 @@ The secret to use for signing is:
         ds.where(delete_condition).delete
       end
       # Update both the status, and set the data json to match.
-      ds.exclude(compound_identity: processor.upserted_identities).update(
+      # Only update rows not already CANCELLED.
+      ds = ds.exclude(Sequel[compound_identity: processor.upserted_identities])
+      ds = ds.where(Sequel[status: nil] | ~Sequel[status: "CANCELLED"])
+      ds.update(
         status: "CANCELLED",
         data: Sequel.lit('data || \'{"STATUS":{"v":"CANCELLED"}}\'::jsonb'),
+        row_updated_at: now,
       )
     end
-    self.admin_dataset { |ds| ds.where(pk: row.fetch(:pk)).update(last_synced_at: Time.now) }
+  end
+
+  # We get all sorts of strange urls, fix up what we can.
+  def _clean_ics_url(url)
+    u = URI(url)
+    # https://xyz.com:80 is invalid, set it to 443 which yields https://xyz.com
+    u.port = 443 if u.scheme == "https" && u.port == 80
+    return u.to_s
   end
 
   def _handle_down_error(e, request_url:, calendar_external_id:)
@@ -205,22 +219,37 @@ The secret to use for signing is:
       when Down::TooManyRedirects
         response_status = 301
         response_body = "<too many redirects>"
-      when Down::TimeoutError, Down::SSLError, Down::ConnectionError, Down::InvalidUrl
+      when Down::NotModified
+        # Do not alert on 304, but do log
+        self.logger.info("icalendar_fetch_not_modified", response_status: 304, request_url:, calendar_external_id:)
+        return
+      when Down::SSLError
+        self._handle_retryable_down_error!(e, request_url:, calendar_external_id:)
+      when Down::TimeoutError, Down::ConnectionError, Down::InvalidUrl, URI::InvalidURIError
         response_status = 0
         response_body = e.to_s
       when Down::ClientError
         raise e if e.response.nil?
         response_status = e.response.code.to_i
+        self._handle_retryable_down_error!(e, request_url:, calendar_external_id:) if
+          self._retryable_client_error?(e, request_url:)
+        # These are all the errors we've seen, we can't do anything about.
+        # In theory we should do this for ALL 4xx errors,
+        # but we'd rather error on the WebhookDB side until we're sure
+        # we want to ignore things.
         expected_errors = [
+          400, 401, 402, 403, # Common access problems we can't do anything about
+          404, 405, # Fundamental issues with the URL given
+          409, 410, # More access problems
           417, # If someone uses an Outlook HTML calendar, fetch gives us a 417
+          429, # Usually 429s are retried (as above), but in some cases they're not.
         ]
         # For most client errors, we can't do anything about it. For example,
         # and 'unshared' URL could result in a 401, 403, 404, or even a 405.
         # For now, other client errors, we can raise on,
         # in case it's something we can fix/work around.
-        is_problem_error = (response_status > 405 || response_status < 400) &&
-          !expected_errors.include?(response_status)
-        raise e if is_problem_error
+        # For example, it's possible something like a 415 is a WebhookDB issue.
+        raise e unless expected_errors.include?(response_status)
         response_body = e.response.body.to_s
       when Down::ServerError
         response_status = e.response.code.to_i
@@ -243,6 +272,32 @@ The secret to use for signing is:
     self.service_integration.organization.alerting.dispatch_alert(message)
   end
 
+  def _retryable_client_error?(e, request_url:)
+    code = e.response.code.to_i
+    # This is a bad domain that returns 429 for most requests.
+    # Tell the org admins it won't sync.
+    return false if code == 429 && request_url.start_with?("https://ical.schedulestar.com")
+    # Other 429s can be retried.
+    return true if code == 429
+    # Otherwise, handle the client error normally, by telling the org admins, or raising.
+    return false
+  end
+
+  def _handle_retryable_down_error!(e, request_url:, calendar_external_id:)
+    # Retry on these, which are hopefully transient.
+    # For now, if they aren't transient, die so we see the job.
+    # We will probably need to do an alert if the retries on exhausted instead.
+    retry_in = rand(4..60).minutes
+    self.logger.debug(
+      "icalendar_fetch_error_retry",
+      response_status: e.respond_to?(:response) ? e.response&.code : 0,
+      request_url:,
+      calendar_external_id:,
+      retry_at: Time.now + retry_in,
+    )
+    raise Amigo::Retry::OrDie.new(10, retry_in)
+  end
+
   class EventProcessor
     attr_reader :upserted_identities
 
@@ -378,13 +433,21 @@ The secret to use for signing is:
 
       schedule = IceCube::Schedule.from_hash(ical_params)
       dont_project_before = Webhookdb::Icalendar.oldest_recurring_event
-      dont_project_after = Time.now + RECURRENCE_PROJECTION
+      dont_project_after = @upserter.now + RECURRENCE_PROJECTION
 
       # Just like google, track the original event id.
       h["recurring_event_id"] = uid
       final_sequence = -1
       begin
-        schedule.send(:enumerate_occurrences, schedule.start_time).each_with_index do |occ, idx|
+        # Pass in a 'closing time' to avoid a denial of service for an impossible rrule.
+        # It is further into the future than the "don't project after"
+        # since using something too short causes the calculation to be short-circuited before it should
+        # (I'm unclear what the ideal value is, but tests will fail with much less than the number here).
+        # This still results in a slow calculation, but there's not much we can do for now.
+        # In the future perhaps we should try to pre-validate common problems.
+        # See spec for examples.
+        dos_cutoff = dont_project_after + 210.days
+        schedule.send(:enumerate_occurrences, schedule.start_time, dos_cutoff).each_with_index do |occ, idx|
           next if occ.start_time < dont_project_before
           # Given the original hash, we will modify some fields.
           e = h.dup
@@ -424,6 +487,7 @@ The secret to use for signing is:
         ical = ical.gsub(/BYMONTHDAY=[\d,]+/, "")
         ical.delete_prefix! ";"
         ical.delete_suffix! ";"
+        ical.squeeze!(";")
       end
       return IceCube::IcalParser.rule_from_ical(ical)
     end
@@ -443,7 +507,23 @@ The secret to use for signing is:
       vevent_lines = []
       in_vevent = false
       while (line = @io.gets)
-        line.rstrip!
+        begin
+          line.rstrip!
+        rescue Encoding::CompatibilityError
+          # We occassionally get incorrectly encoded files.
+          # For example, the response may have a header:
+          #   Content-Type: text/calendar; charset=UTF-8
+          # but the actual encoding is not:
+          #   file -I <filename>
+          #   <filename>: text/calendar; charset=iso-8859-1
+          # In these cases, there's not much we can do.
+          # We can use chardet, but it's a big library and this issue
+          # isn't common enough. Instead, try to force the encoding to utf-8,
+          # which may break some things, but we'll see what happens.
+          line = line.force_encoding("utf-8")
+          line = line.scrub
+          line = line.rstrip
+        end
         if line == "BEGIN:VEVENT"
           in_vevent = true
           vevent_lines << line