webhookdb 1.2.1 → 1.3.0

data/lib/webhookdb/replicator/icalendar_event_v1.rb

@@ -65,7 +65,7 @@ class Webhookdb::Replicator::IcalendarEventV1 < Webhookdb::Replicator::Base
     ruby: lambda do |entry, **|
       self.entry_to_date(entry) if entry.is_a?(Hash) && self.entry_is_date_str?(entry)
     end,
-    sql: ->(_) { raise NotImplementedError },
+    sql: Webhookdb::Replicator::Column::NOT_IMPLEMENTED,
   )
   CONV_DATETIME = Webhookdb::Replicator::Column::IsomorphicProc.new(
     ruby: lambda do |entry, **|
@@ -108,6 +108,8 @@ class Webhookdb::Replicator::IcalendarEventV1 < Webhookdb::Replicator::Base
     sql: ->(_) { raise NotImplementedError },
   )
 
+  def _webhook_response(_request) = Webhookdb::WebhookResponse.ok
+
   def _remote_key_column
     return Webhookdb::Replicator::Column.new(
       :compound_identity,
@@ -126,7 +128,7 @@ class Webhookdb::Replicator::IcalendarEventV1 < Webhookdb::Replicator::Base
     return [
       col.new(:calendar_external_id, TEXT, index: true),
       col.new(:uid, TEXT, data_key: ["UID", "v"], index: true),
-      col.new(:row_updated_at, TIMESTAMP, index: true, defaulter: :now, optional: true),
+      col.new(:row_updated_at, TIMESTAMP, index: true),
       col.new(:last_modified_at,
               TIMESTAMP,
               index: true,
@@ -153,6 +155,8 @@ class Webhookdb::Replicator::IcalendarEventV1 < Webhookdb::Replicator::Base
     ]
   end
 
+  def _timestamp_column_name = :last_modified_at
+
   def _resource_and_event(request)
     return request.body, nil
   end
@@ -165,6 +169,41 @@ class Webhookdb::Replicator::IcalendarEventV1 < Webhookdb::Replicator::Base
     return data
   end
 
+  def _prepare_for_insert(resource, event, request, enrichment)
+    h = super
+    # Events can have a DTSTART, but no DTEND.
+    # https://icalendar.org/iCalendar-RFC-5545/3-6-1-event-component.html
+    # In these cases, we need to:
+    # - Use the duration, given.
+    # - Dates default to the next day.
+    # - Times default to start time.
+    if (_implicit_end_time = h[:start_at] && !h[:end_at])
+      self._set_implicit_end_at(resource, h)
+    elsif (_implicit_end_date = h[:start_date] && !h[:end_date])
+      self._set_implicit_end_date(resource, h)
+    end
+    return h
+  end
+
+  def _set_implicit_end_date(resource, h)
+    if (d = resource["DURATION"])
+      # See https://icalendar.org/iCalendar-RFC-5545/3-3-6-duration.html
+      dur = ActiveSupport::Duration.parse(d.fetch("v"))
+      h[:end_date] = h[:start_date] + dur
+      return
+    end
+    h[:end_date] = h[:start_date] + 1.day
+  end
+
+  def _set_implicit_end_at(resource, h)
+    if (d = resource["DURATION"])
+      dur = ActiveSupport::Duration.parse(d.fetch("v"))
+      h[:end_at] = h[:start_at] + dur
+      return
+    end
+    h[:end_at] = h[:start_at]
+  end
+
   # @return [Array<Webhookdb::Replicator::IndexSpec>]
   def _extra_index_specs
     return [
@@ -220,6 +259,21 @@ class Webhookdb::Replicator::IcalendarEventV1 < Webhookdb::Replicator::Base
         value.gsub!("\\r\\n", "\r\n")
         value.gsub!("\\n", "\n")
         value.gsub!("\\t", "\t")
+        # This line is not tested, since replicating issues with HTTP body encoding
+        # is really tricky (while I love Ruby's unicode handling, trying to replicate
+        # invalid data from other sources is a pain).
+        # However we do get invalid unicode sequences, like:
+        #   DESCRIPTION:\r\nNFL regional registration opens 9/25 and ends 11/20\XAOwww.nflflag.com
+        # which cannot be encoded in JSON:
+        #   Invalid Unicode [a0 77 77 77 2e] at 52 (JSON::GeneratorError)
+        # The only way I can think to handle this is with replacing invalid utf-8 chars
+        # (with the unicode questionmark icon), so they can be represented as JSON.
+        # This fix is here, and not in `Base#_to_json` (like the null char fixes),
+        # since I think this is an issue with feeds like icalendar,
+        # and not something to handle generally
+        # (we may also see this in something like Atom, but perhaps because
+        # atom is XML, not a 'plain' text format, it'll be more rare).
+        value.encode!("UTF-8", invalid: :replace, undef: :replace)
       end
       entry = {"v" => value}
       entry.merge!(params)
@@ -311,6 +365,54 @@ class Webhookdb::Replicator::IcalendarEventV1 < Webhookdb::Replicator::Base
     return
   end
 
+  # Delete CANCELLED events last updated (+row_updated_at+)' in the window between
+  # +stale_at+ to +age_cutoff+. This avoids endlessly adding to the icalendar events table
+  # due to feeds that change UIDs each fetch- events with changed UIDs will become CANCELLED,
+  # and then deleted over time.
+  # @param stale_at [Time] When an event is considered 'stale'.
+  #   If stale events are a big problem, this can be shortened to just a few days.
+  # @param age_cutoff [Time] Where to stop searching for old events.
+  #   This is important to avoid a full table scale when deleting events,
+  #   since otherwise it is like 'row_updated_at < 35.days.ago'.
+  #   Since this routine should run regularly, we should rarely have events more than 35 or 36 days old,
+  #   for example.
+  #   Use +nil+ to use no limit (a full table scan) which may be necessary when running this feature
+  #   for the first time.
+  # @param chunk_size [Integer] The row delete is done in chunks to avoid long locks.
+  #   The default seems safe, but it's exposed as a parameter if you need to play around with it,
+  #   and can be done via configuration if needed at some point.
+  def delete_stale_cancelled_events(
+    stale_at: Webhookdb::Icalendar.stale_cancelled_event_threshold_days.days.ago,
+    age_cutoff: (Webhookdb::Icalendar.stale_cancelled_event_threshold_days + 10).days.ago,
+    chunk_size: 10_000
+  )
+    # Delete in chunks, like:
+    #   DELETE from "public"."icalendar_event_v1_aaaa"
+    #   WHERE pk IN (
+    #     SELECT pk FROM "public"."icalendar_event_v1_aaaa"
+    #     WHERE row_updated_at < (now() - '35 days'::interval)
+    #     LIMIT 10000
+    #   )
+    age = age_cutoff..stale_at
+    self.admin_dataset do |ds|
+      chunk_ds = ds.where(row_updated_at: age, status: "CANCELLED").select(:pk).limit(chunk_size)
+      loop do
+        # Due to conflicts where a feed is being inserted while the delete is happening,
+        # this may raise an error like:
+        #   deadlock detected
+        #   DETAIL:  Process 18352 waits for ShareLock on transaction 435085606; blocked by process 24191.
+        #   Process 24191 waits for ShareLock on transaction 435085589; blocked by process 18352.
+        #   HINT:  See server log for query details.
+        #   CONTEXT:  while deleting tuple (2119119,3) in relation "icalendar_event_v1_aaaa"
+        # Unit testing this is very difficult though, and in practice it is rare,
+        # and normal Sidekiq job retries should be sufficient to handle this.
+        # So we don't explicitly handle deadlocks, but could if it becomes an issue.
+        deleted = ds.where(pk: chunk_ds).delete
+        break if deleted != chunk_size
+      end
+    end
+  end
+
   def calculate_webhook_state_machine
     if (step = self.calculate_dependency_state_machine_step(dependency_help: ""))
       return step

data/lib/webhookdb/replicator/increase_account_number_v1.rb

@@ -11,12 +11,12 @@ class Webhookdb::Replicator::IncreaseAccountNumberV1 < Webhookdb::Replicator::Ba
   def self.descriptor
     return Webhookdb::Replicator::Descriptor.new(
       name: "increase_account_number_v1",
-      ctor: ->(sint) { Webhookdb::Replicator::IncreaseAccountNumberV1.new(sint) },
+      ctor: self,
       feature_roles: [],
       resource_name_singular: "Increase Account Number",
-      supports_webhooks: true,
+      dependency_descriptor: Webhookdb::Replicator::IncreaseAppV1.descriptor,
       supports_backfill: true,
-      api_docs_url: "https://icalendar.org/",
+      api_docs_url: "https://increase.com/documentation/api",
     )
   end
 
@@ -26,52 +26,15 @@ class Webhookdb::Replicator::IncreaseAccountNumberV1 < Webhookdb::Replicator::Ba
 
   def _denormalized_columns
     return [
+      Webhookdb::Replicator::Column.new(:created_at, TIMESTAMP, index: true),
+      Webhookdb::Replicator::Column.new(:updated_at, TIMESTAMP, index: true),
       Webhookdb::Replicator::Column.new(:account_id, TEXT, index: true),
       Webhookdb::Replicator::Column.new(:account_number, TEXT, index: true),
       Webhookdb::Replicator::Column.new(:name, TEXT),
       Webhookdb::Replicator::Column.new(:routing_number, TEXT, index: true),
-      Webhookdb::Replicator::Column.new(
-        :row_created_at,
-        TIMESTAMP,
-        data_key: "created_at",
-        event_key: "created_at",
-        optional: true,
-        defaulter: :now,
-        index: true,
-      ),
-      Webhookdb::Replicator::Column.new(
-        :row_updated_at,
-        TIMESTAMP,
-        data_key: "created_at",
-        event_key: "created_at",
-        defaulter: :now,
-        optional: true,
-        index: true,
-      ),
       Webhookdb::Replicator::Column.new(:status, TEXT),
     ]
   end
 
-  def _timestamp_column_name
-    return :row_updated_at
-  end
-
-  def _update_where_expr
-    return self.qualified_table_sequel_identifier[:row_updated_at] < Sequel[:excluded][:row_updated_at]
-  end
-
-  def _resource_and_event(request)
-    return self._find_resource_and_event(request.body, "account_number")
-  end
-
-  def _upsert_update_expr(inserting, **_kwargs)
-    update = super
-    # Only set created_at if it's not set so the initial insert isn't modified.
-    self._coalesce_excluded_on_update(update, [:row_created_at])
-    return update
-  end
-
-  def _mixin_backfill_url
-    return "#{self.service_integration.api_url}/account_numbers"
-  end
+  def _mixin_object_type = "account_number"
 end

data/lib/webhookdb/replicator/increase_account_transfer_v1.rb

@@ -11,10 +11,10 @@ class Webhookdb::Replicator::IncreaseAccountTransferV1 < Webhookdb::Replicator::
   def self.descriptor
     return Webhookdb::Replicator::Descriptor.new(
       name: "increase_account_transfer_v1",
-      ctor: ->(sint) { Webhookdb::Replicator::IncreaseAccountTransferV1.new(sint) },
+      ctor: self,
       feature_roles: [],
       resource_name_singular: "Increase Account Transfer",
-      supports_webhooks: true,
+      dependency_descriptor: Webhookdb::Replicator::IncreaseAppV1.descriptor,
       supports_backfill: true,
       api_docs_url: "https://increase.com/documentation/api",
     )
@@ -28,34 +28,17 @@ class Webhookdb::Replicator::IncreaseAccountTransferV1 < Webhookdb::Replicator::
     return [
       Webhookdb::Replicator::Column.new(:amount, INTEGER, index: true),
       Webhookdb::Replicator::Column.new(:account_id, TEXT, index: true),
-      Webhookdb::Replicator::Column.new(:canceled_at, TIMESTAMP, data_key: ["cancellation", "canceled_at"],
-                                                                 optional: true,),
       Webhookdb::Replicator::Column.new(:created_at, TIMESTAMP, index: true),
+      Webhookdb::Replicator::Column.new(:updated_at, TIMESTAMP, index: true),
+      Webhookdb::Replicator::Column.new(
+        :canceled_at, TIMESTAMP, data_key: ["cancellation", "canceled_at"], optional: true,
+      ),
       Webhookdb::Replicator::Column.new(:destination_account_id, TEXT, index: true),
       Webhookdb::Replicator::Column.new(:destination_transaction_id, TEXT, index: true),
       Webhookdb::Replicator::Column.new(:status, TEXT),
-      Webhookdb::Replicator::Column.new(:template_id, TEXT),
       Webhookdb::Replicator::Column.new(:transaction_id, TEXT, index: true),
-      Webhookdb::Replicator::Column.new(
-        :updated_at,
-        TIMESTAMP,
-        data_key: "created_at",
-        event_key: "created_at",
-        defaulter: :now,
-        index: true,
-      ),
     ]
   end
 
-  def _update_where_expr
-    return self.qualified_table_sequel_identifier[:updated_at] < Sequel[:excluded][:updated_at]
-  end
-
-  def _resource_and_event(request)
-    return self._find_resource_and_event(request.body, "account_transfer")
-  end
-
-  def _mixin_backfill_url
-    return "#{self.service_integration.api_url}/account_transfers"
-  end
+  def _mixin_object_type = "account_transfer"
 end

data/lib/webhookdb/replicator/increase_account_v1.rb

@@ -11,10 +11,10 @@ class Webhookdb::Replicator::IncreaseAccountV1 < Webhookdb::Replicator::Base
   def self.descriptor
     return Webhookdb::Replicator::Descriptor.new(
       name: "increase_account_v1",
-      ctor: ->(sint) { Webhookdb::Replicator::IncreaseAccountV1.new(sint) },
+      ctor: self,
       feature_roles: [],
       resource_name_singular: "Increase Account",
-      supports_webhooks: true,
+      dependency_descriptor: Webhookdb::Replicator::IncreaseAppV1.descriptor,
       supports_backfill: true,
       api_docs_url: "https://increase.com/documentation/api",
     )
@@ -26,38 +26,14 @@ class Webhookdb::Replicator::IncreaseAccountV1 < Webhookdb::Replicator::Base
 
   def _denormalized_columns
     return [
-      Webhookdb::Replicator::Column.new(:balance, INTEGER, index: true),
-      Webhookdb::Replicator::Column.new(
-        :created_at,
-        TIMESTAMP,
-        data_key: "created_at",
-        defaulter: :now,
-        index: true,
-      ),
-      Webhookdb::Replicator::Column.new(:entity_id, TEXT, index: true),
-      Webhookdb::Replicator::Column.new(:interest_accrued, DECIMAL),
+      Webhookdb::Replicator::Column.new(:created_at, TIMESTAMP, index: true),
+      Webhookdb::Replicator::Column.new(:updated_at, TIMESTAMP, index: true),
       Webhookdb::Replicator::Column.new(:name, TEXT),
+      Webhookdb::Replicator::Column.new(:entity_id, TEXT, index: true),
       Webhookdb::Replicator::Column.new(:status, TEXT),
-      Webhookdb::Replicator::Column.new(
-        :updated_at,
-        TIMESTAMP,
-        data_key: "created_at",
-        event_key: "created_at",
-        defaulter: :now,
-        index: true,
-      ),
+      Webhookdb::Replicator::Column.new(:interest_accrued, DECIMAL),
     ]
   end
 
-  def _update_where_expr
-    return self.qualified_table_sequel_identifier[:updated_at] < Sequel[:excluded][:updated_at]
-  end
-
-  def _resource_and_event(request)
-    return self._find_resource_and_event(request.body, "account")
-  end
-
-  def _mixin_backfill_url
-    return "#{self.service_integration.api_url}/accounts"
-  end
+  def _mixin_object_type = "account"
 end

data/lib/webhookdb/replicator/increase_ach_transfer_v1.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "webhookdb/increase"
 require "webhookdb/replicator/increase_v1_mixin"
 
 class Webhookdb::Replicator::IncreaseACHTransferV1 < Webhookdb::Replicator::Base
@@ -11,10 +10,10 @@ class Webhookdb::Replicator::IncreaseACHTransferV1 < Webhookdb::Replicator::Base
   def self.descriptor
     return Webhookdb::Replicator::Descriptor.new(
       name: "increase_ach_transfer_v1",
-      ctor: ->(sint) { Webhookdb::Replicator::IncreaseACHTransferV1.new(sint) },
+      ctor: self,
       feature_roles: [],
       resource_name_singular: "Increase ACH Transfer",
-      supports_webhooks: true,
+      dependency_descriptor: Webhookdb::Replicator::IncreaseAppV1.descriptor,
       supports_backfill: true,
       api_docs_url: "https://increase.com/documentation/api",
     )
@@ -29,50 +28,13 @@ class Webhookdb::Replicator::IncreaseACHTransferV1 < Webhookdb::Replicator::Base
       Webhookdb::Replicator::Column.new(:account_number, TEXT, index: true),
       Webhookdb::Replicator::Column.new(:account_id, TEXT, index: true),
       Webhookdb::Replicator::Column.new(:amount, INTEGER, index: true),
-      Webhookdb::Replicator::Column.new(
-        :created_at,
-        TIMESTAMP,
-        data_key: "created_at",
-        index: true,
-      ),
+      Webhookdb::Replicator::Column.new(:created_at, TIMESTAMP, index: true),
+      Webhookdb::Replicator::Column.new(:updated_at, TIMESTAMP, index: true),
       Webhookdb::Replicator::Column.new(:routing_number, TEXT, index: true),
       Webhookdb::Replicator::Column.new(:status, TEXT),
       Webhookdb::Replicator::Column.new(:transaction_id, TEXT, index: true),
-      Webhookdb::Replicator::Column.new(
-        :updated_at,
-        TIMESTAMP,
-        data_key: "created_at",
-        event_key: "created_at",
-        defaulter: :now,
-        index: true,
-      ),
     ]
   end
 
-  def _prepare_for_insert(resource, event, request, enrichment)
-    # created_at is marked required, but to skip on nil.
-    # This will preserve its existing value when we update the webhook.
-    resource["created_at"] = nil if event&.fetch("event") == "updated"
-    return super
-  end
-
-  def _upsert_update_expr(inserting, enrichment: nil)
-    update = super
-    update[:data] = Sequel.lit("#{self.service_integration.table_name}.data || excluded.data")
-    return update
-  end
-
-  def _update_where_expr
-    return self.qualified_table_sequel_identifier[:updated_at] < Sequel[:excluded][:updated_at]
-  end
-
-  def _resource_and_event(request)
-    resource, event = self._find_resource_and_event(request.body, "ach_transfer")
-    return nil, nil if (resource && resource["type"]) == "inbound_ach_transfer"
-    return resource, event
-  end
-
-  def _mixin_backfill_url
-    return "#{self.service_integration.api_url}/transfers/achs"
-  end
+  def _mixin_object_type = "ach_transfer"
 end

data/lib/webhookdb/replicator/increase_app_v1.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+class Webhookdb::Replicator::IncreaseAppV1 < Webhookdb::Replicator::Base
+  include Appydays::Loggable
+
+  # @return [Webhookdb::Replicator::Descriptor]
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "increase_app_v1",
+      ctor: self,
+      feature_roles: [],
+      resource_name_singular: "Increase App",
+      resource_name_plural: "Increase App",
+      supports_webhooks: true,
+      supports_backfill: true,
+      install_url: "#{Webhookdb.api_url}/increase",
+      documentation_url: "https://docs.webhookdb.com/guides/increase/",
+      description: "Replicate your Increase data to WebhookDB Cloud in one click using " \
+                   "our [WebhookDB-Increase integration](https://docs.webhookdb.com/guides/increase).",
+    )
+  end
+
+  def _remote_key_column
+    return Webhookdb::Replicator::Column.new(:ignore_id, INTEGER)
+  end
+
+  def _denormalized_columns
+    return []
+  end
+
+  def _upsert_webhook(request, **kw)
+    raise Webhookdb::InvalidPrecondition, "can only handle event payloads" unless request.body.fetch("type") == "event"
+    dispatchable = self.service_integration.dependents.select do |d|
+      d.service_name == "increase_event_v1" || d.replicator.handle_event?(request.body)
+    end
+    dispatchable.each do |sint|
+      sint.replicator.upsert_webhook(request, **kw)
+    end
+    return nil
+  end
+
+  def _fetch_backfill_page(*)
+    return [], nil
+  end
+
+  def webhook_response(request)
+    return Webhookdb::Increase.webhook_response(request, Webhookdb::Increase.webhook_secret)
+  end
+
+  def calculate_webhook_state_machine
+    return self.calculate_backfill_state_machine
+  end
+
+  def calculate_backfill_state_machine
+    step = Webhookdb::Replicator::StateMachineStep.new
+    step.output = %(This replicator is managed automatically using OAuth through Increase.
+Head over to #{self.descriptor.install_url} to learn more.)
+    step.completed
+    return step
+  end
+
+  def build_dependents
+    org = self.service_integration.organization
+    parent_descr = self.descriptor
+    sints = self.service_integration.organization.available_replicators.
+      select { |dd| dd.dependency_descriptor == parent_descr }.
+      map do |dd|
+      Webhookdb::ServiceIntegration.create_disambiguated(
+        dd.name,
+        organization: org,
+        depends_on: self.service_integration,
+      )
+    end
+    sints.
+      select { |sint| sint.replicator.descriptor.supports_backfill? }.
+      each { |sint| sint.replicator._enqueue_backfill_jobs(incremental: false) }
+  end
+end

data/lib/webhookdb/replicator/increase_check_transfer_v1.rb

@@ -11,10 +11,10 @@ class Webhookdb::Replicator::IncreaseCheckTransferV1 < Webhookdb::Replicator::Ba
   def self.descriptor
     return Webhookdb::Replicator::Descriptor.new(
       name: "increase_check_transfer_v1",
-      ctor: ->(sint) { Webhookdb::Replicator::IncreaseCheckTransferV1.new(sint) },
+      ctor: self,
       feature_roles: [],
       resource_name_singular: "Increase Check Transfer",
-      supports_webhooks: true,
+      dependency_descriptor: Webhookdb::Replicator::IncreaseAppV1.descriptor,
       supports_backfill: true,
       api_docs_url: "https://increase.com/documentation/api",
     )
@@ -26,39 +26,33 @@ class Webhookdb::Replicator::IncreaseCheckTransferV1 < Webhookdb::Replicator::Ba
 
   def _denormalized_columns
     return [
+      Webhookdb::Replicator::Column.new(:created_at, TIMESTAMP, index: true),
+      Webhookdb::Replicator::Column.new(:updated_at, TIMESTAMP, index: true),
       Webhookdb::Replicator::Column.new(:account_id, TEXT, index: true),
-      Webhookdb::Replicator::Column.new(:address_line1, TEXT),
-      Webhookdb::Replicator::Column.new(:address_city, TEXT),
-      Webhookdb::Replicator::Column.new(:address_state, TEXT),
-      Webhookdb::Replicator::Column.new(:address_zip, TEXT, index: true),
       Webhookdb::Replicator::Column.new(:amount, INTEGER, index: true),
-      Webhookdb::Replicator::Column.new(:created_at, TIMESTAMP, data_key: "created_at", index: true),
-      Webhookdb::Replicator::Column.new(:mailed_at, TIMESTAMP),
-      Webhookdb::Replicator::Column.new(:recipient_name, TEXT),
+      Webhookdb::Replicator::Column.new(:account_number, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(:routing_number, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(:check_number, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(
+        :recipient_name,
+        TEXT,
+        data_key: ["physical_check", "recipient_name"], optional: true,
+      ),
       Webhookdb::Replicator::Column.new(:status, TEXT),
-      Webhookdb::Replicator::Column.new(:submitted_at, TIMESTAMP, index: true),
-      Webhookdb::Replicator::Column.new(:template_id, TEXT),
-      Webhookdb::Replicator::Column.new(:transaction_id, TEXT, index: true),
       Webhookdb::Replicator::Column.new(
-        :updated_at,
-        TIMESTAMP,
-        data_key: "created_at",
-        event_key: "created_at",
-        defaulter: :now,
-        index: true,
+        :canceled_at, TIMESTAMP, data_key: ["cancellation", "canceled_at"], optional: true, index: true,
+      ),
+      Webhookdb::Replicator::Column.new(
+        :deposited_at, TIMESTAMP, data_key: ["deposit", "deposited_at"], optional: true, index: true,
+      ),
+      Webhookdb::Replicator::Column.new(
+        :mailed_at, TIMESTAMP, data_key: ["mailing", "mailed_at"], optional: true, index: true,
+      ),
+      Webhookdb::Replicator::Column.new(
+        :submitted_at, TIMESTAMP, data_key: ["submission", "submitted_at"], optional: true, index: true,
       ),
     ]
   end
 
-  def _update_where_expr
-    return self.qualified_table_sequel_identifier[:updated_at] < Sequel[:excluded][:updated_at]
-  end
-
-  def _resource_and_event(request)
-    return self._find_resource_and_event(request.body, "check_transfer")
-  end
-
-  def _mixin_backfill_url
-    return "#{self.service_integration.api_url}/check_transfers"
-  end
+  def _mixin_object_type = "check_transfer"
 end

data/lib/webhookdb/replicator/increase_event_v1.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class Webhookdb::Replicator::IncreaseEventV1 < Webhookdb::Replicator::Base
+  include Appydays::Loggable
+  include Webhookdb::Replicator::IncreaseV1Mixin
+
+  # @return [Webhookdb::Replicator::Descriptor]
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "increase_event_v1",
+      ctor: self,
+      feature_roles: [],
+      resource_name_singular: "Increase Event",
+      dependency_descriptor: Webhookdb::Replicator::IncreaseAppV1.descriptor,
+      # Since events are only done through the increase_app_v1,
+      # we don't support normal WebhookDB webhooks. Instead,
+      # they come in via the app. If we wanted to handle webhooks to the normal
+      # /v1/service_integrations/:opaque_id endpoint, rather than /v1/install/increase/webhook,
+      # we'd make this 'true' and have to do work like webhook validation.
+      # supports_webhooks: false,
+      supports_backfill: true,
+      api_docs_url: "https://increase.com/documentation/api",
+    )
+  end
+
+  def _remote_key_column
+    return Webhookdb::Replicator::Column.new(:increase_id, TEXT, data_key: "id")
+  end
+
+  def _denormalized_columns
+    return [
+      Webhookdb::Replicator::Column.new(:associated_object_id, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(:associated_object_type, TEXT),
+      Webhookdb::Replicator::Column.new(:category, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(:created_at, TIMESTAMP, index: true),
+    ]
+  end
+
+  def _timestamp_column_name = :created_at
+  def _mixin_object_type = "event"
+end

data/lib/webhookdb/replicator/increase_limit_v1.rb

@@ -11,10 +11,13 @@ class Webhookdb::Replicator::IncreaseLimitV1 < Webhookdb::Replicator::Base
   def self.descriptor
     return Webhookdb::Replicator::Descriptor.new(
       name: "increase_limit_v1",
-      ctor: ->(sint) { Webhookdb::Replicator::IncreaseLimitV1.new(sint) },
-      feature_roles: [],
+      ctor: self,
+      # This is a legacy resource. Instead, users should set the 'allow/deny ACH debits' flag on account numbers,
+      # or use the Inbound ACH Transfer object, which can send a webhook to accept or reject it.
+      # This flag is here for WebhookDB users who still need access to Limit resources.
+      feature_roles: ["increase_limits"],
       resource_name_singular: "Increase Limit",
-      supports_webhooks: true,
+      dependency_descriptor: Webhookdb::Replicator::IncreaseAppV1.descriptor,
       supports_backfill: true,
       api_docs_url: "https://increase.com/documentation/api",
     )
@@ -34,45 +37,17 @@ class Webhookdb::Replicator::IncreaseLimitV1 < Webhookdb::Replicator::Base
         :row_created_at,
         TIMESTAMP,
         data_key: "created_at",
-        event_key: "created_at",
-        defaulter: :now,
-        optional: true,
-        index: true,
-      ),
-      Webhookdb::Replicator::Column.new(
-        :row_updated_at,
-        TIMESTAMP,
-        data_key: "created_at",
-        event_key: "created_at",
         defaulter: :now,
         optional: true,
         index: true,
       ),
+      Webhookdb::Replicator::Column.new(:row_updated_at, TIMESTAMP, data_key: "updated_at", index: true),
       Webhookdb::Replicator::Column.new(:status, TEXT),
       Webhookdb::Replicator::Column.new(:value, INTEGER),
     ]
   end
 
-  def _timestamp_column_name
-    return :row_updated_at
-  end
+  def _timestamp_column_name = :row_updated_at
 
-  def _update_where_expr
-    return self.qualified_table_sequel_identifier[:row_updated_at] < Sequel[:excluded][:row_updated_at]
-  end
-
-  def _resource_and_event(request)
-    return self._find_resource_and_event(request.body, "limit")
-  end
-
-  def _upsert_update_expr(inserting, **_kwargs)
-    update = super
-    # Only set created_at if it's not set so the initial insert isn't modified.
-    self._coalesce_excluded_on_update(update, [:row_created_at])
-    return update
-  end
-
-  def _mixin_backfill_url
-    return "#{self.service_integration.api_url}/limits"
-  end
+  def _mixin_object_type = "limit"
 end