webhookdb 0.1.0

data/lib/webhookdb/replicator/stripe_refund_v1.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require "stripe"
+require "webhookdb/replicator/stripe_v1_mixin"
+
+class Webhookdb::Replicator::StripeRefundV1 < Webhookdb::Replicator::Base
+  include Appydays::Loggable
+  include Webhookdb::Replicator::StripeV1Mixin
+
+  # @return [Webhookdb::Replicator::Descriptor]
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "stripe_refund_v1",
+      ctor: ->(sint) { Webhookdb::Replicator::StripeRefundV1.new(sint) },
+      feature_roles: [],
+      resource_name_singular: "Stripe Refund",
+      supports_webhooks: true,
+      supports_backfill: true,
+      api_docs_url: "https://stripe.com/docs/api/refunds",
+    )
+  end
+
+  def _remote_key_column
+    return Webhookdb::Replicator::Column.new(:stripe_id, TEXT, data_key: "id")
+  end
+
+  def _denormalized_columns
+    return [
+      Webhookdb::Replicator::Column.new(:amount, INTEGER, index: true),
+      Webhookdb::Replicator::Column.new(:balance_transaction, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(:charge, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(:created, TIMESTAMP, index: true, converter: :tsat),
+      Webhookdb::Replicator::Column.new(:payment_intent, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(:receipt_number, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(:source_transfer_reversal, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(:status, TEXT),
+      Webhookdb::Replicator::Column.new(:transfer_reversal, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(
+        :updated,
+        TIMESTAMP,
+        index: true,
+        data_key: "created",
+        event_key: "created",
+        converter: :tsat,
+      ),
+    ]
+  end
+
+  def _update_where_expr
+    return self.qualified_table_sequel_identifier[:updated] < Sequel[:excluded][:updated]
+  end
+
+  def _mixin_backfill_url
+    return "https://api.stripe.com/v1/refunds"
+  end
+
+  def _mixin_event_type_names
+    return ["charge.refund.updated"]
+  end
+
+  def restricted_key_resource_name = "Charges"
+
+  def _upsert_webhook(request, upsert: true)
+    resource, _event = self._resource_and_event(request)
+    object_type = resource.fetch("object")
+    return super if object_type == "refund"
+
+    # We can hit this when processing a 'charges.updated' webhook for refunds.
+    # We have to pull a list of refunds from the "charges" webhook.
+    # There is a somewhat infuriating nested pagination
+    # mechanism here, where the refunds list in the charge takes this form:
+    #
+    #    "refunds": {
+    #       "object": "list",
+    #       "data": [],
+    #       "has_more": false,
+    #       "url": "/v1/charges/ch_1JG8U9FFYxHXGyKxPaNIdc0b/refunds"
+    #    }
+    #
+    # and the `has_more` and `url` fields contain the information that is rquired to kick off
+    # a paginated backfill. `has_more` is almost always going to be false, because it should be
+    # rare that a charge has more than ten refunds, so we're just going to ignore this concern
+    # for now and issue a DeveloperAlert if pagination is required.
+    refunds_obj = request.body.dig("data", "object", "refunds")
+
+    if refunds_obj.fetch("has_more") == true
+      Webhookdb::DeveloperAlert.new(
+        subsystem: "Stripe Refunds Webhook Error",
+        emoji: ":hook:",
+        fallback: "Full backfill required for integration #{self.service_integration.opaque_id}",
+        fields: [],
+      ).emit
+    end
+
+    refunds_obj.fetch("data").each do |b|
+      new_request = request.dup
+      new_request.body = b
+      super(new_request, upsert:)
+    end
+  end
+end

data/lib/webhookdb/replicator/stripe_subscription_item_v1.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "stripe"
+require "webhookdb/replicator/stripe_v1_mixin"
+
+class Webhookdb::Replicator::StripeSubscriptionItemV1 < Webhookdb::Replicator::Base
+  include Appydays::Loggable
+  include Webhookdb::Replicator::StripeV1Mixin
+
+  # @return [Webhookdb::Replicator::Descriptor]
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "stripe_subscription_item_v1",
+      ctor: ->(sint) { Webhookdb::Replicator::StripeSubscriptionItemV1.new(sint) },
+      feature_roles: [],
+      resource_name_singular: "Stripe Subscription Item",
+      supports_webhooks: true,
+      supports_backfill: true,
+      api_docs_url: "https://stripe.com/docs/api/subscription_items",
+    )
+  end
+
+  def _remote_key_column
+    return Webhookdb::Replicator::Column.new(:stripe_id, TEXT, data_key: "id")
+  end
+
+  def _denormalized_columns
+    return [
+      Webhookdb::Replicator::Column.new(:created, TIMESTAMP, index: true, converter: :tsat),
+      Webhookdb::Replicator::Column.new(:price, TEXT, index: true, data_key: ["price", "id"], optional: true),
+      Webhookdb::Replicator::Column.new(:product, TEXT, index: true, data_key: ["price", "product"], optional: true),
+      Webhookdb::Replicator::Column.new(:quantity, INTEGER),
+      Webhookdb::Replicator::Column.new(:subscription, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(
+        :updated,
+        TIMESTAMP,
+        index: true,
+        data_key: "created",
+        event_key: "created",
+        converter: :tsat,
+      ),
+    ]
+  end
+
+  def _update_where_expr
+    return self.qualified_table_sequel_identifier[:updated] < Sequel[:excluded][:updated]
+  end
+
+  def _mixin_backfill_url
+    return "https://api.stripe.com/v1/subscription_items"
+  end
+
+  def _mixin_event_type_names
+    return []
+  end
+end

data/lib/webhookdb/replicator/stripe_subscription_v1.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "stripe"
+require "webhookdb/replicator/stripe_v1_mixin"
+
+class Webhookdb::Replicator::StripeSubscriptionV1 < Webhookdb::Replicator::Base
+  include Appydays::Loggable
+  include Webhookdb::Replicator::StripeV1Mixin
+
+  # @return [Webhookdb::Replicator::Descriptor]
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "stripe_subscription_v1",
+      ctor: ->(sint) { Webhookdb::Replicator::StripeSubscriptionV1.new(sint) },
+      feature_roles: [],
+      resource_name_singular: "Stripe Subscription",
+      supports_webhooks: true,
+      supports_backfill: true,
+      api_docs_url: "https://stripe.com/docs/api/subscriptions",
+    )
+  end
+
+  def _remote_key_column
+    return Webhookdb::Replicator::Column.new(:stripe_id, TEXT, data_key: "id")
+  end
+
+  def _denormalized_columns
+    return [
+      Webhookdb::Replicator::Column.new(:billing_cycle_anchor, TIMESTAMP, index: true, converter: :tsat),
+      Webhookdb::Replicator::Column.new(:cancel_at, TIMESTAMP, index: true, optional: true, converter: :tsat),
+      Webhookdb::Replicator::Column.new(:canceled_at, TIMESTAMP, index: true, optional: true, converter: :tsat),
+      Webhookdb::Replicator::Column.new(:created, TIMESTAMP, index: true, converter: :tsat),
+      Webhookdb::Replicator::Column.new(:current_period_end, TIMESTAMP, index: true, converter: :tsat),
+      Webhookdb::Replicator::Column.new(:current_period_start, TIMESTAMP, index: true, converter: :tsat),
+      Webhookdb::Replicator::Column.new(:customer, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(:default_payment_method, TEXT),
+      Webhookdb::Replicator::Column.new(:default_source, TEXT),
+      Webhookdb::Replicator::Column.new(:discount, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(:ended_at, TIMESTAMP, index: true, optional: true),
+      Webhookdb::Replicator::Column.new(:latest_invoice, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(:schedule, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(:start_date, TIMESTAMP, index: true, converter: :tsat),
+      Webhookdb::Replicator::Column.new(:status, TEXT),
+      Webhookdb::Replicator::Column.new(:trial_end, TIMESTAMP, optional: true, converter: :tsat),
+      Webhookdb::Replicator::Column.new(:trial_start, TIMESTAMP, optional: true, converter: :tsat),
+      Webhookdb::Replicator::Column.new(
+        :updated,
+        TIMESTAMP,
+        index: true,
+        data_key: "created",
+        event_key: "created",
+        converter: :tsat,
+      ),
+    ]
+  end
+
+  def _update_where_expr
+    return self.qualified_table_sequel_identifier[:updated] < Sequel[:excluded][:updated]
+  end
+
+  def _mixin_backfill_url
+    return "https://api.stripe.com/v1/subscriptions"
+  end
+
+  def _mixin_event_type_names
+    return [
+      "customer.subscription.created",
+      "customer.subscription.deleted",
+      "customer.subscription.pending_update_applied",
+      "customer.subscription.pending_update_expired",
+      "customer.subscription.trial_will_end",
+      "customer.subscription.updated",
+    ]
+  end
+end

data/lib/webhookdb/replicator/stripe_v1_mixin.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require "webhookdb/stripe"
+
+module Webhookdb::Replicator::StripeV1Mixin
+  def _resource_and_event(request)
+    body = request.body
+    return body.fetch("data").fetch("object"), body if body.fetch("object") == "event"
+    return body, nil
+  end
+
+  def _mixin_backfill_url
+    raise NotImplementedError
+  end
+
+  # this array describes which event this webhook should subscribe to
+  # https://stripe.com/docs/api/events/types
+  def _mixin_event_type_names
+    raise NotImplementedError
+  end
+
+  def _webhook_response(request)
+    return Webhookdb::Stripe.webhook_response(request, self.service_integration.webhook_secret)
+  end
+
+  def _timestamp_column_name
+    return :updated
+  end
+
+  def calculate_webhook_state_machine
+    step = Webhookdb::Replicator::StateMachineStep.new
+    unless self.service_integration.webhook_secret.present?
+      step.output = %{You are about to start replicating #{self.resource_name_singular} info into WebhookDB.
+We've made an endpoint available for #{self.resource_name_singular} webhooks:
+
+#{self._webhook_endpoint}
+
+From your Stripe Dashboard, go to Developers -> Webhooks -> Add Endpoint.
+Use the URL above, and choose all of the following events:
+  #{self._mixin_event_type_names.join("\n  ")}
+Then click Add Endpoint.
+
+The page for the webhook will have a 'Signing Secret' section.
+Reveal it, then copy the secret (it will start with `whsec_`).
+      }
+      return step.secret_prompt("secret").webhook_secret(self.service_integration)
+    end
+
+    step.output = %(Great! WebhookDB is now listening for #{self.resource_name_singular} webhooks.
+#{self._query_help_output}
+In order to backfill existing #{self.resource_name_plural}, run this from a shell:
+
+  #{self._backfill_command}
+    )
+    return step.completed
+  end
+
+  def calculate_backfill_state_machine
+    step = Webhookdb::Replicator::StateMachineStep.new
+    unless self.service_integration.backfill_key.present?
+      step.output = %(In order to backfill #{self.resource_name_plural}, we need an API key.
+From your Stripe Dashboard, go to Developers -> API Keys -> Restricted Keys -> Create Restricted Key.
+Create a key with Read access to #{self.restricted_key_resource_name}.
+Submit, then copy the key when Stripe shows it to you:
+      )
+      return step.secret_prompt("Restricted Key").backfill_key(self.service_integration)
+    end
+
+    unless (result = self.verify_backfill_credentials).verified
+      self.service_integration.replicator.clear_backfill_information
+      step.output = result.message
+      return step.secret_prompt("Restricted Key").backfill_key(self.service_integration)
+    end
+
+    step.output = %(Great! We are going to start backfilling your #{self.resource_name_plural}.
+#{self._query_help_output}
+      )
+    return step.completed
+  end
+
+  def restricted_key_resource_name = self.resource_name_plural.gsub(/^Stripe /, "")
+
+  def _verify_backfill_403_err_msg
+    return "It looks like that API Key does not have permission to access #{self.resource_name_singular} Records. " \
+           "Please check the permissions by going to the list of restricted keys and " \
+           "hovering over the information icon in the entry for this key. " \
+           "Once you've verified or corrected the permissions for this key, " \
+           "please reenter the API Key you just created:"
+  end
+
+  def _verify_backfill_401_err_msg
+    return "It looks like that API Key is invalid. Please reenter the API Key you just created:"
+  end
+
+  def _verify_backfill_err_msg
+    return "An error occurred. Please reenter the API Key you just created:"
+  end
+
+  def _fetch_backfill_page(pagination_token, **_kwargs)
+    url = self._mixin_backfill_url
+    url += pagination_token if pagination_token.present?
+    response = Webhookdb::Http.get(
+      url,
+      basic_auth: {username: self.service_integration.backfill_key},
+      logger: self.logger,
+      timeout: Webhookdb::Stripe.http_timeout,
+    )
+    data = response.parsed_response
+    next_page_param = nil
+    if data["has_more"]
+      last_item_id = data["data"][-1]["id"]
+      next_page_param = "?starting_after=" + last_item_id
+    end
+    return data["data"], next_page_param
+  end
+end

data/lib/webhookdb/replicator/transistor_episode_stats_v1.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require "webhookdb/replicator/transistor_v1_mixin"
+
+class Webhookdb::Replicator::TransistorEpisodeStatsV1 < Webhookdb::Replicator::Base
+  include Appydays::Loggable
+  include Webhookdb::Replicator::TransistorV1Mixin
+
+  # @return [Webhookdb::Replicator::Descriptor]
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "transistor_episode_stats_v1",
+      ctor: ->(sint) { Webhookdb::Replicator::TransistorEpisodeStatsV1.new(sint) },
+      feature_roles: [],
+      resource_name_singular: "Transistor Episode Stats",
+      resource_name_plural: "Transistor Episode Stats",
+      dependency_descriptor: Webhookdb::Replicator::TransistorEpisodeV1.descriptor,
+      supports_backfill: true,
+      api_docs_url: "https://developers.transistor.fm/#EpisodeAnalytics",
+    )
+  end
+
+  CONV_PARSE_DMY_DASH = Webhookdb::Replicator::Column::IsomorphicProc.new(
+    ruby: lambda do |s, **_|
+      return Date.strptime(s, "%d-%m-%Y")
+    rescue TypeError, Date::Error
+      return nil
+    end,
+    sql: ->(e) { Sequel.function(:to_date, e, "DD-MM-YYYY") },
+  )
+
+  CONV_REMOTE_KEY = Webhookdb::Replicator::Column::IsomorphicProc.new(
+    ruby: ->(_, resource:, **_) { "#{resource.fetch('episode_id')}-#{resource.fetch('date')}" },
+    # Because this is a non-nullable key, we never need this in SQL
+    sql: ->(_) { Sequel.lit("'do not use'") },
+  )
+
+  def _remote_key_column
+    return Webhookdb::Replicator::Column.new(
+      :compound_identity,
+      TEXT,
+      data_key: "<compound key, see converter>",
+      index: true,
+      optional: true,
+      converter: CONV_REMOTE_KEY,
+    )
+  end
+
+  def _denormalized_columns
+    return [
+      Webhookdb::Replicator::Column.new(:episode_id, TEXT),
+      Webhookdb::Replicator::Column.new(:date, DATE, converter: CONV_PARSE_DMY_DASH),
+      Webhookdb::Replicator::Column.new(:downloads, INTEGER),
+      Webhookdb::Replicator::Column.new(:row_updated_at, TIMESTAMP, defaulter: :now, optional: true),
+    ]
+  end
+
+  def _timestamp_column_name
+    return :row_updated_at
+  end
+
+  def _resource_and_event(request)
+    return request.body, nil
+  end
+
+  def _update_where_expr
+    return self.qualified_table_sequel_identifier[:downloads] !~ Sequel[:excluded][:downloads]
+  end
+
+  def calculate_backfill_state_machine
+    if (step = self.calculate_dependency_state_machine_step(dependency_help: ""))
+      return step
+    end
+    step = Webhookdb::Replicator::StateMachineStep.new
+    step.output = %(Great! That's all the information we need.
+When your Transistor Episodes get added or updated, their stats will be updated in WebhookDB too.
+
+#{self._query_help_output})
+    return step.completed
+  end
+
+  def _backfillers
+    episode_svc = self.service_integration.depends_on.replicator
+    backfillers = episode_svc.admin_dataset(timeout: :fast) do |episode_ds|
+      episode_ds.select(:transistor_id, :created_at).map do |episode|
+        EpisodeStatsBackfiller.new(
+          episode_svc:,
+          episode_stats_svc: self,
+          episode_id: episode[:transistor_id],
+          episode_created_at: episode[:created_at],
+        )
+      end
+    end
+    return backfillers
+  end
+
+  class EpisodeStatsBackfiller < Webhookdb::Backfiller
+    def initialize(episode_svc:, episode_stats_svc:, episode_id:, episode_created_at:)
+      @episode_svc = episode_svc
+      @episode_stats_svc = episode_stats_svc
+      @episode_id = episode_id
+      @episode_created_at = episode_created_at
+      super()
+    end
+
+    def handle_item(item)
+      item["episode_id"] = @episode_id
+      @episode_stats_svc.upsert_webhook_body(item)
+    end
+
+    def fetch_backfill_page(_pagination_token, **_kwargs)
+      analytics_url = "https://api.transistor.fm/v1/analytics/episodes/" + @episode_id
+      # The "downloads" stat gets collected daily but will not change retroactively for a past date.
+      # If there are already rows in the enrichment table matching the episode_id, we want to check
+      # the date of the last entry so that we don't have to upsert information that we know will not
+      # be changed. We allow for a two day buffer before the date of the last entry to account for changes
+      # that may occur on the day of a new entry, while the downloads are accruing.
+      latest_update = @episode_stats_svc.admin_dataset(timeout: :fast) do |ds|
+        ds.where(episode_id: @episode_id).max(:date)
+      end
+      start_date = latest_update.nil? ? @episode_created_at : (latest_update - 2.days)
+      request_body = {
+        start_date: start_date.strftime("%d-%m-%Y"),
+        end_date: Time.now.strftime("%d-%m-%Y"),
+      }
+      response = Webhookdb::Http.get(
+        analytics_url,
+        headers: {"x-api-key" => @episode_svc.service_integration.backfill_key},
+        body: request_body,
+        logger: @episode_stats_svc.logger,
+        timeout: Webhookdb::Transistor.http_timeout,
+      )
+      data = response.parsed_response.dig("data", "attributes", "downloads") || []
+      return data, nil
+    end
+  end
+
+  def on_dependency_webhook_upsert(_replicator, _payload, *)
+    return
+  end
+end

data/lib/webhookdb/replicator/transistor_episode_v1.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require "webhookdb/transistor"
+require "webhookdb/replicator/transistor_v1_mixin"
+require "nokogiri"
+
+class Webhookdb::Replicator::TransistorEpisodeV1 < Webhookdb::Replicator::Base
+  include Appydays::Loggable
+  include Webhookdb::Replicator::TransistorV1Mixin
+
+  # @return [Webhookdb::Replicator::Descriptor]
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "transistor_episode_v1",
+      ctor: ->(sint) { Webhookdb::Replicator::TransistorEpisodeV1.new(sint) },
+      feature_roles: [],
+      resource_name_singular: "Transistor Episode",
+      supports_backfill: true,
+      api_docs_url: "https://developers.transistor.fm/#Episode",
+    )
+  end
+
+  def _denormalized_columns
+    return [
+      Webhookdb::Replicator::Column.new(:author, TEXT, data_key: ["attributes", "author"]),
+      Webhookdb::Replicator::Column.new(
+        :created_at,
+        TIMESTAMP,
+        index: true,
+        data_key: ["attributes", "created_at"],
+      ),
+      Webhookdb::Replicator::Column.new(:duration, INTEGER, data_key: ["attributes", "duration"]),
+      Webhookdb::Replicator::Column.new(:keywords, TEXT, data_key: ["attributes", "keywords"]),
+      Webhookdb::Replicator::Column.new(:number, INTEGER, index: true, data_key: ["attributes", "number"]),
+      Webhookdb::Replicator::Column.new(
+        :published_at,
+        TIMESTAMP,
+        index: true,
+        data_key: ["attributes", "published_at"],
+      ),
+      Webhookdb::Replicator::Column.new(:season, INTEGER, index: true, data_key: ["attributes", "season"]),
+      Webhookdb::Replicator::Column.new(
+        :show_id,
+        TEXT,
+        index: true,
+        data_key: ["relationships", "show", "data", "id"],
+      ),
+      Webhookdb::Replicator::Column.new(:status, TEXT, data_key: ["attributes", "status"]),
+      Webhookdb::Replicator::Column.new(:title, TEXT, data_key: ["attributes", "title"]),
+      Webhookdb::Replicator::Column.new(:type, TEXT, data_key: ["attributes", "type"]),
+      Webhookdb::Replicator::Column.new(
+        :updated_at,
+        TIMESTAMP,
+        index: true,
+        data_key: ["attributes", "updated_at"],
+      ),
+      # Ideally these would have converters, but they'd be very confusing, and when this was built
+      # we only had one transistor user, so we truncated the table instead.
+      Webhookdb::Replicator::Column.new(:api_format, INTEGER, optional: true),
+      Webhookdb::Replicator::Column.new(:logical_summary, TEXT, optional: true),
+      Webhookdb::Replicator::Column.new(:logical_description, TEXT, optional: true),
+    ]
+  end
+
+  def _prepare_for_insert(resource, event, request, enrichment)
+    h = super
+    # Transistor merged their summary and description fields so they're authored
+    # as one big 'description' HTML blob in February 2023. Previous to that,
+    # there were separate summary and description fields
+    # (we call this api_format 1).
+    #
+    # If we have a nil summary, we know this is a 'new' format (api_format 2).
+    # In that case, look for the first line of the HTML,
+    # and treat that as the summary. Anything else in the HTML is treated as
+    # the remaining description. Some care is paid to whitespace, too,
+    # since <br> tags can be used within an element.
+    summary = resource.fetch("attributes").fetch("summary", nil)
+    description = resource.fetch("attributes").fetch("description", nil)
+    if summary.nil?
+      h[:api_format] = 2
+      parsed_desc = Nokogiri::HTML5.fragment(description)
+
+      extracted_summary = self._extract_first_html_line_as_text(parsed_desc)
+      h[:logical_description] = nil
+      if extracted_summary
+        h[:logical_summary] = extracted_summary
+        h[:logical_description] = parsed_desc.to_s.strip if parsed_desc.inner_text.present?
+      else
+        h[:logical_summary] = parsed_desc.to_s.strip
+      end
+    else
+      h[:logical_summary] = summary
+      h[:logical_description] = description
+      h[:api_format] = 1
+    end
+    return h
+  end
+
+  # Usually the Transistor HTML looks like <div>foo<br><br>hello</div>.
+  # Extract 'foo' as text, remove leading <br>, and return <div>hello</div>.
+  def _extract_first_html_line_as_text(element)
+    # Grab the first div or p element, where the text is.
+    first_div = element.css("div, p").first
+    return nil unless first_div
+    # Iterate over each child element:
+    # - If it's a text element, it's part of the first line.
+    # - If it's a br/div/p element, we have reached the end of the first line.
+    # - Otherwise, it's probably some type of style element, and can be appended.
+    first_line_html = +""
+    first_div.children.to_a.each do |child|
+      if child.is_a?(Nokogiri::XML::Text)
+        first_line_html << child.inner_text
+        child.remove
+      elsif child.name == "br"
+        # Remove additional br tags, this is like
+        # removing leading whitespace of the new/remaining description.
+        while (sibling = child.next)
+          break unless sibling.name == "br"
+          sibling.remove
+        end
+        child.remove
+        break
+      elsif BLOCK_ELEMENT_TAGS.include?(child.name)
+        break
+      else
+        first_line_html << child.to_html
+        child.remove
+      end
+    end
+    first_div.remove if first_div.inner_text.blank?
+    return first_line_html.strip
+  end
+
+  BLOCK_ELEMENT_TAGS = ["p", "div"].freeze
+
+  def upsert_has_deps?
+    return true
+  end
+
+  def parse_date_from_api(date_string)
+    return Time.strptime(date_string, "%d-%m-%Y")
+  end
+
+  def _fetch_backfill_page(pagination_token, last_backfilled:)
+    url = "https://api.transistor.fm/v1/episodes"
+    pagination_token = 1 if pagination_token.blank?
+    response = Webhookdb::Http.get(
+      url,
+      headers: {"x-api-key" => self.service_integration.backfill_key},
+      body: {pagination: {page: pagination_token, per: 500}},
+      logger: self.logger,
+      timeout: Webhookdb::Transistor.http_timeout,
+    )
+    data = response.parsed_response
+    episodes = data["data"]
+    current_page = data["meta"]["currentPage"]
+    total_pages = data["meta"]["totalPages"]
+    next_page = (current_page.to_i + 1 if current_page < total_pages)
+
+    if last_backfilled.present?
+      earliest_data_created = episodes.empty? ? Time.at(0) : episodes[-1].dig("attributes", "created_at")
+      paged_to_already_seen_records = earliest_data_created < last_backfilled
+
+      return episodes, nil if paged_to_already_seen_records
+    end
+
+    return episodes, next_page
+  end
+end