webhookdb 0.1.0

data/lib/webhookdb/replicator/convertkit_subscriber_v1.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+require "time"
+require "webhookdb/convertkit"
+require "webhookdb/replicator/convertkit_v1_mixin"
+
+class Webhookdb::Replicator::ConvertkitSubscriberV1 < Webhookdb::Replicator::Base
+  include Appydays::Loggable
+  include Webhookdb::Replicator::ConvertkitV1Mixin
+
+  # @return [Webhookdb::Replicator::Descriptor]
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "convertkit_subscriber_v1",
+      ctor: ->(sint) { Webhookdb::Replicator::ConvertkitSubscriberV1.new(sint) },
+      feature_roles: [],
+      resource_name_singular: "ConvertKit Subscriber",
+      supports_webhooks: true,
+      supports_backfill: true,
+      description: "Replicate ConvertKit subscribers into a database. " \
+                   "This is one of the only ways you can keep track of historical subscriber information " \
+                   "with ConvertKit.",
+      api_docs_url: "https://developers.convertkit.com/#list-subscribers",
+    )
+  end
+
+  def process_state_change(field, value)
+    step = super
+    self._create_webhooks if field == "backfill_secret"
+    return step
+  end
+
+  def _create_webhooks
+    # ConvertKit has made several other webhooks available for the subscriber object, but they all have required
+    # parameters that are pks of other objects that webhookdb knows nothing about.
+
+    # first verify that the webhooks don't exist
+    url = "https://api.convertkit.com/v3/automations/hooks?api_secret=#{self.service_integration.backfill_secret}"
+    response = Webhookdb::Http.get(url, logger: self.logger, timeout: Webhookdb::Convertkit.http_timeout)
+    # the data returned here is a list of the existing webhooks
+    data = response.parsed_response
+
+    # does the "subscriber.subscriber_activate" exist? if not, create it
+    # rubocop:disable Style/GuardClause
+    if data.present?
+      sub_activate_webhook = data.find do |obj|
+        obj.dig("rule", "event", "name") == "subscriber_activate"
+      end
+    end
+    unless sub_activate_webhook.present?
+      Webhookdb::Http.post(
+        "https://api.convertkit.com/v3/automations/hooks",
+        {
+          "api_secret" => self.service_integration.backfill_secret,
+          "target_url" => self.webhook_endpoint,
+          "event" => {"name" => "subscriber.subscriber_activate"},
+        },
+        logger: self.logger,
+        timeout: Webhookdb::Convertkit.http_timeout,
+      )
+      end
+
+    # does the "subscriber.subscriber_activate" exist? if not, create it
+    if data.present?
+      sub_unsubscribe_webhook = data.find do |obj|
+        obj.dig("rule", "event", "name") == "subscriber_activate"
+      end
+    end
+    unless sub_unsubscribe_webhook.present?
+      Webhookdb::Http.post(
+        "https://api.convertkit.com/v3/automations/hooks",
+        {
+          "api_secret" => self.service_integration.backfill_secret,
+          "target_url" => self.webhook_endpoint,
+          "event" => {"name" => "subscriber.subscriber_unsubscribe"},
+        },
+        logger: self.logger,
+        timeout: Webhookdb::Convertkit.http_timeout,
+      )
+    end
+    # rubocop:enable Style/GuardClause
+  end
+
+  def calculate_webhook_state_machine
+    step = Webhookdb::Replicator::StateMachineStep.new
+    step.output = %(
+Great! We've created your ConvertKit Subscribers integration.
+
+ConvertKit supports Subscriber webhooks.
+You have two options for hooking them up:
+
+A: Create the webhook yourself, so you don't need to provide us an API Secret.
+Run this from a shell:
+
+  curl -X POST https://api.convertkit.com/v3/automations/hooks
+     -H 'Content-Type: application/json'\\
+     -d '{ "api_secret": "<your_secret_api_key>",\\
+           "target_url": "#{self._webhook_endpoint}",\\
+           "event": { "name": "subscriber.subscriber_activate" } }'
+  curl -X POST https://api.convertkit.com/v3/automations/hooks
+     -H 'Content-Type: application/json'\\
+     -d '{ "api_secret": "<your_secret_api_key>",\\
+           "target_url": "#{self._webhook_endpoint}",\\
+           "event": { "name": "subscriber.subscriber_unsubscribe" } }'
+
+B: Use WebhookDB to backfill historical data with your API Secret, and when we do this,
+we'll also set up webhooks for new data.
+To start backfilling historical data, run this from a shell:
+
+  #{self._backfill_command}
+
+Once you have data (you set up the webhooks, or ran the 'backfill' command),
+your database will be populated.
+#{self._query_help_output}
+      )
+    return step.completed
+  end
+
+  def _denormalized_columns
+    return [
+      Webhookdb::Replicator::Column.new(
+        :canceled_at,
+        TIMESTAMP,
+        index: true,
+        optional: true,
+        converter: CONV_FIND_CANCELED_AT,
+      ),
+      Webhookdb::Replicator::Column.new(:created_at, TIMESTAMP, data_key: "created_at", index: true),
+      Webhookdb::Replicator::Column.new(:email_address, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(:first_name, TEXT),
+      Webhookdb::Replicator::Column.new(:last_name, TEXT, data_key: ["fields", "last_name"], optional: true),
+      Webhookdb::Replicator::Column.new(:state, TEXT),
+    ]
+  end
+
+  def _timestamp_column_name
+    return :created_at
+  end
+
+  def _resource_and_event(request)
+    body = request.body
+    return body["subscriber"], body if body.key?("subscriber").present?
+    return body, nil
+  end
+
+  def _update_where_expr
+    return self.qualified_table_sequel_identifier[:data] !~ Sequel[:excluded][:data]
+  end
+
+  def _upsert_update_expr(inserting, **_kwargs)
+    update = super
+    state = inserting.fetch(:state)
+    # If the state is active, we want to use canceled_at:nil unconditionally.
+    return update if state == "active"
+    # If it's inactive, we only want to update canceled_at if it's not already set
+    # (coalesce the existing row's canceled_at with the 'time.now' we are passing in).
+    self._coalesce_excluded_on_update(update, [:canceled_at])
+    return update
+  end
+
+  def _fetch_backfill_page(pagination_token, last_backfilled:)
+    pagination_token ||= ["subscribed", 1]
+    list_being_iterated, page = pagination_token
+
+    url = "https://api.convertkit.com/v3/subscribers?api_secret=#{self.service_integration.backfill_secret}&page=#{page}&sort_order=desc"
+    url += "&updated_from=#{last_backfilled.strftime('%FT%TZ')}" if last_backfilled.present?
+    url += "&sort_field=cancelled_at" if list_being_iterated == "cancelled"
+
+    response = Webhookdb::Http.get(url, logger: self.logger, timeout: Webhookdb::Convertkit.http_timeout)
+    data = response.parsed_response
+    current_page = data["page"]
+    total_pages = data["total_pages"]
+    subs = data["subscribers"]
+
+    if last_backfilled.present?
+      earliest_data_created = subs.empty? ? Time.at(0) : subs[-1].fetch("created_at")
+      paged_to_already_seen_records = earliest_data_created < last_backfilled
+
+      if paged_to_already_seen_records && list_being_iterated == "subscribed"
+        # If we are done backfilling from the 'subscribed' list, we can now iterate cancelled
+        return subs, ["cancelled", 1]
+      end
+      if paged_to_already_seen_records && list_being_iterated == "cancelled"
+        # If we are done backfilling from the 'cancelled' list, we are done backfilling
+        return subs, nil
+      end
+    end
+
+    if current_page < total_pages
+      # If we still have pages on this list, go to the next one
+      return subs, [list_being_iterated, current_page + 1]
+    end
+    if list_being_iterated == "subscribed"
+      # If we are done with the 'subscribed' list, we can now iterate cancelled
+      return subs, ["cancelled", 1]
+    end
+    # Otherwise, we're at the last page of our canceled subscribers list
+    return subs, nil
+  end
+end

data/lib/webhookdb/replicator/convertkit_tag_v1.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "time"
+require "webhookdb/convertkit"
+require "webhookdb/replicator/convertkit_v1_mixin"
+
+class Webhookdb::Replicator::ConvertkitTagV1 < Webhookdb::Replicator::Base
+  include Appydays::Loggable
+  include Webhookdb::Replicator::ConvertkitV1Mixin
+
+  # @return [Webhookdb::Replicator::Descriptor]
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "convertkit_tag_v1",
+      ctor: ->(sint) { Webhookdb::Replicator::ConvertkitTagV1.new(sint) },
+      feature_roles: [],
+      resource_name_singular: "ConvertKit Tag",
+      supports_backfill: true,
+      api_docs_url: "https://developers.convertkit.com/#list-tags",
+    )
+  end
+
+  def _denormalized_columns
+    return [
+      Webhookdb::Replicator::Column.new(:created_at, TIMESTAMP, data_key: "created_at", index: true),
+      Webhookdb::Replicator::Column.new(:name, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(:total_subscriptions, INTEGER, from_enrichment: true),
+    ]
+  end
+
+  def _timestamp_column_name
+    return :created_at
+  end
+
+  def _resource_and_event(request)
+    return request.body, nil
+  end
+
+  def _update_where_expr
+    return self.qualified_table_sequel_identifier[:data] !~ Sequel[:excluded][:data]
+  end
+
+  def upsert_has_deps?
+    return true
+  end
+
+  def _store_enrichment_body?
+    return true
+  end
+
+  def _fetch_enrichment(resource, _event, _request)
+    tag_id = resource.fetch("id")
+    url = "https://api.convertkit.com/v3/tags/#{tag_id}/subscriptions?api_secret=#{self.service_integration.backfill_secret}"
+    response = Webhookdb::Http.get(url, logger: self.logger, timeout: Webhookdb::Convertkit.http_timeout)
+    data = response.parsed_response
+    return data
+  end
+
+  def _fetch_backfill_page(_pagination_token, **_kwargs)
+    # this endpoint does not have pagination support
+    url = "https://api.convertkit.com/v3/tags?api_secret=#{self.service_integration.backfill_secret}"
+    response = Webhookdb::Http.get(url, logger: self.logger, timeout: Webhookdb::Convertkit.http_timeout)
+    data = response.parsed_response
+    return data["tags"], nil
+  end
+end

data/lib/webhookdb/replicator/convertkit_v1_mixin.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Webhookdb::Replicator::ConvertkitV1Mixin
+  include Webhookdb::DBAdapter::ColumnTypes
+
+  # ConvertKit gets a decent number of 5xx responses.
+  # Wait for about 20 minutes before the job dies.
+  def backfiller_server_error_retries = 10
+  def backfiller_server_error_backoff = 121.seconds
+
+  def _webhook_response(_request)
+    # Webhook Authentication isn't supported
+    return Webhookdb::WebhookResponse.ok
+  end
+
+  def _remote_key_column
+    return Webhookdb::Replicator::Column.new(:convertkit_id, BIGINT, data_key: "id")
+  end
+
+  def calculate_backfill_state_machine
+    step = Webhookdb::Replicator::StateMachineStep.new
+    if self.service_integration.backfill_secret.blank?
+      step.output = %(Great! We've created your #{self.resource_name_plural} integration.
+
+#{self.resource_name_singular} webhooks are not designed to mirror data, so to fill your database,
+we need to use the API to make requests, which requires your API Secret.
+#{Webhookdb::Convertkit::FIND_API_SECRET_HELP}
+      )
+      return step.secret_prompt("API Secret").backfill_secret(self.service_integration)
+    end
+
+    unless (result = self.verify_backfill_credentials).verified
+      self.service_integration.replicator.clear_backfill_information
+      step.output = result.message
+      return step.secret_prompt("API Secret").backfill_secret(self.service_integration)
+    end
+
+    step.output = %(We'll start backfilling your #{self.resource_name_plural} now,
+and they will show up in your database momentarily.
+#{self._query_help_output}
+    )
+    return step.completed
+  end
+
+  def _verify_backfill_401_err_msg
+    return "It looks like that API Secret is invalid. Please reenter the API Secret:"
+  end
+
+  def _verify_backfill_err_msg
+    return "An error occurred. Please reenter the API Secret:"
+  end
+
+  # Converter for use with the denormalized columns
+  CONV_FIND_CANCELED_AT = Webhookdb::Replicator::Column::IsomorphicProc.new(
+    ruby: lambda do |_, resource:, **_|
+      # Subscribers do not store a cancelation time (nor an updated at time),
+      # so we derive and store it based on their state.
+      # When they become inactive state, we set canceled_at,
+      # and clear it when they are not active.
+      # See the upsert_update_expr for Convertkit Subscriber for the details.
+      resource.fetch("state") == "active" ? nil : Time.now
+    end,
+    sql: nil,
+  )
+end

data/lib/webhookdb/replicator/docgen.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+# Write docs for docs.webhookdb.com Jekyll site.
+class Webhookdb::Replicator::Docgen
+  def self.documentable_descriptors
+    return Webhookdb::Replicator.registry.values.reject do |repl|
+      repl.name.start_with?("webhookdb_", "fake_")
+    end.sort_by(&:name)
+  end
+
+  # @!attribute desc
+  #   @return [Webhookdb::Replicator::Descriptor]
+  attr_reader :desc
+  attr_reader :lines
+
+  # @param desc [Webhookdb::Replicator::Descriptor]
+  def initialize(desc)
+    @desc = desc
+    @lines = []
+    @documentable_descriptors = self.class.documentable_descriptors
+  end
+
+  def sint
+    @sint ||= Webhookdb::ServiceIntegration.new(
+      service_name: desc.name,
+      opaque_id: "svi_fixture",
+      table_name: desc.name + "_fixture",
+    )
+  end
+
+  def repl
+    @repl ||= desc.ctor[sint]
+  end
+
+  def markdown
+    _frontmatter
+    _intro
+    _features
+    _schema
+    _tabledef
+    _prevnext
+    return lines.join("\n")
+  end
+
+  def _frontmatter
+    lines << "---"
+    lines << "title: #{desc.resource_name_singular}"
+    lines << "layout: home"
+    idx = @documentable_descriptors.index(desc)
+    lines << "nav_order: #{(idx + 1) * 10}"
+    lines << "---"
+    lines << ""
+  end
+
+  def _intro
+    lines << "# #{desc.resource_name_singular} (`#{desc.name}`)"
+    if desc.description.present?
+      lines << ""
+      lines << desc.description
+    end
+    if desc.api_docs_url.present?
+      lines << ""
+      lines << "Docs for this API: [#{desc.api_docs_url}](#{desc.api_docs_url})"
+    end
+    lines << ""
+  end
+
+  def _features
+    lines << "## Features"
+    lines << ""
+    lines << "<dl>"
+    if (dep = desc.dependency_descriptor)
+      lines << "<dt>Depends on</dt>"
+      lines << "<dd>To use this replicator, you will need #{refanchor(dep)}. " \
+               "You'll be prompted to create it if you haven't.</dd>"
+      lines << ""
+    end
+    deps = @documentable_descriptors.select { |d| d.dependency_descriptor == desc }
+    if deps.any?
+      lines << "<dt>Dependents</dt>"
+      lines << "<dd>This replicator is required for the creation of the following dependents:"
+      lines << "<ul>"
+      deps.each { |d| lines << "<li>#{refanchor(d)}</li>" }
+      lines << "</ul>"
+      lines << "</dd>"
+      lines << ""
+    end
+    lines << "<dt>Supports Webhooks</dt>"
+    lines << "<dd>#{boolmoji(desc.supports_webhooks?)}</dd>"
+    lines << "<dt>Supports Backfilling</dt>"
+    lines << "<dd>#{boolmoji(desc.supports_backfill?)}</dd>"
+    if desc.enterprise?
+      lines << "<dt>Enterprise Only</dt>"
+      lines << "<dd>Yes</dd>"
+    end
+    lines << ""
+    lines << "</dl>"
+    lines << ""
+  end
+
+  def _schema
+    lines << "## Schema"
+    lines << ""
+    lines << "Tables replicated from #{desc.resource_name_plural} have this schema.
+Note that the data types listed are for Postgres;
+when [replicating to other databases]({% link _concepts/replication_databases.md %}),
+other data types maybe used."
+    lines << ""
+    lines << "| Column | Type | Indexed |"
+    columns = [repl.primary_key_column, repl.remote_key_column]
+    columns.concat(repl.storable_columns)
+    columns << repl.data_column
+    columns.each do |c|
+      name = "`#{c.name}`"
+      (name += "*") if c.name == :data
+      lines << "| #{name} | `#{pgtype(c.type)}` | #{truecheck(c.index)} |"
+    end
+    lines << ""
+    lines << <<~S
+      <span class="fs-3">* The `data` column contains the raw payload from the webhook or API.
+      In many cases there is no canonical form, like if a webhook and API request return
+      two different versions of the same resource.
+      In that case we try to keep the most coherent and detailed resource."</span>
+    S
+  end
+
+  def _tabledef
+    lines << "## Table definition"
+    lines << ""
+    lines << "This definition can also be generated through `webhookdb fixture #{desc.name}`."
+    lines << ""
+    lines << "```sql"
+    lines << repl.create_table_modification.to_s
+    lines << "```"
+    lines << ""
+  end
+
+  def _prevnext
+    idx = @documentable_descriptors.index(desc)
+    raise Webhookdb::InvariantViolation if idx.nil?
+    prevtxt = nexttxt = ""
+    if (rprev = idx.zero? ? nil : @documentable_descriptors[idx - 1])
+      prevtxt = "prev='_integrations/#{rprev.name}.md' prevLabel='#{rprev.name}' "
+    end
+    if (rnext = idx == (@documentable_descriptors.size - 1) ? nil : @documentable_descriptors[idx + 1])
+      nexttxt = "next='_integrations/#{rnext.name}.md' nextLabel='#{rnext.name}'"
+    end
+    lines << "{% include prevnext.html #{prevtxt}#{nexttxt} %}"
+    lines << ""
+  end
+
+  def refhref(d) = "{% link _integrations/#{d.name}.md %}"
+  def refanchor(d) = "<a href=\"#{refhref(d)}\">#{d.name}</a>"
+  def boolmoji(b) = b ? "✅" : "❌"
+  def truecheck(b) = b ? "✅" : ""
+  def pgtype(t) = Webhookdb::DBAdapter::PG::COLTYPE_MAP[t]
+
+  def self.replicator_list_md(descriptors)
+    lines = []
+    descriptors.each do |d|
+      line = "- [#{d.resource_name_singular}]({% link _integrations/#{d.name}.md %})"
+      line += " ([Enterprise]({% link docs/enterprise.md %}) only)" if d.enterprise
+      lines << line
+    end
+    return lines.join("\n")
+  end
+end

data/lib/webhookdb/replicator/email_octopus_campaign_v1.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "webhookdb/email_octopus"
+
+class Webhookdb::Replicator::EmailOctopusCampaignV1 < Webhookdb::Replicator::Base
+  include Appydays::Loggable
+
+  # @return [Webhookdb::Replicator::Descriptor]
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "email_octopus_campaign_v1",
+      ctor: ->(sint) { Webhookdb::Replicator::EmailOctopusCampaignV1.new(sint) },
+      feature_roles: [],
+      resource_name_singular: "Email Octopus Campaign",
+      dependency_descriptor: Webhookdb::Replicator::EmailOctopusListV1.descriptor,
+      supports_backfill: true,
+      api_docs_url: "https://emailoctopus.com/api-documentation",
+    )
+  end
+
+  def _remote_key_column
+    return Webhookdb::Replicator::Column.new(:email_octopus_id, TEXT, data_key: "id")
+  end
+
+  def _denormalized_columns
+    return [
+      Webhookdb::Replicator::Column.new(:name, TEXT),
+      Webhookdb::Replicator::Column.new(:created_at, TIMESTAMP, index: true, converter: :time),
+      Webhookdb::Replicator::Column.new(:sent_at, TIMESTAMP, index: true, converter: :time),
+      Webhookdb::Replicator::Column.new(:status, TEXT),
+      Webhookdb::Replicator::Column.new(:from_name, TEXT, data_key: ["from", "name"]),
+      Webhookdb::Replicator::Column.new(:from_email_address, TEXT, data_key: ["from", "email_address"]),
+      Webhookdb::Replicator::Column.new(:subject, TEXT),
+      Webhookdb::Replicator::Column.new(:row_updated_at, TIMESTAMP, defaulter: :now, optional: true),
+    ]
+  end
+
+  def _resource_and_event(request)
+    return request.body, nil
+  end
+
+  def _update_where_expr
+    return self.qualified_table_sequel_identifier[:data] !~ Sequel[:excluded][:data]
+  end
+
+  def _timestamp_column_name
+    return :row_updated_at
+  end
+
+  def _webhook_response(_request)
+    return Webhookdb::WebhookResponse.ok
+  end
+
+  def on_dependency_webhook_upsert(_replicator, _payload, *)
+    return
+  end
+
+  def calculate_backfill_state_machine
+    if (step = self.calculate_dependency_state_machine_step(dependency_help: ""))
+      return step
+    end
+    step = Webhookdb::Replicator::StateMachineStep.new
+    # We're using the API Key from the dependency, we don't need to ask for it here
+    step.output = %(Great! We are going to start replicating your #{self.resource_name_plural}.
+#{self._query_help_output}
+    )
+    return step.completed
+  end
+
+  def _fetch_backfill_page(pagination_token, **)
+    api_key = self.service_integration.depends_on.replicator.backfill_key!
+    limit = Webhookdb::EmailOctopus.page_size
+    base_url = "https://emailoctopus.com"
+    endpoint_path = pagination_token || "/api/1.6/campaigns?api_key=#{api_key}&limit=#{limit}"
+    response = Webhookdb::Http.get(
+      base_url + endpoint_path,
+      logger: self.logger,
+      timeout: Webhookdb::EmailOctopus.http_timeout,
+    )
+    data = response.parsed_response
+    next_page_link = data.dig("paging", "next")
+    return data["data"], next_page_link
+  end
+end