webhookdb 0.1.0

data/lib/webhookdb/replicator/increase_limit_v1.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require "webhookdb/increase"
+require "webhookdb/replicator/increase_v1_mixin"
+
+class Webhookdb::Replicator::IncreaseLimitV1 < Webhookdb::Replicator::Base
+  include Appydays::Loggable
+  include Webhookdb::Replicator::IncreaseV1Mixin
+
+  # @return [Webhookdb::Replicator::Descriptor]
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "increase_limit_v1",
+      ctor: ->(sint) { Webhookdb::Replicator::IncreaseLimitV1.new(sint) },
+      feature_roles: [],
+      resource_name_singular: "Increase Limit",
+      supports_webhooks: true,
+      supports_backfill: true,
+      api_docs_url: "https://increase.com/documentation/api",
+    )
+  end
+
+  def _remote_key_column
+    return Webhookdb::Replicator::Column.new(:increase_id, TEXT, data_key: "id")
+  end
+
+  def _denormalized_columns
+    return [
+      Webhookdb::Replicator::Column.new(:interval, TEXT),
+      Webhookdb::Replicator::Column.new(:metric, TEXT),
+      Webhookdb::Replicator::Column.new(:model_id, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(:model_type, TEXT),
+      Webhookdb::Replicator::Column.new(
+        :row_created_at,
+        TIMESTAMP,
+        data_key: "created_at",
+        event_key: "created_at",
+        defaulter: :now,
+        optional: true,
+        index: true,
+      ),
+      Webhookdb::Replicator::Column.new(
+        :row_updated_at,
+        TIMESTAMP,
+        data_key: "created_at",
+        event_key: "created_at",
+        defaulter: :now,
+        optional: true,
+        index: true,
+      ),
+      Webhookdb::Replicator::Column.new(:status, TEXT),
+      Webhookdb::Replicator::Column.new(:value, INTEGER),
+    ]
+  end
+
+  def _timestamp_column_name
+    return :row_updated_at
+  end
+
+  def _update_where_expr
+    return self.qualified_table_sequel_identifier[:row_updated_at] < Sequel[:excluded][:row_updated_at]
+  end
+
+  def _resource_and_event(request)
+    return self._find_resource_and_event(request.body, "limit")
+  end
+
+  def _upsert_update_expr(inserting, **_kwargs)
+    update = super
+    # Only set created_at if it's not set so the initial insert isn't modified.
+    self._coalesce_excluded_on_update(update, [:row_created_at])
+    return update
+  end
+
+  def _mixin_backfill_url
+    return "#{self.service_integration.api_url}/limits"
+  end
+end

data/lib/webhookdb/replicator/increase_transaction_v1.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "webhookdb/increase"
+require "webhookdb/replicator/increase_v1_mixin"
+
+class Webhookdb::Replicator::IncreaseTransactionV1 < Webhookdb::Replicator::Base
+  include Appydays::Loggable
+  include Webhookdb::Replicator::IncreaseV1Mixin
+
+  # @return [Webhookdb::Replicator::Descriptor]
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "increase_transaction_v1",
+      ctor: ->(sint) { Webhookdb::Replicator::IncreaseTransactionV1.new(sint) },
+      feature_roles: [],
+      resource_name_singular: "Increase Transaction",
+      supports_webhooks: true,
+      supports_backfill: true,
+      api_docs_url: "https://increase.com/documentation/api",
+    )
+  end
+
+  def _remote_key_column
+    return Webhookdb::Replicator::Column.new(:increase_id, TEXT, data_key: "id")
+  end
+
+  def _denormalized_columns
+    return [
+      Webhookdb::Replicator::Column.new(:account_id, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(:amount, INTEGER, index: true),
+      Webhookdb::Replicator::Column.new(
+        :created_at,
+        TIMESTAMP,
+        data_key: "created_at",
+        optional: true,
+        index: true,
+      ),
+      # date is a legacy field that is not documented in the API,
+      # but is still sent with transactions as of April 2022.
+      # We need to support the v1 schema, but do not want to depend
+      # on Increase continuing to send a transaction resource 'date' field.
+      Webhookdb::Replicator::Column.new(
+        :date,
+        DATE,
+        index: true,
+        data_key: "created_at",
+        optional: true,
+        converter: Webhookdb::Replicator::Column::CONV_TO_UTC_DATE,
+      ),
+      Webhookdb::Replicator::Column.new(:route_id, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(
+        :updated_at,
+        TIMESTAMP,
+        data_key: "created_at",
+        event_key: "created_at",
+        defaulter: :now,
+        optional: true,
+        index: true,
+      ),
+    ]
+  end
+
+  def _update_where_expr
+    return self.qualified_table_sequel_identifier[:updated_at] < Sequel[:excluded][:updated_at]
+  end
+
+  def _resource_and_event(request)
+    return self._find_resource_and_event(request.body, "transaction")
+  end
+
+  def _mixin_backfill_url
+    return "#{self.service_integration.api_url}/transactions"
+  end
+end

data/lib/webhookdb/replicator/increase_v1_mixin.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require "webhookdb/increase"
+
+module Webhookdb::Replicator::IncreaseV1Mixin
+  def _mixin_backfill_url
+    raise NotImplementedError
+  end
+
+  def _webhook_response(request)
+    return Webhookdb::Increase.webhook_response(request, self.service_integration.webhook_secret)
+  end
+
+  def _timestamp_column_name
+    return :updated_at
+  end
+
+  def _find_resource_and_event(body, desired_object_name)
+    return nil unless Webhookdb::Increase.contains_desired_object(body, desired_object_name)
+    return body.fetch("data"), body if body.key?("event") && body.key?("event_id")
+    return body, nil
+  end
+
+  def process_state_change(field, value)
+    # special handling for having a default value for api url
+    value = "https://api.increase.com" if field == "api_url" && value == ""
+    return super(field, value)
+  end
+
+  def calculate_webhook_state_machine
+    step = Webhookdb::Replicator::StateMachineStep.new
+    # if the service integration doesn't exist, create it with some standard values
+    unless self.service_integration.webhook_secret.present?
+      step.output = %(You are about to start replicating #{self.resource_name_plural} info into WebhookDB.
+We've made an endpoint available for #{self.resource_name_singular} webhooks:
+
+#{self._webhook_endpoint}
+
+From your Increase admin dashboard, go to Applications -> Create Webhook.
+In the "Webhook endpoint URL" field you can enter the URL above.
+For the shared secret, you'll have to generate a strong password
+(you can use '#{Webhookdb::Id.rand_enc(16)}')
+and then enter it into the textbox.
+
+Copy that shared secret value.
+      )
+      return step.secret_prompt("secret").webhook_secret(self.service_integration)
+    end
+
+    step.output = %(Great! WebhookDB is now listening for #{self.resource_name_singular} webhooks.
+#{self._query_help_output}
+In order to backfill existing #{self.resource_name_plural}, run this from a shell:
+
+  #{self._backfill_command}
+    )
+    return step.completed
+  end
+
+  def calculate_backfill_state_machine
+    step = Webhookdb::Replicator::StateMachineStep.new
+    unless self.service_integration.backfill_key.present?
+      step.output = %(In order to backfill #{self.resource_name_plural}, we need an API key.
+From your Increase admin dashboard, go to Settings -> Development -> API Keys.
+We'll need the Production key--copy that value to your clipboard.
+      )
+      return step.secret_prompt("API Key").backfill_key(self.service_integration)
+    end
+
+    unless self.service_integration.api_url.present?
+      step.output = %(Great. Now we want to make sure we're sending API requests to the right place.
+For Increase, the API url is different when you are in Sandbox mode and when you are in Production mode.
+For Sandbox mode, the API root url is:
+
+https://sandbox.increase.com
+
+For Production mode, which is our default, it is:
+
+https://api.increase.com
+
+Leave blank to use the default or paste the answer into this prompt.
+      )
+      return step.prompting("API url").api_url(self.service_integration)
+    end
+
+    unless (result = self.verify_backfill_credentials).verified
+      self.service_integration.replicator.clear_backfill_information
+      step.output = result.message
+      return step.secret_prompt("API Key").backfill_key(self.service_integration)
+    end
+
+    step.needs_input = false
+    step.output = %(Great! We are going to start backfilling your #{self.resource_name_plural}.
+#{self._query_help_output}
+      )
+    step.complete = true
+    return step
+  end
+
+  def _verify_backfill_401_err_msg
+    return "It looks like that API Key is invalid. Please reenter the API Key you just created:"
+  end
+
+  def _verify_backfill_err_msg
+    return "An error occurred. Please reenter the API Key you just created:"
+  end
+
+  def _fetch_backfill_page(pagination_token, **_kwargs)
+    query = {}
+    (query[:cursor] = pagination_token) if pagination_token.present?
+    response = Webhookdb::Http.get(
+      self._mixin_backfill_url,
+      query,
+      headers: {"Authorization" => ("Bearer " + self.service_integration.backfill_key)},
+      logger: self.logger,
+      timeout: Webhookdb::Increase.http_timeout,
+    )
+    data = response.parsed_response
+    next_page_param = data.dig("response_metadata", "next_cursor")
+    return data["data"], next_page_param
+  end
+end

data/lib/webhookdb/replicator/increase_wire_transfer_v1.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "webhookdb/increase"
+require "webhookdb/replicator/increase_v1_mixin"
+
+class Webhookdb::Replicator::IncreaseWireTransferV1 < Webhookdb::Replicator::Base
+  include Appydays::Loggable
+  include Webhookdb::Replicator::IncreaseV1Mixin
+
+  # @return [Webhookdb::Replicator::Descriptor]
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "increase_wire_transfer_v1",
+      ctor: ->(sint) { Webhookdb::Replicator::IncreaseWireTransferV1.new(sint) },
+      feature_roles: [],
+      resource_name_singular: "Increase Wire Transfer",
+      supports_webhooks: true,
+      supports_backfill: true,
+      api_docs_url: "https://increase.com/documentation/api",
+    )
+  end
+
+  def _remote_key_column
+    return Webhookdb::Replicator::Column.new(:increase_id, TEXT, data_key: "id")
+  end
+
+  def _denormalized_columns
+    return [
+      Webhookdb::Replicator::Column.new(:account_number, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(:account_id, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(:amount, INTEGER, index: true),
+      Webhookdb::Replicator::Column.new(:approved_at, TIMESTAMP, data_key: ["approval", "approved_at"]),
+      Webhookdb::Replicator::Column.new(:created_at, TIMESTAMP, optional: true, index: true),
+      Webhookdb::Replicator::Column.new(:routing_number, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(:status, TEXT),
+      Webhookdb::Replicator::Column.new(:template_id, TEXT),
+      Webhookdb::Replicator::Column.new(:transaction_id, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(
+        :updated_at,
+        TIMESTAMP,
+        data_key: "created_at",
+        event_key: "created_at",
+        defaulter: :now,
+        optional: true,
+        index: true,
+      ),
+    ]
+  end
+
+  def _resource_and_event(request)
+    return self._find_resource_and_event(request.body, "wire_transfer")
+  end
+
+  def _update_where_expr
+    return self.qualified_table_sequel_identifier[:updated_at] < Sequel[:excluded][:updated_at]
+  end
+
+  def _mixin_backfill_url
+    return "#{self.service_integration.api_url}/wire_transfers"
+  end
+end

data/lib/webhookdb/replicator/intercom_contact_v1.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "webhookdb/intercom"
+require "webhookdb/replicator/intercom_v1_mixin"
+
+class Webhookdb::Replicator::IntercomContactV1 < Webhookdb::Replicator::Base
+  include Appydays::Loggable
+  include Webhookdb::Replicator::IntercomV1Mixin
+
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "intercom_contact_v1",
+      ctor: self,
+      feature_roles: ["intercom"],
+      resource_name_singular: "Intercom Contact",
+      dependency_descriptor: Webhookdb::Replicator::IntercomMarketplaceRootV1.descriptor,
+      supports_backfill: true,
+      api_docs_url: "https://developers.intercom.com/docs/references/rest-api/api.intercom.io/Contacts/",
+    )
+  end
+
+  def _remote_key_column
+    return Webhookdb::Replicator::Column.new(:intercom_id, TEXT, data_key: "id")
+  end
+
+  def _denormalized_columns
+    return [
+      Webhookdb::Replicator::Column.new(:external_id, TEXT),
+      Webhookdb::Replicator::Column.new(:email, TEXT),
+      Webhookdb::Replicator::Column.new(:created_at, TIMESTAMP, converter: :tsat),
+      Webhookdb::Replicator::Column.new(:updated_at, TIMESTAMP, converter: :tsat),
+    ]
+  end
+
+  def _mixin_backfill_url = "https://api.intercom.io/contacts"
+end

data/lib/webhookdb/replicator/intercom_conversation_v1.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "webhookdb/intercom"
+require "webhookdb/replicator/intercom_v1_mixin"
+
+class Webhookdb::Replicator::IntercomConversationV1 < Webhookdb::Replicator::Base
+  include Appydays::Loggable
+  include Webhookdb::Replicator::IntercomV1Mixin
+
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "intercom_conversation_v1",
+      ctor: self,
+      feature_roles: ["intercom"],
+      resource_name_singular: "Intercom Conversation",
+      dependency_descriptor: Webhookdb::Replicator::IntercomMarketplaceRootV1.descriptor,
+      supports_backfill: true,
+      api_docs_url: "https://developers.intercom.com/docs/references/rest-api/api.intercom.io/Conversations/",
+    )
+  end
+
+  def _remote_key_column
+    return Webhookdb::Replicator::Column.new(:intercom_id, TEXT, data_key: "id")
+  end
+
+  def _denormalized_columns
+    return [
+      Webhookdb::Replicator::Column.new(:title, TEXT),
+      Webhookdb::Replicator::Column.new(:state, TEXT),
+      Webhookdb::Replicator::Column.new(:open, BOOLEAN),
+      Webhookdb::Replicator::Column.new(:read, BOOLEAN),
+      Webhookdb::Replicator::Column.new(:created_at, TIMESTAMP, converter: :tsat),
+      Webhookdb::Replicator::Column.new(:updated_at, TIMESTAMP, converter: :tsat),
+    ]
+  end
+
+  def _mixin_backfill_url = "https://api.intercom.io/conversations"
+end

data/lib/webhookdb/replicator/intercom_marketplace_root_v1.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+class Webhookdb::Replicator::IntercomMarketplaceRootV1 < Webhookdb::Replicator::Base
+  include Appydays::Loggable
+
+  # @return [Webhookdb::Replicator::Descriptor]
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "intercom_marketplace_root_v1",
+      ctor: self,
+      feature_roles: ["intercom"],
+      resource_name_singular: "Intercom Auth",
+      resource_name_plural: "Intercom Auth",
+      supports_backfill: true,
+      description: "You can replicate your Intercom data to WebhookDB Cloud in one click using  " \
+                   "the [Intercom App Store](https://www.intercom.com/app-store).",
+    )
+  end
+
+  def _remote_key_column
+    return Webhookdb::Replicator::Column.new(:ignore_id, INTEGER)
+  end
+
+  def _denormalized_columns
+    return []
+  end
+
+  def _upsert_webhook(**_kwargs)
+    raise NotImplementedError("This is a stub integration only for auth purposes.")
+  end
+
+  def _fetch_backfill_page(*)
+    return [], nil
+  end
+
+  def webhook_response(_request)
+    raise NotImplementedError("This is a stub integration only for auth purposes.")
+  end
+
+  def calculate_backfill_state_machine
+    step = Webhookdb::Replicator::StateMachineStep.new
+    step.output = "This integration cannot be modified through the command line."
+    step.completed
+    return step
+  end
+
+  def get_auth_headers
+    return {
+      "Authorization" => "Bearer #{self.service_integration.backfill_key}",
+      "Accept" => "application/json",
+    }
+  end
+
+  def build_dependents
+    org = self.service_integration.organization
+    contact_sint = Webhookdb::ServiceIntegration.create_disambiguated(
+      "intercom_contact_v1",
+      organization: org,
+      depends_on: self.service_integration,
+    )
+    conversation_sint = Webhookdb::ServiceIntegration.create_disambiguated(
+      "intercom_conversation_v1",
+      organization: org,
+      depends_on: self.service_integration,
+    )
+    contact_sint.replicator._enqueue_backfill_jobs(incremental: true)
+    conversation_sint.replicator._enqueue_backfill_jobs(incremental: true)
+  end
+end

data/lib/webhookdb/replicator/intercom_v1_mixin.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module Webhookdb::Replicator::IntercomV1Mixin
+  # @return [Webhookdb::ServiceIntegration]
+
+  # Quick note on these Intercom integrations: although we will technically be bringing in information from webhooks,
+  # all webhooks for the WebhookDB app will use a single endpoint and we use the WebhookDB app's Client Secret for
+  # webhook verification, which means that webhooks actually don't require any setup on the integration level. Thus,
+  # `supports_webhooks` is false.
+  def find_auth_integration
+    # rubocop:disable Naming/MemoizedInstanceVariableName
+    return @auth ||= Webhookdb::Replicator.find_at_root!(self.service_integration,
+                                                         service_name: "intercom_marketplace_root_v1",)
+    # rubocop:enable Naming/MemoizedInstanceVariableName
+  end
+
+  def intercom_auth_headers
+    root_sint = self.find_auth_integration
+    return Webhookdb::Intercom.auth_headers(root_sint.backfill_key)
+  end
+
+  def auth_credentials?
+    auth = self.find_auth_integration
+    return auth.backfill_key.present?
+  end
+
+  def _resource_and_event(request)
+    body = request.body
+    return body.fetch("data").fetch("item"), body if body.fetch("type") == "notification_event"
+    return body, nil
+  end
+
+  def _update_where_expr
+    return self.qualified_table_sequel_identifier[:updated_at] < Sequel[:excluded][:updated_at]
+  end
+
+  def _timestamp_column_name
+    return :updated_at
+  end
+
+  def _webhook_response(request)
+    # info for debugging
+    intercom_auth = request.env["HTTP_X_HUB_SIGNATURE"]
+
+    return Webhookdb::WebhookResponse.error("missing hmac") if intercom_auth.nil?
+    request.body.rewind
+    request_data = request.body.read
+    verified = Webhookdb::Intercom.verify_webhook(request_data, intercom_auth)
+    return Webhookdb::WebhookResponse.ok if verified
+    return Webhookdb::WebhookResponse.error("invalid hmac")
+  end
+
+  # @return [Webhookdb::Replicator::StateMachineStep]
+  def calculate_backfill_state_machine
+    # can inherit credentials from the auth dependency
+    if (step = self.calculate_dependency_state_machine_step(dependency_help: ""))
+      return step
+    end
+    step = Webhookdb::Replicator::StateMachineStep.new
+    step.output = %(We will start replicating #{self.resource_name_singular} information into your WebhookDB database.
+
+#{self._query_help_output(prefix: "Once data is available, you can query #{self.resource_name_plural}.")})
+    return step.completed
+  end
+
+  def on_dependency_webhook_upsert(_replicator, _payload, *)
+    return
+  end
+
+  def _mixin_backfill_url
+    raise NotImplementedError
+  end
+
+  def _fetch_backfill_page(pagination_token, **_kwargs)
+    unless self.auth_credentials?
+      raise Webhookdb::Replicator::CredentialsMissing,
+            "This integration requires that the Intercom Auth integration has a valid Auth Token"
+    end
+
+    query = {per_page: Webhookdb::Intercom.page_size}
+    # Intercom started 500ing with this set to empty.
+    query[:starting_after] = pagination_token if pagination_token
+    begin
+      response = Webhookdb::Http.get(
+        self._mixin_backfill_url,
+        query:,
+        headers: self.intercom_auth_headers,
+        logger: self.logger,
+        timeout: Webhookdb::Intercom.http_timeout,
+      )
+    rescue Webhookdb::Http::Error => e
+      #  We are looking to catch the "api plan restricted" error. This is always a 403 and every
+      # 403 will be an "api plan restricted" error according to the API documentation. Because we
+      # specify the API version in our headers we can expect that this won't change.
+      raise e unless e.status == 403
+      self.logger.warn("intercom_api_restricted", intercom_error: e.body)
+      # We should basically noop here, i.e. pretend that the page is empty, so that we don't trigger
+      # a TypeError in the backfiller.
+      return [], nil
+    end
+    data = response.parsed_response.fetch("data", [])
+    starting_after = response.parsed_response.dig("pages", "next", "starting_after")
+    return data, starting_after
+  end
+end

data/lib/webhookdb/replicator/oauth_refresh_access_token_mixin.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "webhookdb/redis"
+
+module Webhookdb::Replicator::OAuthRefreshAccessTokenMixin
+  EXPIRATION_BUFFER = 60
+
+  def oauth_cache_key_namespace = raise NotImplementedError("return something like 'gcalv1'")
+  def oauth_token_url = raise NotImplementedError("return something like https://someservice/oauth2/token")
+
+  def oauth_http_timeout = raise NotImplementedError("return a timeout value for oauth http requests")
+
+  def oauth_access_token_cache_key(oauth_user_id)
+    parts = [self.oauth_cache_key_namespace, "atok", self.service_integration.id.to_s, oauth_user_id]
+    return Webhookdb::Redis.cache_key(parts)
+  end
+
+  def delete_oauth_access_token(oauth_user_id)
+    key = self.oauth_access_token_cache_key(oauth_user_id)
+    Webhookdb::Redis.cache.with do |r|
+      r.call("DEL", key)
+    end
+  end
+
+  def _with_oauth_access_token(oauth_user_id, get_refresh_token)
+    key = self.oauth_access_token_cache_key(oauth_user_id)
+    Webhookdb::Redis.cache.with do |r|
+      got = r.call("GET", key)
+      if got
+        yield got
+      else
+        self.logger.info "creating_access_token"
+        form_body = URI.encode_www_form(
+          {
+            client_id: self.service_integration.backfill_key,
+            client_secret: self.service_integration.backfill_secret,
+            refresh_token: get_refresh_token.call,
+            grant_type: "refresh_token",
+          },
+        )
+        resp = Webhookdb::Http.post(
+          self.oauth_token_url,
+          form_body,
+          headers: {
+            "Content-Type" => "application/x-www-form-urlencoded",
+            "charset" => "utf-8",
+          },
+          logger: self.logger,
+          timeout: self.oauth_http_timeout,
+        )
+        access_token = resp.parsed_response.fetch("access_token")
+        r.call("SETEX", key, resp.parsed_response["expires_in"] - EXPIRATION_BUFFER, access_token)
+        yield access_token
+      end
+    end
+  end
+
+  def force_set_oauth_access_token(oauth_user_id, access_token, expires_in: 60.minutes.to_i)
+    key = self.oauth_access_token_cache_key(oauth_user_id)
+    Webhookdb::Redis.cache.with do |r|
+      r.call("SETEX", key, expires_in, access_token)
+      return access_token
+    end
+  end
+end