webhookdb 0.1.0

data/lib/webhookdb/replicator/transistor_show_v1.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "webhookdb/replicator/transistor_v1_mixin"
+
+class Webhookdb::Replicator::TransistorShowV1 < Webhookdb::Replicator::Base
+  include Appydays::Loggable
+  include Webhookdb::Replicator::TransistorV1Mixin
+
+  # @return [Webhookdb::Replicator::Descriptor]
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "transistor_show_v1",
+      ctor: ->(sint) { Webhookdb::Replicator::TransistorShowV1.new(sint) },
+      feature_roles: [],
+      resource_name_singular: "Transistor Show",
+      supports_backfill: true,
+      api_docs_url: "https://developers.transistor.fm/#Show",
+    )
+  end
+
+  def _denormalized_columns
+    return [
+      Webhookdb::Replicator::Column.new(:author, TEXT, data_key: ["attributes", "author"]),
+      Webhookdb::Replicator::Column.new(
+        :created_at, TIMESTAMP,
+        index: true,
+        data_key: ["attributes", "created_at"],
+      ),
+      Webhookdb::Replicator::Column.new(:description, TEXT, data_key: ["attributes", "description"]),
+      Webhookdb::Replicator::Column.new(:title, TEXT, data_key: ["attributes", "title"]),
+      Webhookdb::Replicator::Column.new(
+        :updated_at,
+        TIMESTAMP,
+        index: true,
+        data_key: ["attributes", "updated_at"],
+      ),
+      Webhookdb::Replicator::Column.new(:website, TEXT, data_key: ["attributes", "website"]),
+    ]
+  end
+
+  def _fetch_backfill_page(pagination_token, **_kwargs)
+    pagination_token ||= [false, 1]
+    iterating_over_private, page = pagination_token
+
+    url = "https://api.transistor.fm/v1/shows"
+
+    response = Webhookdb::Http.get(
+      url,
+      headers: {"x-api-key" => self.service_integration.backfill_key},
+      body: {pagination: {page:}, private: iterating_over_private},
+      logger: self.logger,
+      timeout: Webhookdb::Transistor.http_timeout,
+    )
+    data = response.parsed_response
+    current_page = data["meta"]["currentPage"]
+    total_pages = data["meta"]["totalPages"]
+    shows = data["data"]
+
+    if current_page < total_pages
+      # If we still have pages on this list, go to the next one
+      return shows, [iterating_over_private, current_page + 1]
+    end
+    # If we are done with the public list, we can now iterate over private shows
+    return shows, [true, 1] unless iterating_over_private
+    # Otherwise we are on the last page of our private list
+    return shows, nil
+  end
+end

data/lib/webhookdb/replicator/transistor_v1_mixin.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Webhookdb::Replicator::TransistorV1Mixin
+  include Webhookdb::DBAdapter::ColumnTypes
+
+  def _webhook_response(_request)
+    # As of 9/15/21 there is no way to verify authenticity of these webhooks
+    return Webhookdb::WebhookResponse.ok
+  end
+
+  def _remote_key_column
+    return Webhookdb::Replicator::Column.new(:transistor_id, TEXT, data_key: "id")
+  end
+
+  def _timestamp_column_name
+    return :updated_at
+  end
+
+  def _resource_and_event(request)
+    body = request.body
+    return body["data"], body if body.key?("data")
+    return body, nil
+  end
+
+  def _update_where_expr
+    return self.qualified_table_sequel_identifier[:updated_at] < Sequel[:excluded][:updated_at]
+  end
+
+  def calculate_backfill_state_machine
+    step = Webhookdb::Replicator::StateMachineStep.new
+    if self.service_integration.backfill_key.blank?
+      step.output = %(Great! We've created your #{self.resource_name_plural} integration.
+
+Transistor does not support #{self.resource_name_singular} webhooks, so to fill your database,
+we need to use the API to make requests, which requires your API Key.
+
+From your Transistor dashboard, go to the "Your Account" page,
+at https://dashboard.transistor.fm/account
+On the left side of the bottom of the page you should be able to see your API key.
+
+Copy that API key.
+      )
+      return step.secret_prompt("API Key").backfill_key(self.service_integration)
+    end
+
+    unless (result = self.verify_backfill_credentials).verified
+      self.service_integration.replicator.clear_backfill_information
+      step.output = result.message
+      return step.secret_prompt("API Key").backfill_key(self.service_integration)
+    end
+
+    step.output = %(Great! We are going to start replicating your #{self.resource_name_plural}.
+#{self._query_help_output}
+    )
+    return step.completed
+  end
+
+  def _verify_backfill_401_err_msg
+    return "It looks like that API Key is invalid. Please reenter the API Key you just created:"
+  end
+
+  def _verify_backfill_err_msg
+    return "An error occurred. Please reenter the API Key you just created:"
+  end
+end

data/lib/webhookdb/replicator/twilio_sms_v1.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require "webhookdb/twilio"
+
+class Webhookdb::Replicator::TwilioSmsV1 < Webhookdb::Replicator::Base
+  include Appydays::Loggable
+
+  # @return [Webhookdb::Replicator::Descriptor]
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "twilio_sms_v1",
+      ctor: ->(sint) { Webhookdb::Replicator::TwilioSmsV1.new(sint) },
+      feature_roles: [],
+      resource_name_singular: "Twilio SMS Message",
+      supports_backfill: true,
+      api_docs_url: "https://www.twilio.com/docs/usage/api#send-an-sms-with-twilios-api",
+    )
+  end
+
+  def _webhook_response(request)
+    auth = request.get_header("Authorization")
+    if auth.nil? || !auth.match(/^Basic /)
+      return Webhookdb::WebhookResponse.new(
+        status: 401,
+        body: "",
+        reason: "challenge",
+        headers: {"Content-Type" => "text/plain", "WWW-Authenticate" => 'Basic realm="Webhookdb"'},
+      )
+    end
+    user_and_pass = Base64.decode64(auth.gsub(/^Basic /, ""))
+    if user_and_pass != self.service_integration.webhook_secret
+      return Webhookdb::WebhookResponse.new(
+        status: 401,
+        body: "",
+        reason: "invalid",
+        headers: {"Content-Type" => "text/plain"},
+      )
+    end
+    return Webhookdb::WebhookResponse.new(
+      status: 202,
+      headers: {"Content-Type" => "text/xml"},
+      body: "<Response></Response>",
+    )
+  end
+
+  def calculate_backfill_state_machine
+    step = Webhookdb::Replicator::StateMachineStep.new
+    unless self.service_integration.backfill_key.present?
+      step.needs_input = true
+      step.output = %(Great! We've created your Twilio SMS integration.
+
+Rather than using your Twilio Webhooks (of which each number can have only one),
+we poll Twilio for changes, and will also backfill historical SMS.
+
+To do this, we need your Account SID and Auth Token.
+Both of these values should be visible from the homepage of your Twilio admin Dashboard.
+      )
+      return step.secret_prompt("Account SID").backfill_key(self.service_integration)
+    end
+
+    unless self.service_integration.backfill_secret.present?
+      return step.secret_prompt("Auth Token").backfill_secret(self.service_integration)
+    end
+
+    unless (result = self.verify_backfill_credentials).verified
+      self.service_integration.replicator.clear_backfill_information
+      step.output = result.message
+      return step.secret_prompt("API Key").backfill_key(self.service_integration)
+    end
+
+    step.output = %(We are going to start replicating your Twilio SMS information, and will keep it updated.
+#{self._query_help_output}
+      )
+    return step.completed
+  end
+
+  def _verify_backfill_401_err_msg
+    return "It looks like that API Key is invalid. Please reenter the API Key you just created:"
+  end
+
+  def _verify_backfill_err_msg
+    return "An error occurred. Please reenter the API Key you just created:"
+  end
+
+  def _remote_key_column
+    return Webhookdb::Replicator::Column.new(:twilio_id, TEXT, data_key: "sid")
+  end
+
+  def _denormalized_columns
+    return [
+      Webhookdb::Replicator::Column.new(
+        :date_created,
+        TIMESTAMP,
+        index: true,
+        converter: Webhookdb::Replicator::Column::CONV_PARSE_TIME,
+      ),
+      Webhookdb::Replicator::Column.new(
+        :date_sent,
+        TIMESTAMP,
+        index: true,
+        converter: Webhookdb::Replicator::Column::CONV_PARSE_TIME,
+      ),
+      Webhookdb::Replicator::Column.new(
+        :date_updated,
+        TIMESTAMP,
+        index: true,
+        converter: Webhookdb::Replicator::Column::CONV_PARSE_TIME,
+      ),
+      Webhookdb::Replicator::Column.new(:direction, TEXT),
+      Webhookdb::Replicator::Column.new(:from, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(:status, TEXT),
+      Webhookdb::Replicator::Column.new(:to, TEXT, index: true),
+    ]
+  end
+
+  def _timestamp_column_name
+    return :date_updated
+  end
+
+  def _resource_and_event(request)
+    return request.body, nil
+  end
+
+  def _update_where_expr
+    return self.qualified_table_sequel_identifier[:date_updated] < Sequel[:excluded][:date_updated]
+  end
+
+  def _fetch_backfill_page(pagination_token, last_backfilled:)
+    url = "https://api.twilio.com"
+    if pagination_token.blank?
+      date_send_max = Date.tomorrow
+      url += "/2010-04-01/Accounts/#{self.service_integration.backfill_key}/Messages.json" \
+             "?PageSize=100&DateSend%3C=#{date_send_max}"
+    else
+      url += pagination_token
+    end
+    response = Webhookdb::Http.get(
+      url,
+      basic_auth: {username: self.service_integration.backfill_key,
+                   password: self.service_integration.backfill_secret,},
+      logger: self.logger,
+      timeout: Webhookdb::Twilio.http_timeout,
+    )
+    data = response.parsed_response
+    messages = data["messages"]
+
+    if last_backfilled.present?
+      earliest_data_created = messages.empty? ? Time.at(0) : messages[-1].fetch("date_created")
+      paged_to_already_seen_records = earliest_data_created < last_backfilled
+
+      return messages, nil if paged_to_already_seen_records
+    end
+
+    return messages, data["next_page_uri"]
+  end
+end

data/lib/webhookdb/replicator/webhook_request.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class Webhookdb::Replicator::WebhookRequest < Webhookdb::TypedStruct
+  attr_accessor :body, :headers, :path, :method
+end

data/lib/webhookdb/replicator/webhookdb_customer_v1.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+class Webhookdb::Replicator::WebhookdbCustomerV1 < Webhookdb::Replicator::Base
+  include Appydays::Loggable
+
+  # @return [Webhookdb::Replicator::Descriptor]
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "webhookdb_customer_v1",
+      ctor: ->(sint) { Webhookdb::Replicator::WebhookdbCustomerV1.new(sint) },
+      feature_roles: ["internal"],
+      resource_name_singular: "WebookDB Customer",
+      supports_webhooks: true,
+    )
+  end
+
+  def _webhook_response(request)
+    sek = request.env["HTTP_WHDB_SECRET"]
+    return Webhookdb::WebhookResponse.ok if sek == self.service_integration.webhook_secret
+    return Webhookdb::WebhookResponse.error("Whdb-Secret header is missing") if sek.nil?
+    return Webhookdb::WebhookResponse.error("Whdb-Secret value does not match configured secret")
+  end
+
+  def _timestamp_column_name
+    return :updated_at
+  end
+
+  def calculate_webhook_state_machine
+    step = Webhookdb::Replicator::StateMachineStep.new
+    self.service_integration.update(webhook_secret: Webhookdb::Id.rand_enc(16)) if
+      self.service_integration.webhook_secret.blank?
+    step.output = %(WebhookDB is now listening for changes to #{self.resource_name_plural}
+and will replicate them into the table for this service integration.
+
+Whenever a #{self.resource_name_singular} changes, a request will be sent to:
+
+  #{self._webhook_endpoint}
+
+With the header:
+
+  Whdb-Secret: #{self.service_integration.webhook_secret}
+
+Which will be received by this running instance, so there's nothing else you have to do.
+
+#{self._query_help_output}
+)
+    return step.completed
+  end
+
+  def _remote_key_column
+    return Webhookdb::Replicator::Column.new(:webhookdb_id, TEXT, data_key: "id")
+  end
+
+  def _denormalized_columns
+    return [
+      Webhookdb::Replicator::Column.new(:created_at, TIMESTAMP, index: true),
+      Webhookdb::Replicator::Column.new(:email, TEXT, index: true),
+      Webhookdb::Replicator::Column.new(
+        :updated_at,
+        TIMESTAMP,
+        index: true,
+        defaulter: Webhookdb::Replicator::Column.defaulter_from_resource_field(:created_at),
+      ),
+    ]
+  end
+
+  def _resource_and_event(request)
+    return request.body, nil
+  end
+
+  def _update_where_expr
+    return self.qualified_table_sequel_identifier[:updated_at] < Sequel[:excluded][:updated_at]
+  end
+end

data/lib/webhookdb/replicator.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+require "appydays/configurable"
+require "webhookdb/typed_struct"
+
+class Webhookdb::Replicator
+  include Appydays::Configurable
+  extend Webhookdb::MethodUtilities
+
+  configurable(:replicator) do
+    setting :always_process_synchronously, false
+  end
+
+  PLUGIN_DIRNAME = "replicator_ext"
+  PLUGIN_DIR = Pathname(__FILE__).dirname + PLUGIN_DIRNAME
+
+  # Raised when there is no service registered for a name.
+  class Invalid < StandardError; end
+
+  # Raised when credentials to interact with a service are not set up.
+  # Usually this is due to a missing dependency.
+  class CredentialsMissing < StandardError; end
+
+  # Statically describe a replicator.
+  class Descriptor < Webhookdb::TypedStruct
+    # @!attribute name
+    # Name of the replicator, like 'stripe_charge_v1'.
+    # Appears externally in many places, so must be meaningful.
+    #   @return [String]
+    attr_reader :name
+
+    # @!attribute ctor
+    # Method invoked with the +Webhookdb::ServiceIntegration+, and should return a new instance
+    # of the integration. Can also be an object that responds to `new`.
+    #   @return [Proc]
+    attr_reader :ctor
+
+    # @!attribute feature_roles
+    # Used for feature flagging functionality.
+    # Usually will be [], but other possible values are:
+    # -'internal' e.g. our fake integration
+    # -'unreleased' for works in progress
+    # -'beta' if we don't want most people to have access
+    #   @return [Array<String>]
+    attr_reader :feature_roles
+
+    # @!attribute resource_name_singular
+    # Name of the resource, like "Acme Sprocket"
+    #   @return [String]
+    attr_reader :resource_name_singular
+
+    # @!attribute resource_name_plural
+    # Defaults to resource_name_singular+s.
+    #   @return [String]
+    attr_reader :resource_name_plural
+
+    # @!attribute dependency_descriptor
+    # The descriptor for the service this one depends on (the parent).
+    #   @return [Webhookdb::Replicator::Descriptor]
+    attr_reader :dependency_descriptor
+
+    # True if this integration supports webhooks (real-time or user-built webhook payloads).
+    attr_reader :supports_webhooks
+    # True if this integration supports user-driven backfilling,
+    # usually by paginating all resources.
+    attr_reader :supports_backfill
+
+    # If this integration has specific documentation,
+    # link its url here. It is used to build custom error messages,
+    # for example in case 'backfill' is called but not supported.
+    attr_reader :documentation_url
+
+    # Markdown description of this replicator.
+    attr_reader :description
+
+    # URL pointing to the provider (not-WebhookDB) docs for this integration.
+    attr_reader :api_docs_url
+
+    # Is this an enterprise-only replicator?
+    attr_reader :enterprise
+
+    def initialize(
+      name:,
+      ctor:,
+      resource_name_singular:,
+      feature_roles:,
+      supports_webhooks: false,
+      supports_backfill: false,
+      resource_name_plural: nil,
+      dependency_descriptor: nil,
+      api_docs_url: "",
+      description: nil,
+      enterprise: false,
+      documentation_url: nil
+    )
+      raise ArgumentError, "must support one or both of webhooks and backfill" unless
+        supports_webhooks || supports_backfill
+      super(
+        name:,
+        resource_name_singular:,
+        feature_roles:,
+        supports_webhooks:,
+        supports_backfill:,
+        dependency_descriptor:,
+        documentation_url:,
+        api_docs_url:,
+        enterprise:
+      )
+      @ctor = ctor.is_a?(Class) ? ctor.method(:new) : ctor
+      @resource_name_plural = resource_name_plural || "#{self.resource_name_singular}s"
+      @description = description || "Replicate #{self.resource_name_plural} into your database."
+      self.feature_roles
+    end
+
+    def inspect
+      return "#{self.class.name}(name: #{self.name})"
+    end
+
+    def ==(other)
+      return self.class == other.class &&
+          self.name == other.name &&
+          self.resource_name_singular == other.resource_name_singular
+    end
+
+    alias supports_webhooks? supports_webhooks
+    alias supports_backfill? supports_backfill
+    alias enterprise? enterprise
+    def webhooks_only? = self.supports_webhooks? && !self.supports_backfill?
+    def backfill_only? = !self.supports_webhooks? && self.supports_backfill?
+    def supports_webhooks_and_backfill? = self.supports_webhooks? && self.supports_backfill?
+  end
+
+  class IndexSpec < Webhookdb::TypedStruct
+    attr_reader :columns, :where
+  end
+
+  class << self
+    # @return [Hash{String => Webhookdb::Replicator::Descriptor}]
+    def registry
+      return @registry ||= {}
+    end
+
+    def register(cls)
+      desc = cls.descriptor
+      raise TypeError, "descriptor must be a Descriptor, got #{desc.class.name}" unless desc.is_a?(Descriptor)
+      self.registry[desc.name] = desc
+    end
+
+    # Return a new replicator for the given integration.
+    #
+    # @param service_integration [Webhookdb::ServiceIntegration]
+    # @return [Webhookdb::Replicator::Base]
+    def create(service_integration)
+      name = service_integration.service_name
+      descr = self.registered!(name)
+      return descr.ctor.call(service_integration)
+    end
+
+    # Returns the service with the given name, or +nil+ if none is registered.
+    # @return [Webhookdb::Replicator::Descriptor]
+    def registered(name)
+      return @registry[name]
+    end
+
+    # @raise [Webhookdb::Replicator::Invalid] When the name is invalid.
+    # @return [Webhookdb::Replicator::Descriptor]
+    def registered!(name)
+      raise ArgumentError, "name cannot be blank" if name.blank?
+      r = self.registered(name)
+      return r if r
+      raise Invalid, name
+    end
+
+    def load_replicators
+      existing_descendants = Webhookdb::Replicator::Base.descendants
+      ["replicator", PLUGIN_DIRNAME].each do |d|
+        Gem.find_files(File.join("webhookdb/#{d}/*.rb")).each do |path|
+          next if path.include?("/spec/")
+          require path
+        end
+      end
+      new_descendants = Webhookdb::Replicator::Base.descendants
+      newly_registered = new_descendants - existing_descendants
+      newly_registered.each { |cls| self.register(cls) }
+      return newly_registered
+    end
+
+    def _require_files(dir)
+      splitter = "webhookdb/" + dir.to_s.rpartition("/").last
+      dir.glob("*.rb").each do |path|
+        base = path.basename.to_s[..-4]
+        require("#{splitter}/#{base}")
+      end
+    end
+
+    # @param sint [Webhookdb::ServiceIntegration]
+    # @return [Webhookdb::ServiceIntegration,nil]
+    def find_root(sint)
+      max_depth = 15
+      parent = sint.depends_on
+      return sint if parent.nil?
+      max_depth.times do
+        return parent if parent.depends_on.nil?
+        parent = parent.depends_on
+      end
+      return nil
+    end
+
+    # @param sint [Webhookdb::ServiceIntegration]
+    # @return [Webhookdb::ServiceIntegration]
+    def find_at_root!(sint, service_name:)
+      root = self.find_root(sint)
+      bad_auth = root&.service_name != service_name
+      raise self::CredentialsMissing, "Could not find root integration for #{sint.inspect}" if bad_auth
+      return root
+    end
+  end
+
+  require "webhookdb/replicator/state_machine_step"
+  require "webhookdb/replicator/column"
+  require "webhookdb/replicator/base"
+  require "webhookdb/replicator/docgen"
+  load_replicators
+end

data/lib/webhookdb/role.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "webhookdb/postgres/model"
+
+class Webhookdb::Role < Webhookdb::Postgres::Model(:roles)
+  # n.b. Because of the uniqueness constraint on "name", there is only one "admin" role. Its meaning
+  # depends on the context: if the customer has this role, they are an admin; if the org membership has
+  # this role, the customer is an org admin.
+  def self.admin_role
+    return Webhookdb.cached_get("role_admin") do
+      self.find_or_create_or_find(name: "admin")
+    end
+  end
+
+  def self.non_admin_role
+    return Webhookdb.cached_get("role_member") do
+      self.find_or_create_or_find(name: "member")
+    end
+  end
+
+  # used to indicate user status within the org, e.g. whether user is an org admin & can create services
+  one_to_many :organization_memberships, class: "Webhookdb::OrganizationMembership"
+
+  # used to indicate user status within the app itself, i.e. whether user is an app admin
+  many_to_many :customers,
+               class: "Webhookdb::Customer",
+               join_table: :roles_customers
+end
+
+# Table: roles
+# ---------------------------------------------------------------------------------------------------------------------------
+# Columns:
+#  id   | integer | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY
+#  name | text    | NOT NULL
+# Indexes:
+#  roles_pkey     | PRIMARY KEY btree (id)
+#  roles_name_key | UNIQUE btree (name)
+# Referenced By:
+#  feature_roles_organizations | feature_roles_organizations_role_id_fkey         | (role_id) REFERENCES roles(id)
+#  organization_memberships    | organization_memberships_membership_role_id_fkey | (membership_role_id) REFERENCES roles(id)
+#  roles_customers             | roles_customers_role_id_fkey                     | (role_id) REFERENCES roles(id)
+# ---------------------------------------------------------------------------------------------------------------------------

data/lib/webhookdb/sentry.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "sentry-ruby"
+require "appydays/configurable"
+require "appydays/loggable"
+
+require "webhookdb"
+
+module Webhookdb::Sentry
+  include Appydays::Configurable
+  include Appydays::Loggable
+
+  configurable(:sentry) do
+    setting :dsn, ""
+
+    # Apply the current configuration to Sentry.
+    # See https://docs.sentry.io/clients/ruby/config/ for more info.
+    after_configured do
+      if self.dsn
+        # See https://github.com/getsentry/sentry-ruby/issues/1756
+        require "sentry-sidekiq"
+        Sentry.init do |config|
+          config.dsn = dsn
+          config.logger = self.logger
+        end
+      else
+        Sentry.instance_variable_set(:@main_hub, nil)
+      end
+    end
+  end
+
+  def self.enabled?
+    return self.dsn.present?
+  end
+end