webhookdb 0.1.0

data/lib/webhookdb/messages/new_customer.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "webhookdb/message/template"
+
+class Webhookdb::Messages::NewCustomer < Webhookdb::Message::Template
+  def self.fixtured(recipient)
+    return self.new(recipient)
+  end
+
+  def initialize(customer)
+    @customer = customer
+    super()
+  end
+end

data/lib/webhookdb/messages/org_database_migration_finished.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "webhookdb/message/template"
+
+class Webhookdb::Messages::OrgDatabaseMigrationFinished < Webhookdb::Message::Template
+  def self.fixtured(recipient)
+    dbm = Webhookdb::Fixtures.organization_database_migration.with_urls.finished.create
+    Webhookdb::Fixtures.organization_membership.org(dbm.organization).customer(recipient).verified.admin.create
+    return self.new(dbm)
+  end
+
+  def initialize(org_database_migration)
+    @org_database_migration = org_database_migration
+    super()
+  end
+
+  def liquid_drops
+    return super.merge(
+      org_name: @org_database_migration.organization.name,
+      destination_host: @org_database_migration.displaysafe_destination_url,
+    )
+  end
+end

data/lib/webhookdb/messages/org_database_migration_started.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "webhookdb/message/template"
+
+class Webhookdb::Messages::OrgDatabaseMigrationStarted < Webhookdb::Message::Template
+  def self.fixtured(recipient)
+    dbm = Webhookdb::Fixtures.organization_database_migration.with_urls.started.create
+    Webhookdb::Fixtures.organization_membership.org(dbm.organization).customer(recipient).verified.admin.create
+    return self.new(dbm)
+  end
+
+  def initialize(org_database_migration)
+    @org_database_migration = org_database_migration
+    super()
+  end
+
+  def liquid_drops
+    return super.merge(
+      org_name: @org_database_migration.organization.name,
+      source_host: @org_database_migration.displaysafe_source_url,
+      destination_host: @org_database_migration.displaysafe_destination_url,
+    )
+  end
+end

data/lib/webhookdb/messages/specs.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "webhookdb/message/template"
+
+module Webhookdb::Messages::Testers
+  class Base < Webhookdb::Message::Template
+    def template_folder
+      return "specs"
+    end
+
+    def layout
+      return nil
+    end
+  end
+
+  class Basic < Base
+  end
+
+  class WithField < Base
+    def initialize(field)
+      @field = field
+      super()
+    end
+
+    def liquid_drops
+      return super.merge(field: @field)
+    end
+  end
+
+  class Nonextant < Base
+  end
+
+  class MissingField < Base
+    def template_name
+      return "with_field"
+    end
+  end
+
+  class WithInclude < Base
+    def liquid_drops
+      return super.merge(field: 3)
+    end
+  end
+
+  class WithPartial < Base
+  end
+
+  class WithLayout < Base
+    def template_name
+      return "basic"
+    end
+
+    def layout
+      return "standard"
+    end
+  end
+end

data/lib/webhookdb/messages/verification.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "webhookdb/message/template"
+
+class Webhookdb::Messages::Verification < Webhookdb::Message::Template
+  def self.fixtured(recipient)
+    code = Webhookdb::Fixtures.reset_code(customer: recipient).create
+    return self.new(code)
+  end
+
+  def initialize(reset_code)
+    @reset_code = reset_code
+    super()
+  end
+
+  def liquid_drops
+    return super.merge(
+      expire_at: @reset_code.expire_at,
+      token: @reset_code.token,
+      email: @reset_code.customer.email,
+    )
+  end
+end

data/lib/webhookdb/method_utilities.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "webhookdb" unless defined?(Webhookdb)
+
+# A collection of methods for declaring other methods.
+#
+#   class MyClass
+#       extend Webhookdb::MethodUtilities
+#
+#       singleton_attr_accessor :types
+#       singleton_method_alias :kinds, :types
+#   end
+#
+#   MyClass.types = [ :pheno, :proto, :stereo ]
+#   MyClass.kinds # => [:pheno, :proto, :stereo]
+#
+module Webhookdb::MethodUtilities
+  ### Creates instance variables and corresponding methods that return their
+  ### values for each of the specified +symbols+ in the singleton of the
+  ### declaring object (e.g., class instance variables and methods if declared
+  ### in a Class).
+  def singleton_attr_reader(*symbols)
+    singleton_class.instance_exec(symbols) do |attrs|
+      attr_reader(*attrs)
+    end
+  end
+
+  ### Create instance variables and corresponding methods that return
+  ### true or false values for each of the specified +symbols+ in the singleton
+  ### of the declaring object.
+  def singleton_predicate_reader(*symbols)
+    singleton_class.extend(Webhookdb::MethodUtilities)
+    singleton_class.attr_predicate(*symbols)
+  end
+
+  ### Creates methods that allow assignment to the attributes of the singleton
+  ### of the declaring object that correspond to the specified +symbols+.
+  def singleton_attr_writer(*symbols)
+    singleton_class.instance_exec(symbols) do |attrs|
+      attr_writer(*attrs)
+    end
+  end
+
+  ### Creates readers and writers that allow assignment to the attributes of
+  ### the singleton of the declaring object that correspond to the specified
+  ### +symbols+.
+  def singleton_attr_accessor(*symbols)
+    symbols.each do |sym|
+      singleton_class.__send__(:attr_accessor, sym)
+    end
+  end
+
+  ### Create predicate methods and writers that allow assignment to the attributes
+  ### of the singleton of the declaring object that correspond to the specified
+  ### +symbols+.
+  def singleton_predicate_accessor(*symbols)
+    singleton_class.extend(Webhookdb::MethodUtilities)
+    singleton_class.attr_predicate_accessor(*symbols)
+  end
+
+  ### Creates an alias for the +original+ method named +newname+.
+  def singleton_method_alias(newname, original)
+    singleton_class.__send__(:alias_method, newname, original)
+  end
+
+  ### Create a reader in the form of a predicate for the given +attrname+.
+  def attr_predicate(attrname)
+    attrname = attrname.to_s.chomp("?")
+    define_method(:"#{attrname}?") do
+      instance_variable_get(:"@#{attrname}") ? true : false
+    end
+  end
+
+  ### Create a reader in the form of a predicate for the given +attrname+
+  ### as well as a regular writer method.
+  def attr_predicate_accessor(attrname)
+    attrname = attrname.to_s.chomp("?")
+    attr_writer(attrname)
+
+    attr_predicate(attrname)
+  end
+end

data/lib/webhookdb/microsoft_calendar.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "appydays/configurable"
+
+module Webhookdb::MicrosoftCalendar
+  include Appydays::Configurable
+  extend Webhookdb::MethodUtilities
+
+  singleton_attr_accessor :calendar_view_start_time
+  singleton_attr_accessor :calendar_view_end_time
+
+  configurable(:microsoft_calendar) do
+    # How many calendars/events should we fetch in a single page?
+    # Higher uses slightly more memory but fewer API calls.
+    # Apparent maximum is 999,999,999.
+    setting :list_page_size, 500
+    # How many rows should we upsert at a time?
+    # Higher is fewer upserts, but can create very large SQL strings,
+    # which can have negative performance.
+    setting :upsert_page_size, 500
+
+    # These should be ISO8601 strings. We use them in our calls to the Microsoft Graph API
+    # to determine the timeframe we are pulling events from.
+    setting :calendar_view_start, Time.new(2022, 10, 1).iso8601
+    setting :calendar_view_end, (Time.new(2022, 10, 1) + 1825.days).iso8601
+
+    setting :http_timeout, 30
+
+    after_configured do
+      self.calendar_view_start_time = Time.parse(self.calendar_view_start)
+      self.calendar_view_end_time = Time.parse(self.calendar_view_end)
+      raise ArgumentError, "calendar view end must be after calendar view start" if
+        self.calendar_view_start_time >= self.calendar_view_end_time
+    end
+  end
+end

data/lib/webhookdb/nextpax.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "appydays/configurable"
+
+module Webhookdb::Nextpax
+  include Appydays::Configurable
+
+  configurable(:nextpax) do
+    setting :constants_sync_cron_expression, "0 */12 * * *"
+    setting :property_changes_cron_expression, "*/1 * * * *"
+    setting :http_timeout, 30
+    setting :page_size, 20
+  end
+end

data/lib/webhookdb/oauth/front.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "webhookdb/front"
+
+class Webhookdb::Oauth::Front < Webhookdb::Oauth::Provider
+  include Appydays::Loggable
+
+  def key = "front"
+  def app_name = "Front"
+  def requires_webhookdb_auth? = true
+  def supports_webhooks? = true
+
+  def authorization_url(state:)
+    return "https://app.frontapp.com/oauth/authorize?response_type=code&redirect_uri=#{Webhookdb::Front.oauth_callback_url}&state=#{state}&client_id=#{Webhookdb::Front.client_id}"
+  end
+
+  def exchange_authorization_code(code:)
+    token = Webhookdb::Http.post(
+      "https://app.frontapp.com/oauth/token",
+      {
+        "code" => code,
+        "redirect_uri" => Webhookdb::Front.oauth_callback_url,
+        "grant_type" => "authorization_code",
+      },
+      logger: self.logger,
+      timeout: Webhookdb::Front.http_timeout,
+      basic_auth: {username: Webhookdb::Front.client_id, password: Webhookdb::Front.client_secret},
+    )
+    return Webhookdb::Oauth::Tokens.new(
+      access_token: token.parsed_response["access_token"],
+      refresh_token: token.parsed_response["refresh_token"],
+    )
+  end
+
+  def build_marketplace_integrations(organization:, tokens:, **)
+    # I asked the dev team at front specifically how to differentiate between instances when receiving webhooks,
+    # and they said to look at the root url of the link provided for the resource in every response. In order to
+    # retrieve that value for the integrations that we'll be finding or creating, we look at this token info
+    # response.
+    front_token_info_resp = Webhookdb::Http.get(
+      "https://api2.frontapp.com/me",
+      headers: Webhookdb::Front.auth_headers(tokens.access_token),
+      logger: self.logger,
+      timeout: Webhookdb::Front.http_timeout,
+    )
+    front_token_info = front_token_info_resp.parsed_response
+    resource_url = front_token_info.dig("_links", "self")
+    instance_root_url = resource_url.nil? ? nil : URI.parse(resource_url).host
+
+    root_sint = Webhookdb::ServiceIntegration.create_disambiguated(
+      "front_marketplace_root_v1",
+      organization:,
+      api_url: instance_root_url,
+      backfill_key: tokens.refresh_token,
+    )
+    root_sint.replicator.build_dependents
+  end
+end

data/lib/webhookdb/oauth/intercom.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "webhookdb/intercom"
+
+class Webhookdb::Oauth::Intercom < Webhookdb::Oauth::Provider
+  include Appydays::Loggable
+
+  def key = "intercom"
+  def app_name = "Intercom"
+  def requires_webhookdb_auth? = false
+  def supports_webhooks? = false
+
+  def authorization_url(state:)
+    return "https://app.intercom.com/oauth?client_id=#{Webhookdb::Intercom.client_id}&state=#{state}"
+  end
+
+  def exchange_authorization_code(code:)
+    token_resp = Webhookdb::Http.post(
+      "https://api.intercom.io/auth/eagle/token",
+      {
+        "client_id" => Webhookdb::Intercom.client_id,
+        "client_secret" => Webhookdb::Intercom.client_secret,
+        "code" => code,
+      },
+      logger: self.logger,
+      timeout: Webhookdb::Intercom.http_timeout,
+    )
+    return Webhookdb::Oauth::Tokens.new(access_token: token_resp.parsed_response["token"])
+  end
+
+  def find_or_create_customer(tokens:, scope:)
+    intercom_user_resp = Webhookdb::Http.get(
+      "https://api.intercom.io/me",
+      headers: Webhookdb::Intercom.auth_headers(tokens.access_token),
+      logger: self.logger,
+      timeout: Webhookdb::Intercom.http_timeout,
+    )
+
+    intercom_user = intercom_user_resp.parsed_response
+    scope[:me_response] = intercom_user
+    intercom_email = intercom_user.fetch("email")
+    return Webhookdb::Customer.find_or_create_for_email(intercom_email)
+  end
+
+  def build_marketplace_integrations(organization:, tokens:, scope:)
+    intercom_user = scope.fetch(:me_response)
+    # The intercom workspace id is used in the intercom webhook endpoint to identify which
+    # service integration to delegate requests to.
+    intercom_workspace_id = intercom_user.dig("app", "id_code")
+    root_sint = Webhookdb::ServiceIntegration.create_disambiguated(
+      "intercom_marketplace_root_v1",
+      organization:,
+      api_url: intercom_workspace_id,
+      backfill_key: tokens.access_token,
+    )
+    root_sint.replicator.build_dependents
+  end
+end

data/lib/webhookdb/oauth/session.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "webhookdb/postgres"
+require "webhookdb/oauth"
+
+class Webhookdb::Oauth::Session < Webhookdb::Postgres::Model(:oauth_sessions)
+  plugin :timestamps
+
+  many_to_one :customer, class: "Webhookdb::Customer"
+  many_to_one :organization, class: "Webhookdb::Organization"
+
+  dataset_module do
+    def usable
+      return self.where(used_at: nil).where { created_at > 30.minutes.ago }
+    end
+  end
+
+  def self.params_for_request(request)
+    return {
+      peer_ip: request.ip,
+      user_agent: request.user_agent || "(unset)",
+    }
+  end
+end

data/lib/webhookdb/oauth.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Webhookdb::Oauth
+  class Tokens < Webhookdb::TypedStruct
+    attr_reader :access_token, :refresh_token
+
+    def initialize(**kwargs)
+      super
+      self.typecheck!(:access_token, String)
+      self.typecheck!(:refresh_token, String, nullable: true)
+    end
+  end
+
+  # rubocop:disable Lint/UnusedMethodArgument
+  class Provider
+    # @return [String] Unique key to identify the provider.
+    def key = raise NotImplementedError
+
+    # @return [String] Name of the app to present to users.
+    def app_name = raise NotImplementedError
+
+    # True if auth with this provider requires the user auth in WebhookDB,
+    # false if we can get their email from the Oauth process.
+    # If the access token can be used to get the 'me' user,
+    # we can usually use their email for the customer,
+    # but this may not be possible for some integrations.
+    def requires_webhookdb_auth? = raise NotImplementedError
+
+    # This is similar to `supports_webhooks` in the Replicator descriptors,
+    # except that this is used to make the success page dynamic.
+    # True if this provider's integrations support webhooks
+    # (real-time or user-built webhook payloads).
+    def supports_webhooks? = raise NotImplementedError
+
+    # @return [String] The Oauth URL to send users to to begin OAuth.
+    def authorization_url(state:) = raise NotImplementedError
+
+    # Exchange the access code (from the authorization url) for access and/or refresh tokens.
+    # @return [Webhookdb::Oauth::Tokens]
+    def exchange_authorization_code(code:) = raise NotImplementedError
+
+    # @param tokens [Webhookdb::Oauth::Tokens]
+    # @param scope [Hash] Used to store data needed in later calls, like when building integrations.
+    # @return [Array{TrueClass, FalseClass, Webhookdb::Customer}]
+    def find_or_create_customer(tokens:, scope:)
+      raise RuntimeError("should not be called") if self.requires_webhookdb_auth?
+      raise NotImplementedError
+    end
+
+    # Create the actual service integrations for the given org.
+    # @param organization [Webhookdb::Organization]
+    # @param tokens [Webhookdb::Oauth::Tokens]
+    # # @param scope [Hash]
+    def build_marketplace_integrations(organization:, tokens:, scope:) = raise NotImplementedError
+  end
+  # rubocop:enable Lint/UnusedMethodArgument
+
+  class << self
+    # @return [String, Class]
+    def register(key, cls)
+      raise "#{key} already registered to #{cls}" if self.registry.include?(key)
+      self.registry[key] = cls
+    end
+
+    # @return [Provider]
+    def provider(key)
+      return self.registry.fetch(key).new
+    end
+
+    # @return [Hash]
+    def registry
+      return @registry ||= {}
+    end
+  end
+end
+
+require "webhookdb/oauth/front"
+Webhookdb::Oauth.register(Webhookdb::Oauth::Front.new.key, Webhookdb::Oauth::Front)
+require "webhookdb/oauth/intercom"
+Webhookdb::Oauth.register(Webhookdb::Oauth::Intercom.new.key, Webhookdb::Oauth::Intercom)

data/lib/webhookdb/organization/alerting.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "appydays/configurable"
+require "appydays/loggable"
+
+class Webhookdb::Organization::Alerting
+  include Appydays::Configurable
+
+  configurable(:alerting) do
+    setting :interval, 24.hours.to_i
+  end
+
+  attr_reader :org
+
+  def initialize(org)
+    @org = org
+  end
+
+  # Dispatch the message template to administrators of the org.
+  # @param message_template [Webhookdb::Message::Template]
+  def dispatch_alert(message_template)
+    unless message_template.respond_to?(:signature)
+      raise Webhookdb::InvalidPrecondition,
+            "message template #{message_template.template_name} must define a #signature method, " \
+            "which is a unique identity for this error type, used for grouping and idempotency"
+    end
+    signature = message_template.signature
+    self.org.admin_customers.each do |c|
+      idemkey = "orgalert-#{signature}-#{c.id}"
+      Webhookdb::Idempotency.every(Webhookdb::Organization::Alerting.interval).under_key(idemkey) do
+        message_template.dispatch_email(c)
+      end
+    end
+  end
+end

data/lib/webhookdb/organization/database_migration.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+class Webhookdb::Organization::DatabaseMigration < Webhookdb::Postgres::Model(:organization_database_migrations)
+  include Webhookdb::Dbutil
+
+  class MigrationInProgress < Webhookdb::DatabaseLocked; end
+  class MigrationAlreadyFinished < StandardError; end
+
+  plugin :timestamps
+  plugin :column_encryption do |enc|
+    enc.column :source_admin_connection_url
+    enc.column :destination_admin_connection_url
+  end
+
+  many_to_one :started_by, class: "Webhookdb::Customer"
+  many_to_one :organization, class: "Webhookdb::Organization"
+
+  dataset_module do
+    def ongoing
+      return self.where(finished_at: nil)
+    end
+  end
+
+  def self.guard_ongoing!(org)
+    dbm = self.where(organization: org).ongoing.first
+    return if dbm.nil?
+    raise MigrationInProgress, "Organization #{org.name} has an ongoing database host migration so " \
+                               "cannot be modified. We'll let admins know when it's done. Try again soon."
+  end
+
+  def self.enqueue(admin_connection_url_raw:, readonly_connection_url_raw:, public_host:, started_by:, organization:)
+    self.guard_ongoing!(organization)
+    self.db.transaction do
+      dbm = self.create(
+        started_by:,
+        organization:,
+        organization_schema: organization.replication_schema,
+        source_admin_connection_url: organization.admin_connection_url_raw,
+        destination_admin_connection_url: admin_connection_url_raw,
+      )
+      organization.update(
+        public_host:,
+        admin_connection_url_raw:,
+        readonly_connection_url_raw:,
+      )
+      return dbm
+    end
+  end
+
+  def displaysafe_source_url
+    return displaysafe_url(self.source_admin_connection_url)
+  end
+
+  def displaysafe_destination_url
+    return displaysafe_url(self.destination_admin_connection_url)
+  end
+
+  def status
+    return "finished" if self.finished_at.present?
+    return "in_progress" if self.started_at.present?
+    return "enqueued"
+  end
+
+  def finished?
+    return !!self.finished_at
+  end
+
+  def migrate
+    raise MigrationAlreadyFinished if self.finished?
+    self.update(started_at: Time.now) if self.started_at.nil?
+    borrow_conn(self.source_admin_connection_url) do |srcdb|
+      borrow_conn(self.destination_admin_connection_url) do |dstdb|
+        self.organization.service_integrations.sort_by(&:id).each do |sint|
+          next if sint.id <= self.last_migrated_service_integration_id
+          self.migrate_service_integration(sint, srcdb, dstdb)
+          self.update(last_migrated_service_integration_id: sint.id, last_migrated_timestamp: nil)
+        end
+      end
+    end
+    self.update(finished_at: Time.now)
+  end
+
+  # @param [Webhookdb::ServiceIntegration] service_integration
+  protected def migrate_service_integration(service_integration, srcdb, dstdb)
+    svc = service_integration.replicator
+    # If the service integration was not synced in the old db, skip it
+    return unless srcdb.table_exists?(svc.qualified_table_sequel_identifier)
+    svc.create_table_modification(if_not_exists: true).execute(dstdb)
+    ds = srcdb[svc.qualified_table_sequel_identifier].order(svc.timestamp_column.name)
+    (ds = ds.where(Sequel[svc.timestamp_column.name] > self.last_migrated_timestamp)) unless
+      self.last_migrated_timestamp.nil?
+    chunksize = Webhookdb::Organization.database_migration_page_size
+    chunk = []
+    ds.paged_each(rows_per_fetch: chunksize, hold: true, cursor_name: "whdb_dbmigration_#{self.id}") do |row|
+      chunk << row
+      if chunk.size >= chunksize
+        self.upsert_chunk(service_integration, dstdb, chunk)
+        chunk.clear
+        Amigo::DurableJob.heartbeat
+      end
+    end
+    self.upsert_chunk(service_integration, dstdb, chunk)
+  end
+
+  # @param [Webhookdb::ServiceIntegration] service_integration
+  protected def upsert_chunk(service_integration, dstdb, chunk)
+    return if chunk.empty?
+    svc = service_integration.replicator
+    chunk.each { |h| h.delete(svc.primary_key_column.name) }
+    tscol = svc.timestamp_column.name
+    dstdb[svc.qualified_table_sequel_identifier].
+      insert_conflict(
+        target: svc.remote_key_column.name,
+        update_where: svc._update_where_expr,
+      ).multi_insert(chunk)
+    self.update(last_migrated_timestamp: chunk.last[tscol])
+  end
+
+  def finish(now: Time.now)
+    self.update(
+      finished_at: now,
+      source_admin_connection_url: displaysafe_source_url,
+      destination_admin_connection_url: displaysafe_destination_url,
+    )
+    return self
+  end
+end
+
+# Table: organization_database_migrations
+# -------------------------------------------------------------------------------------------------------------------------
+# Columns:
+#  id                                   | integer                  | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY
+#  created_at                           | timestamp with time zone | NOT NULL DEFAULT now()
+#  updated_at                           | timestamp with time zone |
+#  started_at                           | timestamp with time zone |
+#  finished_at                          | timestamp with time zone |
+#  organization_id                      | integer                  | NOT NULL
+#  started_by_id                        | integer                  |
+#  source_admin_connection_url          | text                     |
+#  destination_admin_connection_url     | text                     |
+#  organization_schema                  | text                     |
+#  last_migrated_service_integration_id | integer                  | NOT NULL DEFAULT 0
+#  last_migrated_timestamp              | timestamp with time zone |
+# Indexes:
+#  organization_database_migrations_pkey                  | PRIMARY KEY btree (id)
+#  one_inprogress_migration_per_org                       | UNIQUE btree (organization_id) WHERE finished_at IS NULL
+#  organization_database_migrations_organization_id_index | btree (organization_id)
+# Foreign key constraints:
+#  organization_database_migrations_organization_id_fkey | (organization_id) REFERENCES organizations(id) ON DELETE CASCADE
+#  organization_database_migrations_started_by_id_fkey   | (started_by_id) REFERENCES customers(id) ON DELETE SET NULL
+# -------------------------------------------------------------------------------------------------------------------------