webhookdb 0.1.0

data/lib/webhookdb/api/sync_targets.rb

@@ -0,0 +1,232 @@
+# frozen_string_literal: true
+
+require "webhookdb/api"
+require "webhookdb/jobs/sync_target_run_sync"
+
+# rubocop:disable Layout/LineLength
+class Webhookdb::API::SyncTargets < Webhookdb::API::V1
+  class ConnectionUrlType < Grape::Validations::Validators::Base
+    def validate!(params)
+      url = params[:connection_url]
+      case @option
+        when "db", "http"
+          if (err = Webhookdb::SyncTarget.send(:"validate_#{@option}_url", url))
+            raise Grape::Exceptions::Validation.new params: [url], message: err
+          end
+      else
+          raise Grape::Exceptions::Validation.new params: [url], message: "#{@option} is not a valid connection url type"
+      end
+    end
+  end
+
+  resource :organizations do
+    route_param :org_identifier, type: String do
+      resource :sync_targets do
+        [:db, :http].each do |target_type_resource|
+          resource target_type_resource do
+            route_setting :target_type, {key: target_type_resource}
+            helpers do
+              is_db = target_type_resource == :db
+              def target_type
+                tgttype = route.settings[:target_type]
+                # Doesn't appear that this cascades, so we need to specify on every verb method.
+                raise "add `route_setting :target_type, target_type_resource` before verb" unless tgttype
+                return tgttype
+              end
+
+              def db? = target_type == :db
+
+              default_period = Webhookdb::SyncTarget.default_valid_period
+              params :sync_target_params do
+                optional :period_seconds,
+                         type: Integer,
+                         values: Webhookdb::SyncTarget.valid_period(1),
+                         prompt: {
+                           message: "How many seconds between syncs (#{default_period.begin} to #{default_period.end}):",
+                           disable: ->(req) { req.path.end_with?("/update") },
+                         }
+                is_db && optional(:schema,
+                                  type: String,
+                                  db_identifier: true,
+                                  allow_blank: true,
+                                  desc: "Schema (or namespace) to write the table into. Default to no schema/namespace.",)
+                # The description here says there is a default value, but the default value isn't actually saved to the SyncTarget
+                # object--it's inferred in the SyncTarget sync behavior when the table value is a blank string.
+                is_db && optional(:table,
+                                  type: String,
+                                  db_identifier: true,
+                                  allow_blank: true,
+                                  desc: "Table to create and update. Default to match the table name of the service integration.",)
+                !is_db && optional(:page_size,
+                                   type: Integer,
+                                   values: (1..1000),
+                                   desc: "Maximum number of rows in each POST request.",)
+              end
+              params :connection_url do
+                optional :connection_url,
+                         type: String,
+                         prompt: "Enter the #{is_db ? 'database connection string' : 'HTTP endpoint'} that WebhookDB should sync data to:",
+                         connection_url_type: is_db ? "db" : "http"
+              end
+
+              def entity
+                return db? ? Webhookdb::API::DbSyncTargetEntity : Webhookdb::API::HttpSyncTargetEntity
+              end
+
+              def validate_period!(org, value)
+                r = Webhookdb::SyncTarget.valid_period_for(org)
+                return if r.cover?(value)
+                err = "The valid sync period for organization #{org.name} is between #{r.begin} and #{r.end} seconds."
+                invalid!(err, message: err)
+              end
+
+              def verify_connection(url)
+                if db?
+                  Webhookdb::SyncTarget.verify_db_connection(url)
+                else
+                  Webhookdb::SyncTarget.verify_http_connection(url)
+                end
+              rescue Webhookdb::SyncTarget::InvalidConnection => e
+                merror!(400, e.to_s, code: "invalid_sync_target_connection")
+              end
+
+              def predicate = db? ? :db? : :http?
+              def fullname = db? ? "database" : "http"
+              def cmd = db? ? "dbsync" : "httpsync"
+              def tgtname = db? ? "database" : "endpoint"
+            end
+
+            route_setting :target_type, target_type_resource
+            get do
+              org = lookup_org!
+              subs = org.all_sync_targets.filter(&predicate)
+              message = ""
+              if subs.empty?
+                message = "Organization #{org.name} has no #{fullname} sync targets set up.\n" \
+                          "Use `webhookdb #{cmd} create` to synchronization WebhookDB tables to your #{tgtname}."
+              end
+              status 200
+              present_collection subs, with: entity, message:
+            end
+
+            params do
+              use :connection_url
+              use :sync_target_params
+              optional :service_integration_opaque_id,
+                       type: String, allow_blank: false,
+                       desc: "This is a deprecated parameter. In the future, please use `service_integration_identifier`."
+              optional :service_integration_identifier, type: String, allow_blank: false
+              at_least_one_of :service_integration_opaque_id, :service_integration_identifier
+            end
+            route_setting :target_type, target_type_resource
+            post :create do
+              customer = current_customer
+              org = lookup_org!(customer:)
+              ensure_admin!(org, customer:)
+              identifier = params[:service_integration_identifier] || params[:service_integration_opaque_id]
+              sint = lookup_service_integration!(org, identifier)
+
+              validate_period!(sint.organization, params[:period_seconds])
+              verify_connection(params[:connection_url])
+              stgt = Webhookdb::SyncTarget.create(
+                service_integration: sint,
+                connection_url: params[:connection_url],
+                period_seconds: params[:period_seconds],
+                schema: params[:schema] || "",
+                table: params[:table] || "",
+                page_size: params[:page_size],
+                created_by: customer,
+              )
+              message = "Every #{stgt.period_seconds} seconds, data from #{sint.service_name} " \
+                        "in #{sint.table_name} will be synchronized to #{stgt.displaysafe_connection_url}"
+              status 200
+              present stgt, with: entity, message:
+            rescue Grape::Exceptions::Validation => e
+              merror!(400, e.message)
+            end
+
+            route_param :opaque_id do
+              helpers do
+                def lookup!
+                  org = lookup_org!
+                  ensure_admin!(org)
+                  (stgt = org.all_sync_targets_dataset[Sequel[:sync_targets][:opaque_id] => params[:opaque_id]])
+                  merror!(403, "There is no #{fullname} sync target with that id.") if
+                    stgt.nil? || !stgt.send(predicate)
+                  return stgt
+                end
+              end
+              params do
+                optional :user, type: String, prompt: "Username for the connection:"
+                optional :password, type: String, prompt: "Password for the connection:"
+              end
+              route_setting :target_type, target_type_resource
+              post :update_credentials do
+                stgt = lookup!
+                uri = URI(stgt.connection_url)
+                uri.user = params[:user]
+                uri.password = params[:password]
+                verify_connection(uri.to_s)
+                stgt.update(connection_url: uri.to_s)
+                status 200
+                present stgt, with: entity, message: "Connection URL has been updated."
+              end
+
+              params do
+                use :sync_target_params
+              end
+              route_setting :target_type, target_type_resource
+              post :update do
+                stgt = lookup!
+                stgt.table = params[:table] if params.key?(:table)
+                stgt.schema = params[:schema] if params.key?(:schema)
+                stgt.page_size = params[:page_size] if params.key?(:page_size)
+                if params.key?(:period_seconds) && (period_seconds = params[:period_seconds])
+                  validate_period!(stgt.organization, period_seconds)
+                  stgt.period_seconds = period_seconds
+                end
+                save_or_error!(stgt)
+                status 200
+                present stgt, with: entity, message: "#{fullname.capitalize} sync target has been updated."
+              end
+
+              params do
+                optional :confirm, type: String
+              end
+              route_setting :target_type, target_type_resource
+              post :delete do
+                stgt = lookup!
+                prompted_table = stgt.service_integration.table_name
+                if prompted_table != params[:confirm]&.strip
+                  Webhookdb::API::Helpers.prompt_for_required_param!(
+                    request,
+                    :confirm,
+                    "Please confirm your deletion by entering the table name that is being synced by " \
+                    "this #{fullname} sync target (#{prompted_table}):",
+                  )
+                end
+
+                stgt.logger.warn("destroying_sync_target", sync_target_id: stgt.id, customer_id: current_customer.id)
+                stgt.destroy
+                status 200
+                message = "#{fullname.capitalize} sync target has been removed and will no longer sync."
+                present stgt, with: entity, message:
+              end
+
+              route_setting :target_type, target_type_resource
+              post :sync do
+                stgt = lookup!
+                status 200
+                next_sync = stgt.next_possible_sync(now: Time.now)
+                Webhookdb::Jobs::SyncTargetRunSync.perform_at(next_sync, stgt.id)
+                message = "#{fullname.capitalize} sync has been scheduled. It should start at about #{next_sync}."
+                present stgt, with: entity, message:
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
+# rubocop:enable Layout/LineLength

data/lib/webhookdb/api/system.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "grape"
+
+require "webhookdb/api"
+require "webhookdb/postgres"
+
+# Health check and other metadata endpoints.
+class Webhookdb::API::System < Webhookdb::Service
+  format :json
+
+  require "webhookdb/service/helpers"
+  helpers Webhookdb::Service::Helpers
+
+  get :healthz do
+    Webhookdb::Postgres::Model.db.execute("SELECT 1=1")
+    status 200
+    {o: "k"}
+  end
+
+  get :statusz do
+    status 200
+    {
+      env: Webhookdb::RACK_ENV,
+      version: Webhookdb::VERSION,
+      commit: Webhookdb::COMMIT,
+      release: Webhookdb::RELEASE,
+      log_level: Webhookdb.logger.level,
+    }
+  end
+
+  resource :debug do
+    get :echo do
+      pp params.to_h
+    end
+  end
+end

data/lib/webhookdb/api/webhook_subscriptions.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require "webhookdb/api"
+
+class Webhookdb::API::WebhookSubscriptions < Webhookdb::API::V1
+  resource :organizations do
+    route_param :org_identifier, type: String do
+      resource :webhook_subscriptions do
+        desc "Return all webhook subscriptions for the given org, and all integrations."
+        get do
+          org = lookup_org!
+          subs = org.all_webhook_subscriptions
+          message = ""
+          if subs.empty?
+            message = "Organization #{org.name} has no webhook subscriptions set up.\n" \
+                      "Use `webhookdb webhooks create` to set one up."
+          end
+          status 200
+          present_collection subs, with: Webhookdb::API::WebhookSubscriptionEntity, message:
+        end
+
+        params do
+          optional :url, prompt: "Enter the URL that WebhookDB should POST webhooks to:"
+          optional :webhook_secret, prompt: "Enter a random secret used to sign and verify webhooks to the given url:"
+          optional :service_integration_identifier,
+                   type: String,
+                   desc: "If provided, attach the webhook subscription to this integration rather than the org."
+          optional :service_integration_opaque_id,
+                   type: String,
+                   desc: "This is a deprecated parameter. In the future, please use `service_integration_identifier`."
+        end
+        post :create do
+          org = lookup_org!
+          sint = nil
+          identifier = params[:service_integration_identifier] || params[:service_integration_opaque_id]
+          sint = lookup_service_integration!(org, identifier) if identifier.present?
+          webhook_sub = Webhookdb::WebhookSubscription.create(
+            webhook_secret: params[:webhook_secret],
+            deliver_to_url: params[:url],
+            organization: sint ? nil : org,
+            service_integration: sint,
+            created_by: current_customer,
+          )
+          message = if sint
+                      "All webhooks for this #{sint.service_name} integration will be sent to #{params[:url]}"
+          else
+            "All webhooks for all integrations belonging to organization #{org.name} will be sent to #{params[:url]}."
+          end
+          status 200
+          present webhook_sub, with: Webhookdb::API::WebhookSubscriptionEntity, message:
+        end
+
+        route_param :opaque_id, type: String do
+          helpers do
+            def lookup_sub!
+              org = lookup_org!
+              whsub = org.all_webhook_subscriptions_dataset[opaque_id: params[:opaque_id]]
+              merror!(403, "No webhook subscription with that ID exists in that organization.") if whsub.nil?
+              return whsub
+            end
+          end
+
+          post :test do
+            webhook_sub = lookup_sub!
+            webhook_sub.publish_immediate("test", webhook_sub.id)
+            message = "A test event has been sent to #{webhook_sub.deliver_to_url}."
+            status 200
+            present({}, with: Webhookdb::API::BaseEntity, message:)
+          end
+
+          post :delete do
+            webhook_sub = lookup_sub!
+            webhook_sub.delete
+            message = "Events will no longer be sent to #{webhook_sub.deliver_to_url}."
+            status 200
+            present({}, with: Webhookdb::API::BaseEntity, message:)
+          end
+        end
+      end
+    end
+  end
+
+  resource :webhook_subscriptions do
+    post :create do
+      endpoint_removed!
+    end
+    route_param :arg do
+      post :test do
+        endpoint_removed!
+      end
+      post :delete do
+        endpoint_removed!
+      end
+    end
+  end
+end

data/lib/webhookdb/api.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require "grape"
+
+require "webhookdb"
+require "webhookdb/service"
+
+# API is the namespace module for API resources.
+module Webhookdb::API
+  require "webhookdb/api/entities"
+
+  class V1 < Webhookdb::Service
+    def self.inherited(subclass)
+      super
+      subclass.instance_eval do
+        version "v1", using: :path
+        format :json
+
+        require "webhookdb/service/helpers"
+        helpers Webhookdb::Service::Helpers
+        require "webhookdb/api/helpers"
+        helpers Webhookdb::API::Helpers
+        require "webhookdb/api/connstr_auth"
+
+        helpers do
+          def verified_customer!
+            c = current_customer
+            forbidden! unless c.phone_verified?
+            return c
+          end
+
+          # Lookup the organization.
+          # @param identifier [String] Can be passed in, or is extracted from the route params
+          #   (:org_identifier, or :org if :org_identiier is '-').
+          # @param customer [Webhookdb::Customer] Authed customer. Will use current_customer if nil.
+          # @param allow_connstr_auth [Boolean] True to use Webhookdb::API::ConnstrAuth.
+          #   See module for more details.
+          def lookup_org!(identifier=nil, customer: nil, allow_connstr_auth: false)
+            identifier ||= params[:org_identifier]
+            if identifier == "-"
+              identifier = params[:org]
+              merror!(400, "must supply 'org_identifier' or 'org' param", code: "missing_org") unless identifier
+            end
+            # Run this first to verify authentication before other lookups.
+            customer ||= allow_connstr_auth ? current_customer? : current_customer
+            # Can return multiple orgs, including ones the user cannot access
+            orgs = Webhookdb::Organization.with_identifier(identifier).all
+            merror!(403, "There is no organization with that identifier.") if orgs.empty?
+            if customer
+              # This is scoped to just orgs the user can access. We check if the identifier
+              # matches multiple orgs, in which case it's ambiguous.
+              memberships = customer.verified_memberships_dataset.where(organization: orgs).limit(2).all
+              permission_error!("You don't have permissions with that organization.") if memberships.empty?
+              merror!(500, "ambiguous", alert: true) if memberships.size > 1 # TODO: better message, tests
+              return memberships.first.organization
+            end
+            raise "something went wrong" unless allow_connstr_auth
+            org = Webhookdb::API::ConnstrAuth.find_authed(orgs, request)
+            unauthenticated! if org.nil?
+            return org
+          end
+
+          def ensure_admin!(org=nil, customer: nil)
+            customer ||= current_customer
+            org ||= lookup_org!
+            has_no_admin = org.verified_memberships_dataset.
+              where(customer:, membership_role: Webhookdb::Role.admin_role).
+              empty?
+            permission_error!("You don't have admin privileges with #{org.name}.") if has_no_admin
+          end
+        end
+
+        before do
+          Sentry.configure_scope do |scope|
+            scope.set_tags(application: "public-api")
+          end
+        end
+
+        before_validation do
+          # We want to strip control characters out of the string inputs
+          # Found the control character regex here:
+          #    https://www.appsloveworld.com/ruby/100/167/how-to-remove-control-characters-in-ruby
+          #
+          rgx = /\e\[[^\x40-\x7E]*[\x40-\x7E]/
+          self.params.each do |k, v|
+            params[k] = v.gsub(rgx, "") if v.is_a?(String)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/webhookdb/apps.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "grape"
+require "grape-swagger"
+require "sidekiq/web"
+require "sidekiq/cron/web"
+
+require "webhookdb/api"
+require "webhookdb/async"
+require "webhookdb/sentry"
+require "webhookdb/service"
+
+require "webhookdb/api/auth"
+require "webhookdb/api/db"
+require "webhookdb/api/demo"
+require "webhookdb/api/install"
+require "webhookdb/api/me"
+require "webhookdb/api/organizations"
+require "webhookdb/api/replay"
+require "webhookdb/api/service_integrations"
+require "webhookdb/api/services"
+require "webhookdb/api/stripe"
+require "webhookdb/api/subscriptions"
+require "webhookdb/api/sync_targets"
+require "webhookdb/api/system"
+require "webhookdb/api/webhook_subscriptions"
+
+require "webhookdb/admin_api/auth"
+require "webhookdb/admin_api/customers"
+require "webhookdb/admin_api/database_documents"
+require "webhookdb/admin_api/message_deliveries"
+require "webhookdb/admin_api/roles"
+
+require "webterm/apps"
+
+module Webhookdb::Apps
+  class API < Webhookdb::Service
+    mount Webhookdb::API::Auth
+    mount Webhookdb::API::Db
+    mount Webhookdb::API::Demo
+    mount Webhookdb::API::Install
+    mount Webhookdb::API::Me
+    mount Webhookdb::API::Organizations
+    mount Webhookdb::API::Replay
+    mount Webhookdb::API::ServiceIntegrations
+    mount Webhookdb::API::Services
+    mount Webhookdb::API::Stripe
+    mount Webhookdb::API::Subscriptions
+    mount Webhookdb::API::SyncTargets
+    mount Webhookdb::API::System
+    mount Webhookdb::API::WebhookSubscriptions
+    add_swagger_documentation if ENV["RACK_ENV"] == "development"
+  end
+
+  class AdminAPI < Webhookdb::Service
+    mount Webhookdb::AdminAPI::Auth
+    mount Webhookdb::AdminAPI::DatabaseDocuments
+    mount Webhookdb::AdminAPI::MessageDeliveries
+    mount Webhookdb::AdminAPI::Roles
+    mount Webhookdb::AdminAPI::Customers
+    add_swagger_documentation if ENV["RACK_ENV"] == "development"
+  end
+
+  SidekiqWeb = Rack::Builder.new do
+    use Sentry::Rack::CaptureExceptions if Webhookdb::Sentry.enabled?
+    use Rack::Auth::Basic, "Protected Area" do |username, password|
+      # Protect against timing attacks: (https://codahale.com/a-lesson-in-timing-attacks/)
+      # - Use & (do not use &&) so that it doesn't short circuit.
+      # - Use digests to stop length information leaking
+      Rack::Utils.secure_compare(
+        ::Digest::SHA256.hexdigest(username),
+        ::Digest::SHA256.hexdigest(Webhookdb::Async.web_username),
+      ) & Rack::Utils.secure_compare(
+        ::Digest::SHA256.hexdigest(password),
+        ::Digest::SHA256.hexdigest(Webhookdb::Async.web_password),
+      )
+    end
+    use Rack::Session::Cookie, secret: Webhookdb::Service.session_secret, same_site: true, max_age: 86_400
+    run Sidekiq::Web
+  end
+
+  Webterm = Rack::Builder.new do
+    use(Rack::SslEnforcer, {redirect_html: false}) if Webhookdb::Webterm.enforce_ssl
+    use Rack::Deflater
+    use Rack::ConditionalGet
+    use Rack::ETag
+    map "/" do
+      use Webhookdb::Webterm::RedirectIndexHtmlToRoot
+      use Webhookdb::Webterm::ServeIndexHtmlFromRoot
+      run Webhookdb::Webterm::Files
+    end
+  end
+end

data/lib/webhookdb/async/audit_logger.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "amigo/audit_logger"
+require "appydays/loggable"
+
+class Webhookdb::Async::AuditLogger < Amigo::AuditLogger
+  include Appydays::Loggable
+
+  sidekiq_options queue: "critical"
+
+  MAX_STR_LEN = 64
+  STR_PREFIX_LEN = 12
+
+  def perform(event_json)
+    j2 = event_json.dup
+    j2["payload"] = self.trim_long_strings(j2["payload"], max_str_len: MAX_STR_LEN, str_prefix_len: STR_PREFIX_LEN)
+    super(j2)
+  end
+
+  def trim_long_strings(v, max_str_len:, str_prefix_len:)
+    case v
+      when Hash
+        v.transform_values do |hv|
+          self.trim_long_strings(hv, max_str_len:, str_prefix_len:)
+        end
+      when Array
+        v.map { |item| self.trim_long_strings(item, max_str_len:, str_prefix_len:) }
+      when String
+        if v.size > max_str_len
+          v[..str_prefix_len] + "..."
+        else
+          v
+        end
+      else
+        v
+    end
+  end
+end

data/lib/webhookdb/async/autoscaler.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "amigo/autoscaler"
+require "amigo/autoscaler/heroku"
+require "webhookdb/async"
+require "webhookdb/heroku"
+
+module Webhookdb::Async::Autoscaler
+  include Appydays::Configurable
+  include Appydays::Loggable
+
+  AVAILABLE_PROVIDERS = ["heroku"].freeze
+
+  def self._check_provider!
+    return if AVAILABLE_PROVIDERS.include?(self.provider)
+    return if !self.enabled && self.provider.blank?
+    raise "invalid AUTOSCALER_PROVIDER: '#{self.provider}', one of: #{AVAILABLE_PROVIDERS.join(', ')}"
+  end
+
+  configurable(:autoscaler) do
+    setting :enabled, false
+    setting :provider, ""
+    setting :latency_threshold, 10
+    setting :alert_interval, 180
+    setting :poll_interval, 30
+    setting :max_additional_workers, 2
+    setting :latency_restored_threshold, 0
+    setting :hostname_regex, /^web\.1$/, convert: ->(s) { Regexp.new(s) }
+
+    after_configured do
+      self._check_provider!
+    end
+  end
+
+  class << self
+    def enabled? = self.enabled
+
+    def start
+      raise "already started" unless @instance.nil?
+      case self.provider
+        when "heroku"
+          opts = {heroku: Webhookdb::Heroku.client, max_additional_workers: self.max_additional_workers}
+          (opts[:app_id_or_app_name] = self.heroku_app_id_or_app_name) if
+            self.heroku_app_id_or_app_name
+          (opts[:formation_id_or_formation_type] = self.heroku_formation_id_or_formation_type) if
+            self.heroku_formation_id_or_formation_type
+          @impl = Amigo::Autoscaler::Heroku.new(**opts)
+        else
+          self._check_provider!
+      end
+      @instance = Amigo::Autoscaler.new(
+        poll_interval: self.poll_interval,
+        latency_threshold: self.latency_threshold,
+        hostname_regex: self.hostname_regex,
+        handlers: [self.method(:scale_up)],
+        alert_interval: self.alert_interval,
+        latency_restored_threshold: self.latency_restored_threshold,
+        latency_restored_handlers: [self.method(:scale_down)],
+        log: ->(level, msg, kw={}) { self.logger.send(level, msg, kw) },
+      )
+      return @instance.start
+    end
+
+    def stop
+      raise "not started" if @instance.nil?
+      @instance.stop
+    end
+
+    def scale_up(names_and_latencies, depth:, duration:, **)
+      scale_action = @impl.scale_up(names_and_latencies, depth:, duration:, **)
+      kw = {queues: names_and_latencies, depth:, duration:, scale_action:}
+      self.logger.warn("high_latency_queues_event", **kw)
+      Sentry.with_scope do |scope|
+        scope&.set_extras(**kw)
+        Sentry.capture_message("Some queues have a high latency")
+      end
+    end
+
+    def scale_down(depth:, duration:, **)
+      scale_action = @impl.scale_down(depth:, duration:, **)
+      self.logger.warn("high_latency_queues_resolved", depth:, duration:, scale_action:)
+    end
+  end
+end