webhookdb 1.2.1 → 1.3.0

data/lib/sequel/text_searchable.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+module SequelTextSearchable
+  VERSION = "0.0.1"
+
+  INDEX_MODES = [:async, :sync, :off].freeze
+  DEFAULT_MODE = :async
+
+  class << self
+    def index_mode = @index_mode || DEFAULT_MODE
+
+    def index_mode=(v)
+      raise ArgumentError, "mode #{v.inspect} must be one of: #{INDEX_MODES}" unless
+        INDEX_MODES.include?(v)
+      @index_mode = v
+    end
+
+    def searchable_models = @searchable_models ||= []
+
+    # Return the global threadpool for :async indexing.
+    # Use at most a couple threads; if the work gets backed up,
+    # have the caller run it. If the threads die,
+    # the text update is lost, so we don't want to let it queue up forever.
+    def threadpool
+      return @threadpool ||= Concurrent::ThreadPoolExecutor.new(
+        min_threads: 1,
+        max_threads: 2,
+        max_queue: 10,
+        fallback_policy: :caller_runs,
+      )
+    end
+
+    # Set your own threadpool.
+    attr_writer :threadpool
+
+    def reindex_all
+      return self.searchable_models.sum(&:text_search_reindex_all)
+    end
+  end
+end

data/lib/webhookdb/admin_api/auth.rb

@@ -8,7 +8,28 @@ class Webhookdb::AdminAPI::Auth < Webhookdb::AdminAPI::V1
   resource :auth do
     desc "Return the current administrator customer."
     get do
-      present admin_customer, with: Webhookdb::AdminAPI::CurrentCustomerEntity, env:
+      present admin_customer, with: Webhookdb::AdminAPI::Entities::CurrentCustomer, env:
+    end
+
+    params do
+      requires :email, type: String
+      requires :password, type: String
+    end
+
+    resource :login do
+      auth(:skip)
+      params do
+        requires :email, type: String
+        requires :password, type: String
+      end
+      post do
+        (me = Webhookdb::Customer.with_email(params[:email])) or unauthenticated!
+        me.authenticate(params[:password]) or unauthenticated!
+        unauthenticated! unless me.admin?
+        set_customer(me)
+        status 200
+        present admin_customer, with: Webhookdb::AdminAPI::Entities::CurrentCustomer, env:
+      end
     end
 
     resource :impersonate do
@@ -17,7 +38,7 @@ class Webhookdb::AdminAPI::Auth < Webhookdb::AdminAPI::V1
         Webhookdb::Service::Auth::Impersonation.new(env["warden"]).off(admin_customer)
 
         status 200
-        present admin_customer, with: Webhookdb::AdminAPI::CurrentCustomerEntity, env:
+        present admin_customer, with: Webhookdb::AdminAPI::Entities::CurrentCustomer, env:
       end
 
       route_param :customer_id, type: Integer do
@@ -28,7 +49,7 @@ class Webhookdb::AdminAPI::Auth < Webhookdb::AdminAPI::V1
           Webhookdb::Service::Auth::Impersonation.new(env["warden"]).on(target)
 
           status 200
-          present target, with: Webhookdb::AdminAPI::CurrentCustomerEntity, env:
+          present target, with: Webhookdb::AdminAPI::Entities::CurrentCustomer, env:
         end
       end
     end

data/lib/webhookdb/admin_api/data_provider.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+require "webhookdb/admin_api"
+
+class Webhookdb::AdminAPI::DataProvider < Webhookdb::AdminAPI::V1
+  include Webhookdb::AdminAPI::Entities
+
+  class CustomerRoleModel < Sequel::Model
+    many_to_one :customer, class: "Webhookdb::Customer"
+    many_to_one :role, class: "Webhookdb::Role"
+  end
+  CustomerRoleModel.set_dataset(Webhookdb::Postgres::Model.db[:roles_customers])
+
+  TYPEINFO = {
+    backfill_jobs: [Webhookdb::BackfillJob, BackfillJob],
+    customers: [Webhookdb::Customer, Customer],
+    customer_reset_codes: [Webhookdb::Customer::ResetCode, CustomerResetCode],
+    customer_roles: [CustomerRoleModel, CustomerRole],
+    logged_webhooks: [Webhookdb::LoggedWebhook, LoggedWebhook],
+    message_bodies: [Webhookdb::Message::Body, MessageBody],
+    message_deliveries: [Webhookdb::Message::Delivery, MessageDelivery],
+    organization_database_migrations: [Webhookdb::Organization::DatabaseMigration, OrganizationDatabaseMigration],
+    organization_memberships: [Webhookdb::OrganizationMembership, OrganizationMembership],
+    organizations: [Webhookdb::Organization, Organization],
+    replicated_databases: [nil, nil],
+    roles: [Webhookdb::Role, Role],
+    saved_queries: [Webhookdb::SavedQuery, SavedQuery],
+    saved_views: [Webhookdb::SavedView, SavedView],
+    service_integrations: [Webhookdb::ServiceIntegration, ServiceIntegration],
+    subscriptions: [Webhookdb::Subscription, Subscription],
+    sync_targets: [Webhookdb::SyncTarget, SyncTarget],
+    webhook_subscriptions: [Webhookdb::WebhookSubscription, WebhookSubscription],
+    webhook_subscription_deliveries: [Webhookdb::WebhookSubscription::Delivery, WebhookSubscriptionDelivery],
+  }.freeze
+
+  TYPES_FOR_RESOURCES = TYPEINFO.transform_values { |v| v[0] }.freeze
+  ENTITIES_FOR_TYPES = TYPEINFO.values.to_h { |v| [v[0], v[1]] }.freeze
+
+  resource :data_provider do
+    helpers do
+      params :data_provider_pagination do
+        requires :page, type: Integer
+        requires :per_page, type: Integer
+      end
+
+      params :data_provider_sort do
+        requires :field, type: Symbol
+        requires :order, type: Symbol, values: [:ASC, :DESC]
+      end
+
+      params :base_params do
+        requires :resource, type: Symbol, values: TYPES_FOR_RESOURCES.keys
+        optional :meta, type: JSON
+      end
+
+      params :base_record_params do
+        use :base_params
+        requires :id, type: Integer
+      end
+
+      params :list_params do
+        use :base_params
+        optional :pagination, type: JSON do
+          use :data_provider_pagination
+        end
+        optional :sort, type: JSON do
+          use :data_provider_sort
+        end
+        optional :filter, type: JSON
+      end
+
+      params :many_ids_params do
+        use :base_params
+        requires :ids, type: [Integer]
+      end
+
+      def lookup_model_type
+        rt = TYPES_FOR_RESOURCES.fetch(params.fetch(:resource))
+        return rt
+      end
+
+      def lookup_model
+        cls = lookup_model_type
+        m = cls[id: params.fetch(:id)]
+        merror!(403, "No #{params[:resource]} with pk #{params[:id]}", code: "forbidden") if m.nil?
+        return m
+      end
+
+      def lookup_entity(cls)
+        return ENTITIES_FOR_TYPES.fetch(cls)
+      end
+
+      def present_one(item, item_entity)
+        data = item_entity.represent(item)
+        e = {data:}
+        present e
+      end
+
+      def present_dataset(ds, item_entity)
+        data = ds.all.map { |o| item_entity.represent(o) }
+        total = ds.respond_to?(:pagination_record_count) ? ds.pagination_record_count : ds.limit(nil).offset(nil).count
+        e = {data:, total:}
+        present e
+      end
+
+      def apply_list_params(ods)
+        ds = ods
+        if (filter = params[:filter])
+          search_term = filter.delete(:q)
+          ds = ds.text_search(search_term) if search_term && ds.respond_to?(:text_search)
+          ds = ds.where(filter.to_h.transform_keys(&:to_sym)) if filter.present?
+        end
+        if (sort = params[:sort])
+          sort_col = sort[:field]
+          unless sort_col.to_s.include?(".")
+            # If the sort column includes a '.', we have to special case it.
+            # Right now we don't have any allowed sort cols.
+            # It is possible to make an unsupported column sortable in react-admin, causing the backend to error,
+            # but it's not easy to clear out your filters/sort settings once this happens.
+            # So instead, just avoid causing the error, and just no-op if an invalid column is passed in.
+            order = Sequel.send(sort[:order].to_s.downcase, sort_col, nulls: :last)
+            ds = ds.order(order)
+          end
+        else
+          ds = ds.order(Sequel.desc(ds.model.primary_key))
+        end
+        if (pagination = params[:pagination])
+          ds = ds.paginate(pagination[:page], pagination[:per_page])
+        end
+        return ds
+      end
+    end
+
+    params do
+      use :base_record_params
+    end
+    post :get_one do
+      model = lookup_model
+      status 200
+      present_one model, lookup_entity(model.class)
+    end
+
+    params do
+      use :list_params
+    end
+    post :get_list do
+      model_cls = lookup_model_type
+      ds = model_cls.dataset
+      ds = apply_list_params(ds)
+      status 200
+      present_dataset ds, lookup_entity(model_cls)
+    end
+
+    params do
+      use :many_ids_params
+    end
+    post :get_many do
+      model_cls = lookup_model_type
+      ds = model_cls.dataset
+      ds = ds.where(id: params.fetch(:ids))
+      status 200
+      present_dataset ds, lookup_entity(model_cls)
+    end
+
+    params do
+      use :list_params
+      requires :target, type: Symbol
+      requires :id, type: Integer
+    end
+    post :get_many_reference do
+      if params[:resource] == :replicated_databases
+        (org = Webhookdb::Organization[params[:id]]) or forbidden!
+        rows = org.admin_connection do |db|
+          rows = db[Sequel[:information_schema][:tables]].
+            where(table_schema: org.replication_schema).
+            select(
+              Sequel[:table_name].as(:id),
+              :table_name,
+              Sequel.expr { pg_size_pretty(pg_total_relation_size(quote_ident(table_name))) }.as(:size_pretty),
+              Sequel.expr { pg_relation_size(quote_ident(table_name)) }.as(:size),
+            ).all
+          rows
+        end
+        status 200
+        present({data: rows, total: rows.length})
+      else
+        model_cls = lookup_model_type
+        ds = model_cls.dataset
+        ds = ds.where(params[:target] => params[:id])
+        ds = apply_list_params(ds)
+        status 200
+        present_dataset ds, lookup_entity(model_cls)
+      end
+    end
+  end
+end

data/lib/webhookdb/admin_api/entities.rb

@@ -5,62 +5,177 @@ require "grape_entity"
 require "webhookdb/service/entities"
 require "webhookdb/admin_api" unless defined? Webhookdb::AdminAPI
 
-module Webhookdb::AdminAPI
-  CurrentCustomerEntity = Webhookdb::Service::Entities::CurrentCustomer
-  MoneyEntity = Webhookdb::Service::Entities::Money
-  TimeRangeEntity = Webhookdb::Service::Entities::TimeRange
+module Webhookdb::AdminAPI::Entities
+  CurrentCustomer = Webhookdb::Service::Entities::CurrentCustomer
+  Money = Webhookdb::Service::Entities::Money
+  TimeRange = Webhookdb::Service::Entities::TimeRange
 
-  class BaseEntity < Webhookdb::Service::Entities::Base; end
+  class Base < Webhookdb::Service::Entities::Base
+    expose :id, if: ->(o) { o.respond_to?(:id) }
+    expose :created_at, if: ->(o) { o.respond_to?(:created_at) }
+    expose :updated_at, if: ->(o) { o.respond_to?(:updated_at) }
+    expose :soft_deleted_at, if: ->(o) { o.respond_to?(:soft_deleted_at) }
+    expose :admin_link, if: ->(o, _) { o.respond_to?(:admin_link) }
+  end
 
-  class RoleEntity < BaseEntity
+  class Related < Grape::Entity
     expose :id
-    expose :name
   end
 
-  class CustomerEntity < BaseEntity
-    expose :id
-    expose :created_at
+  class BackfillJob < Base
+    expose :service_integration, with: Related
+    expose :organization, with: Related, &self.delegate_to(:service_integration, :organization)
+    expose :started_at
+    expose :finished_at
+    expose :opaque_id
+    expose :parent_job, with: Related
+    expose :created_by, with: Related
+    expose :incremental
+  end
+
+  class Customer < Base
     expose :email
     expose :name
     expose :note
   end
 
-  class CustomerResetCodes < BaseEntity
-    expose :id
-    expose :created_at
+  class CustomerResetCode < Base
     expose :transport
     expose :token
     expose :used
     expose :expire_at
+    expose :customer, with: Related
   end
 
-  class DetailedCustomerEntity < CustomerEntity
-    expose :roles do |instance|
-      instance.roles.map(&:name)
+  class CustomerRole < Base
+    expose :customer do
+      expose :id, &self.delegate_to(:customer_id)
+      expose :email, &self.delegate_to(:customer, :email)
+    end
+    expose :role do
+      expose :id, &self.delegate_to(:role_id)
+      expose :name, &self.delegate_to(:role, :name)
     end
-    expose :reset_codes, with: CustomerResetCodes
   end
 
-  class MessageBodyEntity < BaseEntity
-    expose :id
+  class LoggedWebhook < Base
+    expose :inserted_at
+    expose :truncated_at
+    expose :request_body
+    expose(:request_headers) { |inst| inst.request_headers.to_a }
+    expose :response_status
+    expose :service_integration_opaque_id
+    expose :service_integration, with: Related
+    expose :organization, with: Related
+    expose :request_method
+    expose :request_path
+  end
+
+  class MessageBody < Base
     expose :content
     expose :mediatype
+    expose :delivery, with: Related
   end
 
-  class MessageDeliveryEntity < BaseEntity
-    expose :id
-    expose :created_at
-    expose :updated_at
-    expose :soft_deleted_at
+  class MessageDelivery < Base
+    expose :extra_fields
+    expose :recipient, with: Related
+    expose :sent_at
     expose :template
+    expose :to
     expose :transport_type
     expose :transport_service
     expose :transport_message_id
-    expose :sent_at
-    expose :to
   end
 
-  class MessageDeliveryWithBodiesEntity < MessageDeliveryEntity
-    expose :bodies, with: MessageBodyEntity
+  class Organization < Base
+    expose :name
+    expose :key
+  end
+
+  class OrganizationDatabaseMigration < Base
+    expose :organization, with: Related
+    expose :started_at
+    expose :finished_at
+    expose :started_by, with: Related
+    expose :organization_schema
+    expose :last_migrated_service_integration_id
+    expose :last_migrated_service_integration, with: Related
+    expose :last_migrated_timestamp
+  end
+
+  class OrganizationMembership < Base
+    expose :organization, with: Related
+    expose :customer, with: Related
+    expose :membership_role, with: Related
+    expose :verified
+    expose :invitation_code
+    expose :is_default
+  end
+
+  class Role < Base
+    expose :name
+  end
+
+  class SavedQuery < Base
+    expose :organization, with: Related
+    expose :created_by, with: Related
+    expose :opaque_id
+    expose :description
+    expose :sql
+    expose :public
+  end
+
+  class SavedView < Base
+    expose :organization, with: Related
+    expose :created_by, with: Related
+    expose :name
+    expose :sql
+  end
+
+  class ServiceIntegration < Base
+    expose :organization, with: Related
+    expose :table_name
+    expose :service_name
+    expose :opaque_id
+    expose :last_backfilled_at
+    expose :depends_on, with: Related
+    expose :skip_webhook_verification
+  end
+
+  class Subscription < Base
+    expose :organization, with: Related
+    expose :stripe_id
+    expose :stripe_customer_id
+    expose :stripe_json
+  end
+
+  class SyncTarget < Base
+    expose :organization, with: Related
+    expose :service_integration, with: Related
+    expose :created_by, with: Related
+    expose :opaque_id
+    expose :period_seconds
+    expose :schema
+    expose :table
+    expose :last_synced_at
+    expose :last_applied_schema
+    expose :page_size
+  end
+
+  class WebhookSubscription < Base
+    expose :organization, with: Related
+    expose :service_integration, with: Related
+    expose :created_by, with: Related
+    expose :opaque_id
+    expose :deliver_to_url
+    expose :deactivated_at
+  end
+
+  class WebhookSubscriptionDelivery < Base
+    expose :attempt_timestamps
+    expose :attempt_http_response_statuses
+    expose :payload
+    expose :webhook_subscription, with: Related
   end
 end

data/lib/webhookdb/admin_api.rb

@@ -16,8 +16,6 @@ module Webhookdb::AdminAPI
         version "v1", using: :path
         format :json
 
-        content_type :csv, "text/csv"
-
         require "webhookdb/service/helpers"
         helpers Webhookdb::Service::Helpers
 

data/lib/webhookdb/api/auth.rb

@@ -26,13 +26,12 @@ class Webhookdb::API::Auth < Webhookdb::API::V1
 
     params do
       optional :email,
-               type: String,
-               coerce_with: NormalizedEmail,
+               type: NormalizedEmail,
                prompt: {
                  message: "Welcome to WebhookDB!\nPlease enter your email:",
                  demo_mode_proc: ->(r) { r.params[:email] = "demo@webhookdb.com" },
                }
-      optional :token, type: String
+      optional :token, type: TrimmedString
     end
     post do
       message = ""
@@ -86,9 +85,9 @@ class Webhookdb::API::Auth < Webhookdb::API::V1
     end
 
     params do
-      requires :form_name, type: String
-      optional :email, type: String
-      optional :name, type: String
+      requires :form_name, type: TrimmedString
+      optional :email, type: TrimmedString
+      optional :name, type: TrimmedString
       optional :message, type: String
     end
     post :contact do

data/lib/webhookdb/api/db.rb

@@ -6,15 +6,17 @@ require "webhookdb/api"
 require "webhookdb/replicator"
 
 class Webhookdb::API::Db < Webhookdb::API::V1
+  include Webhookdb::Service::Types
+
   helpers do
     params :fdw do
       optional :message_fdw, type: Boolean
       optional :message_views, type: Boolean
       optional :message_all, type: Boolean
-      requires :remote_server_name, type: String
-      requires :fetch_size, type: String
-      requires :local_schema, type: String
-      requires :view_schema, type: String
+      requires :remote_server_name, type: TrimmedString
+      requires :fetch_size, type: TrimmedString
+      requires :local_schema, type: TrimmedString
+      requires :view_schema, type: TrimmedString
     end
 
     def run_fdw
@@ -38,6 +40,29 @@ class Webhookdb::API::Db < Webhookdb::API::V1
   end
 
   resource :db do
+    [:get, :post].each do |httpmethod|
+      desc "Execute an arbitrary query in a replication database. Same as /db/<org>/sql but safe for CORS usage." do
+        headers Webhookdb::API::ConnstrAuth.headers_desc
+      end
+      params do
+        requires :org_identifier, type: TrimmedString
+        optional :query, type: String, desc: "SQL to run."
+        optional :query_base64, type: TrimmedString, desc: "Base64 encoded SQL. Mostly used for GET requests."
+        exactly_one_of :query, :query_base64
+      end
+      send(httpmethod, :run_sql) do
+        use_http_expires_caching(5.minutes)
+        org = lookup_org!(allow_connstr_auth: true)
+        unless (query = params[:query])
+          query = Base64.urlsafe_decode64(params[:query_base64])
+        end
+        r, msg = execute_readonly_query(org, query)
+        merror!(403, msg, code: "invalid_query") if r.nil?
+        status 200
+        present({rows: r.rows, headers: r.columns, max_rows_reached: r.max_rows_reached})
+      end
+    end
+
     route_param :org_identifier, type: String do
       desc "Returns the connection string"
       get :connection do
@@ -64,7 +89,7 @@ class Webhookdb::API::Db < Webhookdb::API::V1
 
       desc "Enqueues a database migration."
       params do
-        optional :admin_url, type: String, prompt: {
+        optional :admin_url, type: TrimmedString, prompt: {
           message: "ADMIN Postgres connection URL, in the form 'postgres://user:password@host:port/dbname',\n" \
                    "that is capable of administrative operations on your database,\n" \
                    "such as creating and dropping schemas and tables.\n" \
@@ -73,7 +98,7 @@ class Webhookdb::API::Db < Webhookdb::API::V1
           disable: ->(_) { !Webhookdb::Organization::DbBuilder.allow_public_migrations },
         }
 
-        optional :readonly_url, type: String, prompt: {
+        optional :readonly_url, type: TrimmedString, prompt: {
           message: "READONLY Postgres connection URL.\n" \
                    "This string is displayed when you ask for your organization's connection information.\n" \
                    "If you are okay with this being your ADMIN URL, leave it blank.\n" \

data/lib/webhookdb/api/entities.rb

@@ -9,7 +9,11 @@ module Webhookdb::API
   MoneyEntity = Webhookdb::Service::Entities::Money
   TimeRangeEntity = Webhookdb::Service::Entities::TimeRange
 
-  class BaseEntity < Webhookdb::Service::Entities::Base; end
+  class BaseEntity < Webhookdb::Service::Entities::Base
+    expose :message do |_instance, options|
+      options[:message] || ""
+    end
+  end
 
   class OrganizationEntity < BaseEntity
     expose :id
@@ -43,11 +47,13 @@ module Webhookdb::API
     expose :verified_memberships, with: OrganizationMembershipEntity
     expose :verified_memberships_formatted do |instance|
       lines = instance.verified_memberships.map { |m| "#{m.organization.display_string}: #{m.status}" }
+      lines.sort!
       lines.join("\n")
     end
     expose :invited_memberships, as: :invitations, with: OrganizationMembershipEntity
     expose :invitations_formatted do |instance|
       lines = instance.invited_memberships.map { |m| "#{m.organization.display_string}: #{m.invitation_code}" }
+      lines.sort!
       lines.join("\n")
     end
     expose :display_headers do |_|

data/lib/webhookdb/api/helpers.rb

@@ -164,15 +164,15 @@ module Webhookdb::API::Helpers
     request_headers = {}
     raise LocalJumpError unless block_given?
     begin
+      request_headers = request.headers.dup
+      if (content_type = env["CONTENT_TYPE"])
+        request_headers["Content-Type"] = content_type
+      end
       sint = yield
       return if sint == :pass
       raise "error instead of return nil if there is no service integration" if sint.nil?
       opaque_id = sint.opaque_id
       organization_id = sint.organization_id
-      request_headers = request.headers.dup
-      if (content_type = env["CONTENT_TYPE"])
-        request_headers["Content-Type"] = content_type
-      end
       svc = Webhookdb::Replicator.create(sint).dispatch_request_to(request)
       svc.preprocess_headers_for_logging(request_headers)
       handling_sint = svc.service_integration
@@ -205,6 +205,7 @@ module Webhookdb::API::Helpers
             path: process_kwargs[:request_path],
             headers: process_kwargs[:headers],
             body: process_kwargs[:body],
+            rack_request: request,
           )
           inserted = svc.upsert_webhook(whreq)
           s_body = svc.synchronous_processing_response_body(upserted: inserted, request: whreq)
@@ -256,25 +257,5 @@ module Webhookdb::API::Helpers
   #   On query success, return <QueryResult, nil>.
   #   On DatabaseError, return <nil, message>.
   #   On other types of errors, raise.
-  def execute_readonly_query(org, sql)
-    result = org.execute_readonly_query(sql)
-    return result, nil
-  rescue Sequel::DatabaseError => e
-    self.logger.error("db_query_database_error", error: e)
-    # We want to handle InsufficientPrivileges and UndefinedTable explicitly
-    # since we can hint the user at what to do.
-    # Otherwise, we should just return the Postgres exception.
-    msg = ""
-    case e.wrapped_exception
-      when PG::UndefinedTable
-        missing_table = e.wrapped_exception.message.match(/relation (.+) does not/)&.captures&.first
-        msg = "The table #{missing_table} does not exist. Run `webhookdb db tables` to see available tables." if
-          missing_table
-      when PG::InsufficientPrivilege
-        msg = "You do not have permission to perform this query. Queries must be read-only."
-      else
-        msg = e.wrapped_exception.message
-    end
-    return [nil, msg]
-  end
+  def execute_readonly_query(org, sql) = org.execute_readonly_query_with_help(sql)
 end