webhookdb 1.4.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1e9c0cec9d89ca341198512d1ae986100a6db250a1d26403529888ec6f71f467
-  data.tar.gz: 2819c28d8615b488d5e29163f3c9962d3d1b75777b9f9cc6c3540f468864bca9
+  metadata.gz: 4fa962b21d64eb6d04e0e55247340e7d07c592c360a56b26227fc3449e55b148
+  data.tar.gz: dd97a7e221331c5f15e9a39d6c32d14bbb37ab24943c7af5fbdb7d44c3f55fc3
 SHA512:
-  metadata.gz: 024a72c5913586de41647332309b051be213be9cc5aeb54f4269e8ea3d06e671fce56cdf28f38f00895a4ea37aee22cf877be4e5403a619fed82f032aee57671
-  data.tar.gz: 0ccaeb11294b44a69787c555422f6fc72e573c96e3332a3e2d247858df3c65914a530a62a707a8d8c6d83b6a4ef8820c70e3a40949ceb6ce6b23cd5c2ef409a8
+  metadata.gz: efaa0af60718483dbd227c1af6e65583de80315b6161bb267e35a3d9564b806229ed3b90496bd7c0f91f7dcffcc3ec84fe20d2ab5f902650bffaa42f0e9b50fe
+  data.tar.gz: 1a2ead61ea98ed53526a8bf9990bf3ecb90c3404380c1c8db42919cb7afd84dc5e1b54e5793987864723a0673cb983efba2c8271135699451ca5a8f6544d2602

data/db/migrations/026_undo_integration_backfill_cursor.rb

@@ -2,8 +2,10 @@
 
 Sequel.migration do
   change do
+    # rubocop:disable Sequel/IrreversibleMigration
     alter_table(:service_integrations) do
       drop_column :backfill_cursor
     end
+    # rubocop:enable Sequel/IrreversibleMigration
   end
 end

data/db/migrations/032_remove_db_defaults.rb

@@ -2,11 +2,13 @@
 
 Sequel.migration do
   change do
+    # rubocop:disable Sequel/IrreversibleMigration
     alter_table(:sync_targets) do
       set_column_default :page_size, nil
     end
     alter_table(:organizations) do
       set_column_default :minimum_sync_seconds, nil
     end
+    # rubocop:enable Sequel/IrreversibleMigration
   end
 end

data/db/migrations/043_text_search.rb

@@ -19,10 +19,12 @@ Sequel.migration do
       :webhook_subscription_deliveries,
       :webhook_subscriptions,
     ]
+    # rubocop:disable Sequel/IrreversibleMigration
     searchable.each do |tbl|
       alter_table(tbl) do
         add_column :text_search, :tsvector
       end
     end
+    # rubocop:enable Sequel/IrreversibleMigration
   end
 end

data/db/migrations/047_sync_parallelism.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:sync_targets) do
+      add_column :parallelism, :integer, null: false, default: 0
+    end
+  end
+end

data/db/migrations/048_sync_stats.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:sync_targets) do
+      add_column :sync_stats, :jsonb, null: false, default: "[]"
+    end
+  end
+end

data/db/migrations/049_error_handlers.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    create_table(:organization_error_handlers) do
+      primary_key :id
+      timestamptz :created_at, null: false, default: Sequel.function(:now)
+      timestamptz :updated_at
+
+      text :opaque_id, null: false, unique: true
+
+      foreign_key :organization_id, :organizations, null: false
+      foreign_key :created_by_id, :customers, null: true, on_delete: :set_null
+
+      text :url, null: false
+    end
+  end
+end

data/db/migrations/050_logged_webhook_indices.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  no_transaction
+  up do
+    # These are optimized for Webhookdb::LoggedWebhook.trim. See code for details/coupling.
+    run "CREATE INDEX CONCURRENTLY logged_webhooks_trim_unowned_idx ON logged_webhooks(inserted_at) " \
+        "WHERE organization_id IS NULL"
+    run "CREATE INDEX CONCURRENTLY logged_webhooks_trim_success_idx ON logged_webhooks(inserted_at) " \
+        "WHERE organization_id IS NOT NULL AND response_status < 400 AND truncated_at IS NULL"
+    run "CREATE INDEX CONCURRENTLY logged_webhooks_delete_success_idx ON logged_webhooks(inserted_at) " \
+        "WHERE organization_id IS NOT NULL AND response_status < 400 AND truncated_at IS NOT NULL"
+    run "CREATE INDEX CONCURRENTLY logged_webhooks_trim_failures_idx ON logged_webhooks(inserted_at) " \
+        "WHERE organization_id IS NOT NULL AND response_status >= 400 AND truncated_at IS NULL"
+    run "CREATE INDEX CONCURRENTLY logged_webhooks_delete_failures_idx ON logged_webhooks(inserted_at) " \
+        "WHERE organization_id IS NOT NULL AND response_status >= 400 AND truncated_at IS NOT NULL"
+  end
+  down do
+    run "DROP INDEX logged_webhooks_trim_unowned_idx"
+    run "DROP INDEX logged_webhooks_trim_success_idx"
+    run "DROP INDEX logged_webhooks_delete_success_idx"
+    run "DROP INDEX logged_webhooks_trim_failures_idx"
+    run "DROP INDEX logged_webhooks_delete_failures_idx"
+  end
+end

data/db/migrations/051_partitioning.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:service_integrations) do
+      add_column :partition_value, :int, default: 0, null: false
+    end
+  end
+end

data/integration/async_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "rspec/eventually"
-
 require "webhookdb/async"
 
 RSpec.describe "async workers", :integration do

data/integration/service_integrations_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "rspec/eventually"
-
 RSpec.describe "service integrations", :integration do
   def catch_missing_db(default)
     yield

data/lib/amigo/durable_job.rb

@@ -351,8 +351,8 @@ module Amigo::DurableJob
     # Space-separate multiple env vars.
     setting :server_env_vars, ["DATABASE_URL"], convert: ->(s) { s.split.map(&:strip) }
 
-    setting :schema_name, :public, convert: ->(s) { s.to_sym }
-    setting :table_name, :durable_jobs, convert: ->(s) { s.to_sym }
+    setting :schema_name, :public, convert: lambda(&:to_sym)
+    setting :table_name, :durable_jobs, convert: lambda(&:to_sym)
 
     after_configured do
       self.storage_database_urls = self.server_urls.dup

data/lib/amigo/job_in_context.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Amigo
+end
+
+module Amigo::JobInContext
+  class ServerMiddleware
+    def call(worker, job, queue, &)
+      Sidekiq::Context.with(worker:, queue:, job_hash: job, &)
+    end
+  end
+end

data/lib/webhookdb/api/entities.rb

@@ -136,10 +136,12 @@ module Webhookdb::API
     expose :opaque_id
     expose :service_integration, with: ServiceIntegrationEntity
     expose :period_seconds
+    expose :page_size
     expose :displaysafe_connection_url, as: :connection_url
     expose :table
     expose :schema
     expose :last_synced_at
+    expose :latency, &self.delegate_to(:latency, :to_i)
     expose :associated_type
     expose :associated_id
     expose :associated_object_display
@@ -155,8 +157,9 @@ module Webhookdb::API
         [:associated_object_display, "Associated"],
         [:schema_and_table_string, "Table"],
         [:last_synced_at, "Last Synced"],
+        [:latency, "Latency"],
         [:period_seconds, "Period"],
-        [:page_size, "Page"],
+        [:page_size, "Page Size"],
       ]
     end
   end
@@ -168,8 +171,9 @@ module Webhookdb::API
         [:connection_url, "URL"],
         [:associated_object_display, "Associated"],
         [:last_synced_at, "Last Synced"],
+        [:latency, "Latency"],
         [:period_seconds, "Period"],
-        [:page_size, "Page"],
+        [:page_size, "Page Size"],
       ]
     end
   end

data/lib/webhookdb/api/error_handlers.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require "webhookdb/api"
+
+class Webhookdb::API::ErrorHandlers < Webhookdb::API::V1
+  include Webhookdb::Service::Types
+
+  CREATE_PROMPT = "What is the URL that WebhookDB should POST to when a replicator fails? " \
+                  "See #{Webhookdb::Organization::ErrorHandler::DOCS_URL} for more information:".freeze
+
+  resource :organizations do
+    route_param :org_identifier do
+      resource :error_handlers do
+        helpers do
+          def lookup!
+            org = lookup_org!
+            eh = org.error_handlers_dataset[opaque_id: params[:opaque_id].strip]
+            merror!(403, "There is no error handler with that id.") if eh.nil?
+            set_request_tags(error_handler_opaque_id: eh.opaque_id)
+            return eh
+          end
+
+          def guard_editable!(customer, org)
+            return if has_admin?(org, customer:)
+            permission_error!("You must be an org admin to modify error handlers.")
+          end
+
+          def url_valid?(u)
+            begin
+              uri = URI(u)
+            rescue URI::InvalidURIError
+              return false
+            end
+            return true if uri.scheme == "sentry"
+            return false unless uri.scheme&.start_with?("http")
+            return true
+          end
+        end
+
+        desc "Returns a list of all error handlers associated with the org."
+        get do
+          handlers = lookup_org!.error_handlers
+          message = ""
+          if handlers.empty?
+            message = "This organization doesn't have any error handlers yet.\n" \
+                      "Use `webhookdb error-handler create` to set one up."
+          end
+          present_collection handlers, with: ErrorHandlerEntity, message:
+        end
+
+        desc "Creates a new error handler."
+        params do
+          optional :url, type: TrimmedString, prompt: CREATE_PROMPT
+        end
+        post :create do
+          cust = current_customer
+          org = lookup_org!
+          guard_editable!(cust, org)
+          unless url_valid?(params[:url])
+            msg = "URL is malformed. It should be a URL like https://foo.bar/path, http://foo.bar:123, etc. " \
+                  "See #{Webhookdb::Organization::ErrorHandler::DOCS_URL} for more info."
+            merror!(400, msg)
+          end
+          eh = Webhookdb::Organization::ErrorHandler.create(
+            organization: org,
+            url: params[:url],
+            created_by: cust,
+          )
+          message = "Whenever one of your replicators errors, WebhookDB will alert the given URL. " \
+                    "See #{Webhookdb::Organization::ErrorHandler::DOCS_URL} for more information."
+          status 200
+          present eh, with: ErrorHandlerEntity, message:
+        end
+
+        route_param :opaque_id, type: String do
+          get do
+            eh = lookup!
+            status 200
+            present eh, with: ErrorHandlerEntity
+          end
+
+          post :delete do
+            customer = current_customer
+            eh = lookup!
+            guard_editable!(customer, eh.organization)
+            eh.destroy
+            status 200
+            present eh, with: ErrorHandlerEntity,
+                        message: "You have successfully deleted the error handler '#{eh.opaque_id}'."
+          end
+        end
+      end
+    end
+  end
+
+  class ErrorHandlerEntity < Webhookdb::API::BaseEntity
+    expose :opaque_id, as: :id
+    expose :url
+
+    def self.display_headers
+      return [[:id, "ID"], [:url, "Url"]]
+    end
+  end
+end

data/lib/webhookdb/api/helpers.rb

@@ -122,7 +122,14 @@ module Webhookdb::API::Helpers
     sints = org.service_integrations_dataset.
       where(Sequel[service_name: identifier] | Sequel[table_name: identifier] | Sequel[opaque_id: identifier]).
       limit(2).all
-    return sints.first if sints.size == 1
+    if sints.size == 1
+      sint = sints.first
+      set_request_tags(
+        service_integration_service_name: sint.service_name,
+        service_integration_table_name: sint.table_name,
+      )
+      return sint
+    end
     merror!(403, "There is no service integration with that identifier.") if sints.empty?
     dupe_attr = nil
     alternative = nil

data/lib/webhookdb/api/icalproxy.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "webhookdb/api"
+require "webhookdb/jobs/icalendar_enqueue_syncs_for_urls"
+
+class Webhookdb::API::Icalproxy < Webhookdb::API::V1
+  resource :icalproxy do
+    resource :webhook do
+      post do
+        merror!(402, "Api key not configured") unless Webhookdb::Icalendar.proxy_api_key.present?
+        request_key = request.env["HTTP_AUTHORIZATION"] || ""
+        configured_key = "Apikey #{Webhookdb::Icalendar.proxy_api_key}"
+        verified = request_key && ActiveSupport::SecurityUtils.secure_compare(request_key, configured_key)
+        merror!(401, "Invalid api key") unless verified
+        urls = params.fetch(:urls)
+        Webhookdb::Jobs::IcalendarEnqueueSyncsForUrls.perform_async(urls)
+        status 202
+        present({o: "k"})
+      end
+    end
+  end
+end

data/lib/webhookdb/api/install.rb

@@ -13,6 +13,7 @@ class Webhookdb::API::Install < Webhookdb::API::V1
       def lookup_session!
         session = Webhookdb::Oauth::Session.usable.where(oauth_state: params[:state]).first
         error!("Forbidden", 302, {"Location" => "/v1/install/#{oauth_provider.key}/forbidden"}) if session.nil?
+        set_request_tags(session_id: session.id, session_state: session.oauth_state)
         return session
       end
 
@@ -73,7 +74,7 @@ class Webhookdb::API::Install < Webhookdb::API::V1
         def exchange_authorization_code(code)
           return oauth_provider.exchange_authorization_code(code:)
         rescue Webhookdb::Http::Error => e
-          logger.warn "oauth_exchange_error", exception: e
+          logger.warn "oauth_exchange_error", e
           url = "#{Webhookdb.api_url}/v1/install/#{oauth_provider.key}"
           raise FormError.new(
             "Something went wrong getting your access token from #{oauth_provider.app_name}. " \

data/lib/webhookdb/api/saved_queries.rb

@@ -15,6 +15,7 @@ class Webhookdb::API::SavedQueries < Webhookdb::API::V1
             # We can add other identifiers in the future
             cq = org.saved_queries_dataset[opaque_id: params[:query_identifier]]
             merror!(403, "There is no saved query with that identifier.") if cq.nil?
+            set_request_tags(saved_query_opaque_id: cq.opaque_id)
             return cq
           end
 

data/lib/webhookdb/api/saved_views.rb

@@ -13,6 +13,7 @@ class Webhookdb::API::SavedViews < Webhookdb::API::V1
             org = lookup_org!
             cq = org.saved_views_dataset[name: params[:name].strip]
             merror!(403, "There is no view with that name.") if cq.nil?
+            set_request_tags(saved_view_name: cq.name)
             return cq
           end
 

data/lib/webhookdb/api/service_integrations.rb

@@ -179,7 +179,7 @@ If the list does not look correct, you can contact support at #{Webhookdb.suppor
             begin
               svc.upsert_webhook_body(body)
             rescue KeyError, TypeError => e
-              self.logger.error "immediate_upsert", error: e
+              self.logger.error "immediate_upsert", e
               err_msg = "Sorry! Looks like something has gone wrong. " \
                         "Check your schema or contact support at #{Webhookdb.support_email}."
               merror!(400, err_msg)

data/lib/webhookdb/api/sync_targets.rb

@@ -151,6 +151,7 @@ class Webhookdb::API::SyncTargets < Webhookdb::API::V1
                   (stgt = org.all_sync_targets_dataset[Sequel[:sync_targets][:opaque_id] => params[:opaque_id]])
                   merror!(403, "There is no #{fullname} sync target with that id.") if
                     stgt.nil? || !stgt.send(predicate)
+                  set_request_tags(stgt.log_tags)
                   return stgt
                 end
               end
@@ -204,7 +205,6 @@ class Webhookdb::API::SyncTargets < Webhookdb::API::V1
                   )
                 end
 
-                stgt.logger.warn("destroying_sync_target", sync_target_id: stgt.id, customer_id: current_customer.id)
                 stgt.destroy
                 status 200
                 message = "#{fullname.capitalize} sync target has been removed and will no longer sync."

data/lib/webhookdb/api/system.rb

@@ -37,6 +37,11 @@ class Webhookdb::API::System < Webhookdb::Service
     }
   end
 
+  post :sink do
+    status 204
+    body ""
+  end
+
   if ["development", "test"].include?(Webhookdb::RACK_ENV)
     resource :debug do
       resource :echo do

data/lib/webhookdb/api/webhook_subscriptions.rb

@@ -52,6 +52,7 @@ class Webhookdb::API::WebhookSubscriptions < Webhookdb::API::V1
               org = lookup_org!
               whsub = org.all_webhook_subscriptions_dataset[opaque_id: params[:opaque_id]]
               merror!(403, "No webhook subscription with that ID exists in that organization.") if whsub.nil?
+              set_request_tags(webhook_subscription_id: whsub.id)
               return whsub
             end
           end

data/lib/webhookdb/api.rb

@@ -52,11 +52,14 @@ module Webhookdb::API
               memberships = customer.verified_memberships_dataset.where(organization: orgs).limit(2).all
               permission_error!("You don't have permissions with that organization.") if memberships.empty?
               merror!(500, "ambiguous", alert: true) if memberships.size > 1 # TODO: better message, tests
-              return memberships.first.organization
+              org = memberships.first.organization
+              set_request_tags(organization: org.key)
+              return org
             end
             raise "something went wrong" unless allow_connstr_auth
             org = Webhookdb::API::ConnstrAuth.find_authed(orgs, request)
             unauthenticated! if org.nil?
+            set_request_tags(organization: org.key)
             return org
           end
 

data/lib/webhookdb/apps.rb

@@ -15,6 +15,8 @@ require "webhookdb/service"
 require "webhookdb/api/auth"
 require "webhookdb/api/db"
 require "webhookdb/api/demo"
+require "webhookdb/api/error_handlers"
+require "webhookdb/api/icalproxy"
 require "webhookdb/api/install"
 require "webhookdb/api/me"
 require "webhookdb/api/organizations"
@@ -77,6 +79,8 @@ module Webhookdb::Apps
     mount Webhookdb::API::Auth
     mount Webhookdb::API::Db
     mount Webhookdb::API::Demo
+    mount Webhookdb::API::ErrorHandlers
+    mount Webhookdb::API::Icalproxy
     mount Webhookdb::API::Install
     mount Webhookdb::API::Me
     mount Webhookdb::API::Organizations

data/lib/webhookdb/async/autoscaler.rb

@@ -28,6 +28,7 @@ module Webhookdb::Async::Autoscaler
     setting :hostname_regex, /^web\.1$/, convert: ->(s) { Regexp.new(s) }
     setting :heroku_app_id_or_app_name, "", key: "HEROKU_APP_NAME"
     setting :heroku_formation_id_or_formation_type, "worker"
+    setting :sentry_alert_interval, 180
 
     after_configured do
       self._check_provider!
@@ -65,6 +66,7 @@ module Webhookdb::Async::Autoscaler
         latency_restored_threshold: self.latency_restored_threshold,
         latency_restored_handlers: [self.method(:scale_down)],
         log: ->(level, msg, kw={}) { self.logger.send(level, msg, kw) },
+        on_unhandled_exception: ->(e) { Sentry.capture_exception(e) },
       )
       return @instance.start
     end
@@ -78,10 +80,18 @@ module Webhookdb::Async::Autoscaler
       scale_action = @impl.scale_up(names_and_latencies, depth:, duration:, **)
       kw = {queues: names_and_latencies, depth:, duration:, scale_action:}
       self.logger.warn("high_latency_queues_event", **kw)
+      self._alert_sentry_latency(kw)
+    end
+
+    def _alert_sentry_latency(kw)
+      call_sentry = @last_called_sentry.nil? ||
+        @last_called_sentry < (Time.now - self.sentry_alert_interval)
+      return unless call_sentry
       Sentry.with_scope do |scope|
         scope&.set_extras(**kw)
         Sentry.capture_message("Some queues have a high latency")
       end
+      @last_called_sentry = Time.now
     end
 
     def scale_down(depth:, duration:, **)

data/lib/webhookdb/async/job.rb

@@ -14,5 +14,9 @@ module Webhookdb::Async::Job
     def with_log_tags(tags, &)
       Webhookdb::Async::JobLogger.with_log_tags(tags, &)
     end
+
+    def set_job_tags(tags)
+      Webhookdb::Async::JobLogger.set_job_tags(**tags)
+    end
   end
 end

data/lib/webhookdb/async/scheduled_job.rb

@@ -14,5 +14,9 @@ module Webhookdb::Async::ScheduledJob
     def with_log_tags(tags, &)
       Webhookdb::Async::JobLogger.with_log_tags(tags, &)
     end
+
+    def set_job_tags(**tags)
+      Webhookdb::Async::JobLogger.set_job_tags(**tags)
+    end
   end
 end

data/lib/webhookdb/async.rb

@@ -2,6 +2,7 @@
 
 require "amigo/retry"
 require "amigo/durable_job"
+require "amigo/job_in_context"
 require "amigo/rate_limited_error_handler"
 require "appydays/configurable"
 require "appydays/loggable"
@@ -62,6 +63,7 @@ module Webhookdb::Async
           ttl: self.error_reporting_ttl,
         )
         config.death_handlers << Webhookdb::Async::JobLogger.method(:death_handler)
+        config.server_middleware.add(Amigo::JobInContext::ServerMiddleware)
         config.server_middleware.add(Amigo::DurableJob::ServerMiddleware)
         # We use the dead set to move jobs that we need to retry manually
         config.options[:dead_max_jobs] = 999_999_999

data/lib/webhookdb/backfiller.rb

@@ -88,16 +88,29 @@ class Webhookdb::Backfiller
       return k, inserting
     end
 
+    # Return the conditional update expression.
+    # Usually this is:
+    # - +nil+ if +conditional_upsert?+ is false.
+    # - the +_update_where_expr+ if +conditional_upsert?+ is true.
+    # - Can be overridden by a subclass if they need to use a specific conditional update expression
+    #   in certain cases (should be rare).
+    def update_where_expr = self.conditional_upsert? ? self.upserting_replicator._update_where_expr : nil
+
+    # The upsert 'UPDATE' expression, calculated using the first row of a multi-row upsert.
+    # Defaults to +_upsert_update_expr+, but may need to be overridden in rare cases.
+    def upsert_update_expr(first_inserting_row) = self.upserting_replicator._upsert_update_expr(first_inserting_row)
+
     def flush_pending_inserts
       return if self.dry_run?
       return if self.pending_inserts.empty?
       rows_to_insert = self.pending_inserts.values
-      update_where = self.conditional_upsert? ? self.upserting_replicator._update_where_expr : nil
+      update_where_expr = self.update_where_expr
+      update_expr = self.upserting_replicator._upsert_update_expr(rows_to_insert.first)
       self.upserting_replicator.admin_dataset(timeout: :fast) do |ds|
         insert_ds = ds.insert_conflict(
-          target: self.upserting_replicator._remote_key_column.name,
-          update: self.upserting_replicator._upsert_update_expr(rows_to_insert.first),
-          update_where:,
+          target: self.upserting_replicator._upsert_conflict_target,
+          update: update_expr,
+          update_where: update_where_expr,
         )
         insert_ds.multi_insert(rows_to_insert)
       end