RubyGems - webhookdb - Versions diffs - 1.3.1 → 1.5.0 - Mend

webhookdb 1.3.1 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (164) hide show

checksums.yaml +4 -4
data/admin-dist/assets/{index-6aebf805.js → index-9306dd28.js} +39 -39
data/admin-dist/index.html +1 -1
data/data/messages/templates/errors/generic_backfill.email.liquid +30 -0
data/data/messages/templates/errors/icalendar_fetch.email.liquid +8 -2
data/data/messages/templates/specs/with_fields.email.liquid +6 -0
data/db/migrations/026_undo_integration_backfill_cursor.rb +2 -0
data/db/migrations/032_remove_db_defaults.rb +2 -0
data/db/migrations/043_text_search.rb +2 -0
data/db/migrations/045_system_log.rb +15 -0
data/db/migrations/046_indices.rb +14 -0
data/db/migrations/047_sync_parallelism.rb +9 -0
data/db/migrations/048_sync_stats.rb +9 -0
data/db/migrations/049_error_handlers.rb +18 -0
data/db/migrations/050_logged_webhook_indices.rb +25 -0
data/db/migrations/051_partitioning.rb +9 -0
data/integration/async_spec.rb +0 -2
data/integration/service_integrations_spec.rb +0 -2
data/lib/amigo/durable_job.rb +2 -2
data/lib/amigo/job_in_context.rb +12 -0
data/lib/webhookdb/admin.rb +6 -0
data/lib/webhookdb/admin_api/data_provider.rb +1 -0
data/lib/webhookdb/admin_api/entities.rb +8 -0
data/lib/webhookdb/aggregate_result.rb +1 -1
data/lib/webhookdb/api/entities.rb +6 -2
data/lib/webhookdb/api/error_handlers.rb +104 -0
data/lib/webhookdb/api/helpers.rb +25 -1
data/lib/webhookdb/api/icalproxy.rb +22 -0
data/lib/webhookdb/api/install.rb +2 -1
data/lib/webhookdb/api/organizations.rb +6 -0
data/lib/webhookdb/api/saved_queries.rb +1 -0
data/lib/webhookdb/api/saved_views.rb +1 -0
data/lib/webhookdb/api/service_integrations.rb +2 -1
data/lib/webhookdb/api/sync_targets.rb +1 -1
data/lib/webhookdb/api/system.rb +5 -0
data/lib/webhookdb/api/webhook_subscriptions.rb +1 -0
data/lib/webhookdb/api.rb +4 -1
data/lib/webhookdb/apps.rb +4 -0
data/lib/webhookdb/async/autoscaler.rb +10 -0
data/lib/webhookdb/async/job.rb +4 -0
data/lib/webhookdb/async/scheduled_job.rb +4 -0
data/lib/webhookdb/async.rb +2 -0
data/lib/webhookdb/backfiller.rb +17 -4
data/lib/webhookdb/concurrent.rb +96 -0
data/lib/webhookdb/connection_cache.rb +57 -10
data/lib/webhookdb/console.rb +1 -1
data/lib/webhookdb/customer/reset_code.rb +1 -1
data/lib/webhookdb/customer.rb +5 -4
data/lib/webhookdb/database_document.rb +1 -1
data/lib/webhookdb/db_adapter/default_sql.rb +1 -14
data/lib/webhookdb/db_adapter/partition.rb +14 -0
data/lib/webhookdb/db_adapter/partitioning.rb +8 -0
data/lib/webhookdb/db_adapter/pg.rb +77 -5
data/lib/webhookdb/db_adapter/snowflake.rb +15 -6
data/lib/webhookdb/db_adapter.rb +25 -3
data/lib/webhookdb/dbutil.rb +2 -0
data/lib/webhookdb/errors.rb +34 -0
data/lib/webhookdb/fixtures/logged_webhooks.rb +4 -0
data/lib/webhookdb/fixtures/organization_error_handlers.rb +20 -0
data/lib/webhookdb/http.rb +30 -16
data/lib/webhookdb/icalendar.rb +30 -9
data/lib/webhookdb/jobs/amigo_test_jobs.rb +1 -1
data/lib/webhookdb/jobs/backfill.rb +21 -25
data/lib/webhookdb/jobs/create_mirror_table.rb +3 -4
data/lib/webhookdb/jobs/deprecated_jobs.rb +3 -0
data/lib/webhookdb/jobs/emailer.rb +2 -1
data/lib/webhookdb/jobs/front_signalwire_message_channel_sync_inbound.rb +15 -0
data/lib/webhookdb/jobs/icalendar_delete_stale_cancelled_events.rb +7 -2
data/lib/webhookdb/jobs/icalendar_enqueue_syncs.rb +74 -11
data/lib/webhookdb/jobs/icalendar_enqueue_syncs_for_urls.rb +22 -0
data/lib/webhookdb/jobs/icalendar_sync.rb +21 -9
data/lib/webhookdb/jobs/increase_event_handler.rb +3 -2
data/lib/webhookdb/jobs/{logged_webhook_replay.rb → logged_webhooks_replay.rb} +5 -3
data/lib/webhookdb/jobs/message_dispatched.rb +1 -0
data/lib/webhookdb/jobs/model_event_system_log_tracker.rb +112 -0
data/lib/webhookdb/jobs/monitor_metrics.rb +29 -0
data/lib/webhookdb/jobs/organization_database_migration_notify.rb +32 -0
data/lib/webhookdb/jobs/organization_database_migration_run.rb +4 -6
data/lib/webhookdb/jobs/organization_error_handler_dispatch.rb +26 -0
data/lib/webhookdb/jobs/prepare_database_connections.rb +1 -0
data/lib/webhookdb/jobs/process_webhook.rb +11 -12
data/lib/webhookdb/jobs/renew_watch_channel.rb +10 -10
data/lib/webhookdb/jobs/replication_migration.rb +5 -2
data/lib/webhookdb/jobs/reset_code_create_dispatch.rb +1 -2
data/lib/webhookdb/jobs/scheduled_backfills.rb +2 -2
data/lib/webhookdb/jobs/send_invite.rb +3 -2
data/lib/webhookdb/jobs/send_test_webhook.rb +1 -3
data/lib/webhookdb/jobs/send_webhook.rb +4 -5
data/lib/webhookdb/jobs/stale_row_deleter.rb +31 -0
data/lib/webhookdb/jobs/sync_target_enqueue_scheduled.rb +3 -0
data/lib/webhookdb/jobs/sync_target_run_sync.rb +9 -15
data/lib/webhookdb/jobs/{webhook_subscription_delivery_attempt.rb → webhook_subscription_delivery_event.rb} +5 -8
data/lib/webhookdb/liquid/expose.rb +1 -1
data/lib/webhookdb/liquid/filters.rb +1 -1
data/lib/webhookdb/liquid/partial.rb +2 -2
data/lib/webhookdb/logged_webhook/resilient.rb +3 -3
data/lib/webhookdb/logged_webhook.rb +16 -2
data/lib/webhookdb/message/email_transport.rb +1 -1
data/lib/webhookdb/message/transport.rb +1 -1
data/lib/webhookdb/message.rb +55 -4
data/lib/webhookdb/messages/error_generic_backfill.rb +47 -0
data/lib/webhookdb/messages/error_icalendar_fetch.rb +5 -0
data/lib/webhookdb/messages/error_signalwire_send_sms.rb +2 -0
data/lib/webhookdb/messages/specs.rb +16 -0
data/lib/webhookdb/organization/alerting.rb +56 -6
data/lib/webhookdb/organization/database_migration.rb +2 -2
data/lib/webhookdb/organization/db_builder.rb +5 -4
data/lib/webhookdb/organization/error_handler.rb +141 -0
data/lib/webhookdb/organization.rb +76 -10
data/lib/webhookdb/postgres/model.rb +1 -0
data/lib/webhookdb/postgres/model_utilities.rb +2 -0
data/lib/webhookdb/postgres.rb +3 -4
data/lib/webhookdb/replicator/base.rb +202 -68
data/lib/webhookdb/replicator/base_stale_row_deleter.rb +165 -0
data/lib/webhookdb/replicator/column.rb +2 -0
data/lib/webhookdb/replicator/email_octopus_contact_v1.rb +0 -1
data/lib/webhookdb/replicator/fake.rb +106 -88
data/lib/webhookdb/replicator/front_signalwire_message_channel_app_v1.rb +131 -61
data/lib/webhookdb/replicator/github_repo_v1_mixin.rb +17 -0
data/lib/webhookdb/replicator/icalendar_calendar_v1.rb +197 -32
data/lib/webhookdb/replicator/icalendar_event_v1.rb +20 -44
data/lib/webhookdb/replicator/icalendar_event_v1_partitioned.rb +33 -0
data/lib/webhookdb/replicator/intercom_contact_v1.rb +1 -0
data/lib/webhookdb/replicator/intercom_conversation_v1.rb +1 -0
data/lib/webhookdb/replicator/intercom_v1_mixin.rb +49 -6
data/lib/webhookdb/replicator/partitionable_mixin.rb +116 -0
data/lib/webhookdb/replicator/shopify_v1_mixin.rb +1 -1
data/lib/webhookdb/replicator/signalwire_message_v1.rb +31 -1
data/lib/webhookdb/replicator/sponsy_v1_mixin.rb +1 -1
data/lib/webhookdb/replicator/transistor_episode_stats_v1.rb +0 -1
data/lib/webhookdb/replicator/transistor_episode_v1.rb +11 -5
data/lib/webhookdb/replicator/webhook_request.rb +8 -0
data/lib/webhookdb/replicator.rb +6 -3
data/lib/webhookdb/service/helpers.rb +4 -0
data/lib/webhookdb/service/middleware.rb +6 -2
data/lib/webhookdb/service/view_api.rb +1 -1
data/lib/webhookdb/service.rb +10 -10
data/lib/webhookdb/service_integration.rb +19 -1
data/lib/webhookdb/signalwire.rb +1 -1
data/lib/webhookdb/spec_helpers/async.rb +0 -4
data/lib/webhookdb/spec_helpers/sentry.rb +32 -0
data/lib/webhookdb/spec_helpers/shared_examples_for_replicators.rb +239 -64
data/lib/webhookdb/spec_helpers.rb +1 -0
data/lib/webhookdb/sync_target.rb +202 -34
data/lib/webhookdb/system_log_event.rb +9 -0
data/lib/webhookdb/tasks/admin.rb +1 -1
data/lib/webhookdb/tasks/annotate.rb +1 -1
data/lib/webhookdb/tasks/db.rb +13 -1
data/lib/webhookdb/tasks/docs.rb +1 -1
data/lib/webhookdb/tasks/fixture.rb +1 -1
data/lib/webhookdb/tasks/message.rb +1 -1
data/lib/webhookdb/tasks/regress.rb +1 -1
data/lib/webhookdb/tasks/release.rb +1 -1
data/lib/webhookdb/tasks/sidekiq.rb +1 -1
data/lib/webhookdb/tasks/specs.rb +1 -1
data/lib/webhookdb/version.rb +1 -1
data/lib/webhookdb/webhook_subscription.rb +3 -4
data/lib/webhookdb.rb +34 -8
metadata +114 -64
data/lib/webhookdb/jobs/customer_created_notify_internal.rb +0 -22
data/lib/webhookdb/jobs/organization_database_migration_notify_finished.rb +0 -21
data/lib/webhookdb/jobs/organization_database_migration_notify_started.rb +0 -21
/data/lib/webhookdb/jobs/{logged_webhook_resilient_replay.rb → logged_webhooks_resilient_replay.rb} +0 -0
/data/lib/webhookdb/jobs/{webhook_resource_notify_integrations.rb → webhookdb_resource_notify_integrations.rb} +0 -0

data/admin-dist/index.html CHANGED Viewed

@@ -115,7 +115,7 @@
         </style>
         <link rel="preconnect" href="https://fonts.gstatic.com" />
         <link href="https://fonts.googleapis.com/css2?family=Open+Sans:wght@300;500;600;700&display=swap" rel="stylesheet">
-      <script type="module" crossorigin src="/admin/assets/index-6aebf805.js"></script>
+      <script type="module" crossorigin src="/admin/assets/index-9306dd28.js"></script>
     </head>
     <body>

data/data/messages/templates/errors/generic_backfill.email.liquid ADDED Viewed

@@ -0,0 +1,30 @@
+{% expose subject %}WebhookDB: {{ friendly_name }} Error{% endexpose %}
+{% partial 'greeting' %}
+<p>
+    WebhookDB encountered an error trying to sync {{ friendly_name }} data using your configured credentials. We have the following information about the failure:
+</p>
+<ul>
+    <li>Service Integration Name: {{ service_name }}, ID: <code>{{ opaque_id }}</code></li>
+    <li>Request: <code>{{ request_method }} {{ request_url }}</code></li>
+    <li>Response Status: <code>{{ response_status }}</code></li>
+    <li>Body: <code>{{  response_body }}</code></li>
+</ul>
+<p>
+    Usually this indicates an API key has been revoked or some other misconfiguration, rather than being an error in WebhookDB itself.
+</p>
+<p>To fix this integration, head over to <a href="{{ app_url }}">{{ app_url }}</a>, log in, and run:</p>
+<pre>
+    webhookdb integrations reset {{ opaque_id }}
+</pre>
+<p>
+    If you no longer wish to use this integration, it can be deleted using:
+</p>
+<pre>
+    webhookdb integrations delete {{ opaque_id }}
+</pre>
+<p>We'll continue to send daily emails when this happens, so please do fix this up when you get a chance.</p>
+<p>Please file an issue at {{ oss_repo }} or email <a href="mailto:{{ support_email }}">{{ support_email }}</a> if you need any help.</p>
+{% partial 'signoff' %}

data/data/messages/templates/errors/icalendar_fetch.email.liquid CHANGED Viewed

@@ -1,4 +1,4 @@
-{% expose subject %}WebhookDB: ICalendar Error{% endexpose %}
+ {% expose subject %}WebhookDB: ICalendar Error{% endexpose %}
 {% partial 'greeting' %}
@@ -6,8 +6,9 @@
     WebhookDB encountered an error syncing an ICalendar feed. We have the following information about the failure:
 </p>
 <ul>
+    <li>Organization: {{ org_name }} (key: <code>{{ org_key }}</code>)</li>
     <li>Service Integration Name: {{ service_name }}, ID: <code>{{ opaque_id }}</code></li>
-    <li>Calendar ID (you send this to us when adding the calendar): <code>{{ external_calendar_id }}</code></li>
+    <li>Calendar ID (you sent this when adding the calendar): <code>{{ external_calendar_id }}</code></li>
     <li>Request: <code>{{ request_method }} {{ request_url }}</code></li>
     <li>Response Status: <code>{{ response_status }}</code></li>
     <li>Body: <code>{{  response_body }}</code></li>
@@ -22,7 +23,12 @@
 <p>
     If this calendar is no longer shared, it should be removed.
     Use a DELETE request, as per <a href="{{ docs_url }}/guides/icalendar/#delete">{{ docs_url }}/guides/icalendar/#delete</a>.
+    For example, here is the cURL to run to delete this from the shell:
 </p>
+<pre>
+    $ export WHDB_WEBHOOK_SECRET=`webhookdb integration info --org={{ org_key }} --field=webhook_secret {{ opaque_id }}`
+    $ curl -X POST -d '{"type":"DELETE","external_id":"{{ external_calendar_id }}"}' -H "Content-Type: application/json" -H "Whdb-Webhook-Secret: ${WHDB_WEBHOOK_SECRET}" "{{ webhook_endpoint }}"
+</pre>
 <p>We'll continue to send daily emails when this happens, so please do fix this up when you get a chance.</p>
 <p>Please file an issue at {{ oss_repo }} if you need any help.</p>

data/data/messages/templates/specs/with_fields.email.liquid ADDED Viewed

@@ -0,0 +1,6 @@
+{% expose subject %}subject with multiple fields{% endexpose %}
+a: {{ a }}
+b: {{ b }}
+c: {{ c }}
+d: {{ d }}
+e: {{ e }}

data/db/migrations/026_undo_integration_backfill_cursor.rb CHANGED Viewed

@@ -2,8 +2,10 @@
 Sequel.migration do
   change do
+    # rubocop:disable Sequel/IrreversibleMigration
     alter_table(:service_integrations) do
       drop_column :backfill_cursor
     end
+    # rubocop:enable Sequel/IrreversibleMigration
   end
 end

data/db/migrations/032_remove_db_defaults.rb CHANGED Viewed

@@ -2,11 +2,13 @@
 Sequel.migration do
   change do
+    # rubocop:disable Sequel/IrreversibleMigration
     alter_table(:sync_targets) do
       set_column_default :page_size, nil
     end
     alter_table(:organizations) do
       set_column_default :minimum_sync_seconds, nil
     end
+    # rubocop:enable Sequel/IrreversibleMigration
   end
 end

data/db/migrations/043_text_search.rb CHANGED Viewed

@@ -19,10 +19,12 @@ Sequel.migration do
       :webhook_subscription_deliveries,
       :webhook_subscriptions,
     ]
+    # rubocop:disable Sequel/IrreversibleMigration
     searchable.each do |tbl|
       alter_table(tbl) do
         add_column :text_search, :tsvector
       end
     end
+    # rubocop:enable Sequel/IrreversibleMigration
   end
 end

data/db/migrations/045_system_log.rb ADDED Viewed

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+Sequel.migration do
+  change do
+    create_table(:system_log_events) do
+      primary_key :id, type: :bigserial
+      timestamptz :at, null: false, default: Sequel.function(:now), index: true
+      text :title, null: false, default: ""
+      text :body, null: false, default: ""
+      text :link, null: false, default: ""
+      foreign_key :actor_id, :customers
+      column :text_search, :tsvector
+    end
+  end
+end

data/db/migrations/046_indices.rb ADDED Viewed

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+Sequel.migration do
+  change do
+    alter_table(:service_integrations) do
+      add_index :depends_on_id
+      add_index :service_name
+    end
+    alter_table(:message_deliveries) do
+      add_index :soft_deleted_at
+    end
+  end
+end

data/db/migrations/047_sync_parallelism.rb ADDED Viewed

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+Sequel.migration do
+  change do
+    alter_table(:sync_targets) do
+      add_column :parallelism, :integer, null: false, default: 0
+    end
+  end
+end

data/db/migrations/048_sync_stats.rb ADDED Viewed

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+Sequel.migration do
+  change do
+    alter_table(:sync_targets) do
+      add_column :sync_stats, :jsonb, null: false, default: "[]"
+    end
+  end
+end

data/db/migrations/049_error_handlers.rb ADDED Viewed

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+Sequel.migration do
+  change do
+    create_table(:organization_error_handlers) do
+      primary_key :id
+      timestamptz :created_at, null: false, default: Sequel.function(:now)
+      timestamptz :updated_at
+      text :opaque_id, null: false, unique: true
+      foreign_key :organization_id, :organizations, null: false
+      foreign_key :created_by_id, :customers, null: true, on_delete: :set_null
+      text :url, null: false
+    end
+  end
+end

data/db/migrations/050_logged_webhook_indices.rb ADDED Viewed

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+Sequel.migration do
+  no_transaction
+  up do
+    # These are optimized for Webhookdb::LoggedWebhook.trim. See code for details/coupling.
+    run "CREATE INDEX CONCURRENTLY logged_webhooks_trim_unowned_idx ON logged_webhooks(inserted_at) " \
+        "WHERE organization_id IS NULL"
+    run "CREATE INDEX CONCURRENTLY logged_webhooks_trim_success_idx ON logged_webhooks(inserted_at) " \
+        "WHERE organization_id IS NOT NULL AND response_status < 400 AND truncated_at IS NULL"
+    run "CREATE INDEX CONCURRENTLY logged_webhooks_delete_success_idx ON logged_webhooks(inserted_at) " \
+        "WHERE organization_id IS NOT NULL AND response_status < 400 AND truncated_at IS NOT NULL"
+    run "CREATE INDEX CONCURRENTLY logged_webhooks_trim_failures_idx ON logged_webhooks(inserted_at) " \
+        "WHERE organization_id IS NOT NULL AND response_status >= 400 AND truncated_at IS NULL"
+    run "CREATE INDEX CONCURRENTLY logged_webhooks_delete_failures_idx ON logged_webhooks(inserted_at) " \
+        "WHERE organization_id IS NOT NULL AND response_status >= 400 AND truncated_at IS NOT NULL"
+  end
+  down do
+    run "DROP INDEX logged_webhooks_trim_unowned_idx"
+    run "DROP INDEX logged_webhooks_trim_success_idx"
+    run "DROP INDEX logged_webhooks_delete_success_idx"
+    run "DROP INDEX logged_webhooks_trim_failures_idx"
+    run "DROP INDEX logged_webhooks_delete_failures_idx"
+  end
+end

data/db/migrations/051_partitioning.rb ADDED Viewed

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+Sequel.migration do
+  change do
+    alter_table(:service_integrations) do
+      add_column :partition_value, :int, default: 0, null: false
+    end
+  end
+end

data/integration/async_spec.rb CHANGED Viewed

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
-require "rspec/eventually"
 require "webhookdb/async"
 RSpec.describe "async workers", :integration do

data/integration/service_integrations_spec.rb CHANGED Viewed

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
-require "rspec/eventually"
 RSpec.describe "service integrations", :integration do
   def catch_missing_db(default)
     yield

data/lib/amigo/durable_job.rb CHANGED Viewed

@@ -351,8 +351,8 @@ module Amigo::DurableJob
     # Space-separate multiple env vars.
     setting :server_env_vars, ["DATABASE_URL"], convert: ->(s) { s.split.map(&:strip) }
-    setting :schema_name, :public, convert: ->(s) { s.to_sym }
-    setting :table_name, :durable_jobs, convert: ->(s) { s.to_sym }
+    setting :schema_name, :public, convert: lambda(&:to_sym)
+    setting :table_name, :durable_jobs, convert: lambda(&:to_sym)
     after_configured do
       self.storage_database_urls = self.server_urls.dup

data/lib/amigo/job_in_context.rb ADDED Viewed

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+module Amigo
+end
+module Amigo::JobInContext
+  class ServerMiddleware
+    def call(worker, job, queue, &)
+      Sidekiq::Context.with(worker:, queue:, job_hash: job, &)
+    end
+  end
+end

data/lib/webhookdb/admin.rb CHANGED Viewed

@@ -1,4 +1,10 @@
 # frozen_string_literal: true
 module Webhookdb::Admin
+  module Linked
+    protected def _admin_datatype = self.class.dataset.first_source_table
+    protected def _admin_id = self.pk
+    protected def _admin_display = "show"
+    def admin_link = "#{Webhookdb.admin_url}/admin#/#{_admin_datatype}/#{_admin_id}/#{_admin_display}"
+  end
 end

data/lib/webhookdb/admin_api/data_provider.rb CHANGED Viewed

@@ -29,6 +29,7 @@ class Webhookdb::AdminAPI::DataProvider < Webhookdb::AdminAPI::V1
     service_integrations: [Webhookdb::ServiceIntegration, ServiceIntegration],
     subscriptions: [Webhookdb::Subscription, Subscription],
     sync_targets: [Webhookdb::SyncTarget, SyncTarget],
+    system_log_events: [Webhookdb::SystemLogEvent, SystemLogEvent],
     webhook_subscriptions: [Webhookdb::WebhookSubscription, WebhookSubscription],
     webhook_subscription_deliveries: [Webhookdb::WebhookSubscription::Delivery, WebhookSubscriptionDelivery],
   }.freeze

data/lib/webhookdb/admin_api/entities.rb CHANGED Viewed

@@ -163,6 +163,14 @@ module Webhookdb::AdminAPI::Entities
     expose :page_size
   end
+  class SystemLogEvent < Base
+    expose :at
+    expose :title
+    expose :body
+    expose :link
+    expose :actor_id
+  end
   class WebhookSubscription < Base
     expose :organization, with: Related
     expose :service_integration, with: Related

data/lib/webhookdb/aggregate_result.rb CHANGED Viewed

@@ -18,7 +18,7 @@ require "webhookdb" unless defined?(Webhookdb)
 #   end
 #   return ag.finish
 #
-class Webhookdb::AggregateResult < StandardError
+class Webhookdb::AggregateResult < Webhookdb::WebhookdbError
   attr_reader :successes, :failures, :errors
   def initialize(existing=nil)

data/lib/webhookdb/api/entities.rb CHANGED Viewed

@@ -136,10 +136,12 @@ module Webhookdb::API
     expose :opaque_id
     expose :service_integration, with: ServiceIntegrationEntity
     expose :period_seconds
+    expose :page_size
     expose :displaysafe_connection_url, as: :connection_url
     expose :table
     expose :schema
     expose :last_synced_at
+    expose :latency, &self.delegate_to(:latency, :to_i)
     expose :associated_type
     expose :associated_id
     expose :associated_object_display
@@ -155,8 +157,9 @@ module Webhookdb::API
         [:associated_object_display, "Associated"],
         [:schema_and_table_string, "Table"],
         [:last_synced_at, "Last Synced"],
+        [:latency, "Latency"],
         [:period_seconds, "Period"],
-        [:page_size, "Page"],
+        [:page_size, "Page Size"],
       ]
     end
   end
@@ -168,8 +171,9 @@ module Webhookdb::API
         [:connection_url, "URL"],
         [:associated_object_display, "Associated"],
         [:last_synced_at, "Last Synced"],
+        [:latency, "Latency"],
         [:period_seconds, "Period"],
-        [:page_size, "Page"],
+        [:page_size, "Page Size"],
       ]
     end
   end

data/lib/webhookdb/api/error_handlers.rb ADDED Viewed

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+require "webhookdb/api"
+class Webhookdb::API::ErrorHandlers < Webhookdb::API::V1
+  include Webhookdb::Service::Types
+  CREATE_PROMPT = "What is the URL that WebhookDB should POST to when a replicator fails? " \
+                  "See #{Webhookdb::Organization::ErrorHandler::DOCS_URL} for more information:".freeze
+  resource :organizations do
+    route_param :org_identifier do
+      resource :error_handlers do
+        helpers do
+          def lookup!
+            org = lookup_org!
+            eh = org.error_handlers_dataset[opaque_id: params[:opaque_id].strip]
+            merror!(403, "There is no error handler with that id.") if eh.nil?
+            set_request_tags(error_handler_opaque_id: eh.opaque_id)
+            return eh
+          end
+          def guard_editable!(customer, org)
+            return if has_admin?(org, customer:)
+            permission_error!("You must be an org admin to modify error handlers.")
+          end
+          def url_valid?(u)
+            begin
+              uri = URI(u)
+            rescue URI::InvalidURIError
+              return false
+            end
+            return true if uri.scheme == "sentry"
+            return false unless uri.scheme&.start_with?("http")
+            return true
+          end
+        end
+        desc "Returns a list of all error handlers associated with the org."
+        get do
+          handlers = lookup_org!.error_handlers
+          message = ""
+          if handlers.empty?
+            message = "This organization doesn't have any error handlers yet.\n" \
+                      "Use `webhookdb error-handler create` to set one up."
+          end
+          present_collection handlers, with: ErrorHandlerEntity, message:
+        end
+        desc "Creates a new error handler."
+        params do
+          optional :url, type: TrimmedString, prompt: CREATE_PROMPT
+        end
+        post :create do
+          cust = current_customer
+          org = lookup_org!
+          guard_editable!(cust, org)
+          unless url_valid?(params[:url])
+            msg = "URL is malformed. It should be a URL like https://foo.bar/path, http://foo.bar:123, etc. " \
+                  "See #{Webhookdb::Organization::ErrorHandler::DOCS_URL} for more info."
+            merror!(400, msg)
+          end
+          eh = Webhookdb::Organization::ErrorHandler.create(
+            organization: org,
+            url: params[:url],
+            created_by: cust,
+          )
+          message = "Whenever one of your replicators errors, WebhookDB will alert the given URL. " \
+                    "See #{Webhookdb::Organization::ErrorHandler::DOCS_URL} for more information."
+          status 200
+          present eh, with: ErrorHandlerEntity, message:
+        end
+        route_param :opaque_id, type: String do
+          get do
+            eh = lookup!
+            status 200
+            present eh, with: ErrorHandlerEntity
+          end
+          post :delete do
+            customer = current_customer
+            eh = lookup!
+            guard_editable!(customer, eh.organization)
+            eh.destroy
+            status 200
+            present eh, with: ErrorHandlerEntity,
+                        message: "You have successfully deleted the error handler '#{eh.opaque_id}'."
+          end
+        end
+      end
+    end
+  end
+  class ErrorHandlerEntity < Webhookdb::API::BaseEntity
+    expose :opaque_id, as: :id
+    expose :url
+    def self.display_headers
+      return [[:id, "ID"], [:url, "Url"]]
+    end
+  end
+end

data/lib/webhookdb/api/helpers.rb CHANGED Viewed

@@ -122,7 +122,14 @@ module Webhookdb::API::Helpers
     sints = org.service_integrations_dataset.
       where(Sequel[service_name: identifier] | Sequel[table_name: identifier] | Sequel[opaque_id: identifier]).
       limit(2).all
-    return sints.first if sints.size == 1
+    if sints.size == 1
+      sint = sints.first
+      set_request_tags(
+        service_integration_service_name: sint.service_name,
+        service_integration_table_name: sint.table_name,
+      )
+      return sint
+    end
     merror!(403, "There is no service integration with that identifier.") if sints.empty?
     dupe_attr = nil
     alternative = nil
@@ -171,6 +178,23 @@ module Webhookdb::API::Helpers
       sint = yield
       return if sint == :pass
       raise "error instead of return nil if there is no service integration" if sint.nil?
+      # If this is a valid GET request (ie, the replicator doesn't do useful auth),
+      # and it's from a bot, we want to block it, otherwise we'd process invalid webhooks.
+      #
+      # This is almost never a problem, because almost all services perform some validation,
+      # and the auth info isn't in a GET URL; but in some cases, that won't be the case,
+      # and we protect against it here.
+      #
+      # An example would be a bot getting a link preview.
+      if request.get? && !request.GET["skipbotcheck"] && Browser.new(request.user_agent).bot?
+        # IMPORTANT: Run the Browser code last since it has to run a bunch of regexes to detect a bot.
+        env["api.format"] = :binary
+        header "Content-Type", "text/plain"
+        body("This route is for receiving webhooks and HTTP calls, not for bots. " \
+             "Call this endpoint with the query param 'skipbotcheck=true' to bypass this check.")
+        status 403
+        return
+      end
       opaque_id = sint.opaque_id
       organization_id = sint.organization_id
       svc = Webhookdb::Replicator.create(sint).dispatch_request_to(request)

data/lib/webhookdb/api/icalproxy.rb ADDED Viewed

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+require "webhookdb/api"
+require "webhookdb/jobs/icalendar_enqueue_syncs_for_urls"
+class Webhookdb::API::Icalproxy < Webhookdb::API::V1
+  resource :icalproxy do
+    resource :webhook do
+      post do
+        merror!(402, "Api key not configured") unless Webhookdb::Icalendar.proxy_api_key.present?
+        request_key = request.env["HTTP_AUTHORIZATION"] || ""
+        configured_key = "Apikey #{Webhookdb::Icalendar.proxy_api_key}"
+        verified = request_key && ActiveSupport::SecurityUtils.secure_compare(request_key, configured_key)
+        merror!(401, "Invalid api key") unless verified
+        urls = params.fetch(:urls)
+        Webhookdb::Jobs::IcalendarEnqueueSyncsForUrls.perform_async(urls)
+        status 202
+        present({o: "k"})
+      end
+    end
+  end
+end

data/lib/webhookdb/api/install.rb CHANGED Viewed

@@ -13,6 +13,7 @@ class Webhookdb::API::Install < Webhookdb::API::V1
       def lookup_session!
         session = Webhookdb::Oauth::Session.usable.where(oauth_state: params[:state]).first
         error!("Forbidden", 302, {"Location" => "/v1/install/#{oauth_provider.key}/forbidden"}) if session.nil?
+        set_request_tags(session_id: session.id, session_state: session.oauth_state)
         return session
       end
@@ -73,7 +74,7 @@ class Webhookdb::API::Install < Webhookdb::API::V1
         def exchange_authorization_code(code)
           return oauth_provider.exchange_authorization_code(code:)
         rescue Webhookdb::Http::Error => e
-          logger.warn "oauth_exchange_error", exception: e
+          logger.warn "oauth_exchange_error", e
           url = "#{Webhookdb.api_url}/v1/install/#{oauth_provider.key}"
           raise FormError.new(
             "Something went wrong getting your access token from #{oauth_provider.app_name}. " \

data/lib/webhookdb/api/organizations.rb CHANGED Viewed

@@ -206,6 +206,12 @@ class Webhookdb::API::Organizations < Webhookdb::API::V1
             {title: "Customer", value: "(#{c.id}) #{c.email}", short: false},
           ],
         ).emit
+        Webhookdb::SystemLogEvent.create(
+          title: "Organization Closure Requested",
+          body: "#{org.name} (#{org.key})",
+          link: org.admin_link,
+          actor: c,
+        )
         step = Webhookdb::Replicator::StateMachineStep.new.completed
         step.output = "Thanks! We've received the request to close your #{org.name} organization. " \
                       "We'll be in touch within 2 business days confirming removal."

data/lib/webhookdb/api/saved_queries.rb CHANGED Viewed

@@ -15,6 +15,7 @@ class Webhookdb::API::SavedQueries < Webhookdb::API::V1
             # We can add other identifiers in the future
             cq = org.saved_queries_dataset[opaque_id: params[:query_identifier]]
             merror!(403, "There is no saved query with that identifier.") if cq.nil?
+            set_request_tags(saved_query_opaque_id: cq.opaque_id)
             return cq
           end

data/lib/webhookdb/api/saved_views.rb CHANGED Viewed

@@ -13,6 +13,7 @@ class Webhookdb::API::SavedViews < Webhookdb::API::V1
             org = lookup_org!
             cq = org.saved_views_dataset[name: params[:name].strip]
             merror!(403, "There is no view with that name.") if cq.nil?
+            set_request_tags(saved_view_name: cq.name)
             return cq
           end

data/lib/webhookdb/api/service_integrations.rb CHANGED Viewed

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
+require "browser"
 require "grape"
 require "oj"
@@ -178,7 +179,7 @@ If the list does not look correct, you can contact support at #{Webhookdb.suppor
             begin
               svc.upsert_webhook_body(body)
             rescue KeyError, TypeError => e
-              self.logger.error "immediate_upsert", error: e
+              self.logger.error "immediate_upsert", e
               err_msg = "Sorry! Looks like something has gone wrong. " \
                         "Check your schema or contact support at #{Webhookdb.support_email}."
               merror!(400, err_msg)

data/lib/webhookdb/api/sync_targets.rb CHANGED Viewed

@@ -151,6 +151,7 @@ class Webhookdb::API::SyncTargets < Webhookdb::API::V1
                   (stgt = org.all_sync_targets_dataset[Sequel[:sync_targets][:opaque_id] => params[:opaque_id]])
                   merror!(403, "There is no #{fullname} sync target with that id.") if
                     stgt.nil? || !stgt.send(predicate)
+                  set_request_tags(stgt.log_tags)
                   return stgt
                 end
               end
@@ -204,7 +205,6 @@ class Webhookdb::API::SyncTargets < Webhookdb::API::V1
                   )
                 end
-                stgt.logger.warn("destroying_sync_target", sync_target_id: stgt.id, customer_id: current_customer.id)
                 stgt.destroy
                 status 200
                 message = "#{fullname.capitalize} sync target has been removed and will no longer sync."

data/lib/webhookdb/api/system.rb CHANGED Viewed

@@ -37,6 +37,11 @@ class Webhookdb::API::System < Webhookdb::Service
     }
   end
+  post :sink do
+    status 204
+    body ""
+  end
   if ["development", "test"].include?(Webhookdb::RACK_ENV)
     resource :debug do
       resource :echo do

data/lib/webhookdb/api/webhook_subscriptions.rb CHANGED Viewed

@@ -52,6 +52,7 @@ class Webhookdb::API::WebhookSubscriptions < Webhookdb::API::V1
               org = lookup_org!
               whsub = org.all_webhook_subscriptions_dataset[opaque_id: params[:opaque_id]]
               merror!(403, "No webhook subscription with that ID exists in that organization.") if whsub.nil?
+              set_request_tags(webhook_subscription_id: whsub.id)
               return whsub
             end
           end

data/lib/webhookdb/api.rb CHANGED Viewed

@@ -52,11 +52,14 @@ module Webhookdb::API
               memberships = customer.verified_memberships_dataset.where(organization: orgs).limit(2).all
               permission_error!("You don't have permissions with that organization.") if memberships.empty?
               merror!(500, "ambiguous", alert: true) if memberships.size > 1 # TODO: better message, tests
-              return memberships.first.organization
+              org = memberships.first.organization
+              set_request_tags(organization: org.key)
+              return org
             end
             raise "something went wrong" unless allow_connstr_auth
             org = Webhookdb::API::ConnstrAuth.find_authed(orgs, request)
             unauthenticated! if org.nil?
+            set_request_tags(organization: org.key)
             return org
           end