webhookdb 1.3.0 → 1.4.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (63) hide show
  1. checksums.yaml +4 -4
  2. data/admin-dist/assets/{index-6aebf805.js → index-9306dd28.js} +39 -39
  3. data/admin-dist/index.html +1 -1
  4. data/data/messages/templates/errors/generic_backfill.email.liquid +30 -0
  5. data/data/messages/templates/errors/icalendar_fetch.email.liquid +8 -2
  6. data/data/messages/templates/specs/with_fields.email.liquid +6 -0
  7. data/db/migrations/045_system_log.rb +15 -0
  8. data/db/migrations/046_indices.rb +14 -0
  9. data/lib/webhookdb/admin.rb +6 -0
  10. data/lib/webhookdb/admin_api/data_provider.rb +1 -0
  11. data/lib/webhookdb/admin_api/entities.rb +8 -0
  12. data/lib/webhookdb/aggregate_result.rb +1 -1
  13. data/lib/webhookdb/api/helpers.rb +17 -0
  14. data/lib/webhookdb/api/organizations.rb +6 -0
  15. data/lib/webhookdb/api/service_integrations.rb +1 -0
  16. data/lib/webhookdb/connection_cache.rb +29 -3
  17. data/lib/webhookdb/console.rb +1 -1
  18. data/lib/webhookdb/customer/reset_code.rb +1 -1
  19. data/lib/webhookdb/customer.rb +3 -2
  20. data/lib/webhookdb/db_adapter.rb +1 -1
  21. data/lib/webhookdb/dbutil.rb +2 -0
  22. data/lib/webhookdb/errors.rb +34 -0
  23. data/lib/webhookdb/http.rb +1 -1
  24. data/lib/webhookdb/jobs/deprecated_jobs.rb +1 -0
  25. data/lib/webhookdb/jobs/model_event_system_log_tracker.rb +105 -0
  26. data/lib/webhookdb/jobs/monitor_metrics.rb +29 -0
  27. data/lib/webhookdb/jobs/renew_watch_channel.rb +3 -0
  28. data/lib/webhookdb/message/transport.rb +1 -1
  29. data/lib/webhookdb/message.rb +53 -2
  30. data/lib/webhookdb/messages/error_generic_backfill.rb +45 -0
  31. data/lib/webhookdb/messages/error_icalendar_fetch.rb +3 -0
  32. data/lib/webhookdb/messages/specs.rb +16 -0
  33. data/lib/webhookdb/organization/alerting.rb +7 -3
  34. data/lib/webhookdb/organization/database_migration.rb +1 -1
  35. data/lib/webhookdb/organization/db_builder.rb +1 -1
  36. data/lib/webhookdb/organization.rb +14 -1
  37. data/lib/webhookdb/postgres/model.rb +1 -0
  38. data/lib/webhookdb/postgres.rb +2 -1
  39. data/lib/webhookdb/replicator/base.rb +66 -39
  40. data/lib/webhookdb/replicator/column.rb +2 -0
  41. data/lib/webhookdb/replicator/fake.rb +6 -0
  42. data/lib/webhookdb/replicator/front_signalwire_message_channel_app_v1.rb +28 -19
  43. data/lib/webhookdb/replicator/icalendar_calendar_v1.rb +55 -11
  44. data/lib/webhookdb/replicator/intercom_v1_mixin.rb +25 -4
  45. data/lib/webhookdb/replicator/signalwire_message_v1.rb +31 -0
  46. data/lib/webhookdb/replicator/transistor_episode_v1.rb +11 -5
  47. data/lib/webhookdb/replicator/webhook_request.rb +8 -0
  48. data/lib/webhookdb/replicator.rb +2 -2
  49. data/lib/webhookdb/service/view_api.rb +1 -1
  50. data/lib/webhookdb/service.rb +10 -10
  51. data/lib/webhookdb/service_integration.rb +14 -1
  52. data/lib/webhookdb/spec_helpers/shared_examples_for_replicators.rb +153 -64
  53. data/lib/webhookdb/sync_target.rb +7 -5
  54. data/lib/webhookdb/system_log_event.rb +9 -0
  55. data/lib/webhookdb/version.rb +1 -1
  56. data/lib/webhookdb/webhook_subscription.rb +1 -1
  57. data/lib/webhookdb.rb +31 -7
  58. metadata +32 -16
  59. data/lib/webhookdb/jobs/customer_created_notify_internal.rb +0 -22
  60. /data/lib/webhookdb/jobs/{logged_webhook_replay.rb → logged_webhooks_replay.rb} +0 -0
  61. /data/lib/webhookdb/jobs/{logged_webhook_resilient_replay.rb → logged_webhooks_resilient_replay.rb} +0 -0
  62. /data/lib/webhookdb/jobs/{webhook_subscription_delivery_attempt.rb → webhook_subscription_delivery_event.rb} +0 -0
  63. /data/lib/webhookdb/jobs/{webhook_resource_notify_integrations.rb → webhookdb_resource_notify_integrations.rb} +0 -0
@@ -115,7 +115,7 @@
115
115
  </style>
116
116
  <link rel="preconnect" href="https://fonts.gstatic.com" />
117
117
  <link href="https://fonts.googleapis.com/css2?family=Open+Sans:wght@300;500;600;700&display=swap" rel="stylesheet">
118
- <script type="module" crossorigin src="/admin/assets/index-6aebf805.js"></script>
118
+ <script type="module" crossorigin src="/admin/assets/index-9306dd28.js"></script>
119
119
  </head>
120
120
 
121
121
  <body>
@@ -0,0 +1,30 @@
1
+ {% expose subject %}WebhookDB: {{ friendly_name }} Error{% endexpose %}
2
+
3
+ {% partial 'greeting' %}
4
+
5
+ <p>
6
+ WebhookDB encountered an error trying to sync {{ friendly_name }} data using your configured credentials. We have the following information about the failure:
7
+ </p>
8
+ <ul>
9
+ <li>Service Integration Name: {{ service_name }}, ID: <code>{{ opaque_id }}</code></li>
10
+ <li>Request: <code>{{ request_method }} {{ request_url }}</code></li>
11
+ <li>Response Status: <code>{{ response_status }}</code></li>
12
+ <li>Body: <code>{{ response_body }}</code></li>
13
+ </ul>
14
+ <p>
15
+ Usually this indicates an API key has been revoked or some other misconfiguration, rather than being an error in WebhookDB itself.
16
+ </p>
17
+ <p>To fix this integration, head over to <a href="{{ app_url }}">{{ app_url }}</a>, log in, and run:</p>
18
+ <pre>
19
+ webhookdb integrations reset {{ opaque_id }}
20
+ </pre>
21
+ <p>
22
+ If you no longer wish to use this integration, it can be deleted using:
23
+ </p>
24
+ <pre>
25
+ webhookdb integrations delete {{ opaque_id }}
26
+ </pre>
27
+ <p>We'll continue to send daily emails when this happens, so please do fix this up when you get a chance.</p>
28
+ <p>Please file an issue at {{ oss_repo }} or email <a href="mailto:{{ support_email }}">{{ support_email }}</a> if you need any help.</p>
29
+
30
+ {% partial 'signoff' %}
@@ -1,4 +1,4 @@
1
- {% expose subject %}WebhookDB: ICalendar Error{% endexpose %}
1
+ {% expose subject %}WebhookDB: ICalendar Error{% endexpose %}
2
2
 
3
3
  {% partial 'greeting' %}
4
4
 
@@ -6,8 +6,9 @@
6
6
  WebhookDB encountered an error syncing an ICalendar feed. We have the following information about the failure:
7
7
  </p>
8
8
  <ul>
9
+ <li>Organization: {{ org_name }} (key: <code>{{ org_key }}</code>)</li>
9
10
  <li>Service Integration Name: {{ service_name }}, ID: <code>{{ opaque_id }}</code></li>
10
- <li>Calendar ID (you send this to us when adding the calendar): <code>{{ external_calendar_id }}</code></li>
11
+ <li>Calendar ID (you sent this when adding the calendar): <code>{{ external_calendar_id }}</code></li>
11
12
  <li>Request: <code>{{ request_method }} {{ request_url }}</code></li>
12
13
  <li>Response Status: <code>{{ response_status }}</code></li>
13
14
  <li>Body: <code>{{ response_body }}</code></li>
@@ -22,7 +23,12 @@
22
23
  <p>
23
24
  If this calendar is no longer shared, it should be removed.
24
25
  Use a DELETE request, as per <a href="{{ docs_url }}/guides/icalendar/#delete">{{ docs_url }}/guides/icalendar/#delete</a>.
26
+ For example, here is the cURL to run to delete this from the shell:
25
27
  </p>
28
+ <pre>
29
+ $ export WHDB_WEBHOOK_SECRET=`webhookdb integration info --org={{ org_key }} --field=webhook_secret {{ opaque_id }}`
30
+ $ curl -X POST -d '{"type":"DELETE","external_id":"{{ external_calendar_id }}"}' -H "Content-Type: application/json" -H "Whdb-Webhook-Secret: ${WHDB_WEBHOOK_SECRET}" "{{ webhook_endpoint }}"
31
+ </pre>
26
32
  <p>We'll continue to send daily emails when this happens, so please do fix this up when you get a chance.</p>
27
33
  <p>Please file an issue at {{ oss_repo }} if you need any help.</p>
28
34
 
@@ -0,0 +1,6 @@
1
+ {% expose subject %}subject with multiple fields{% endexpose %}
2
+ a: {{ a }}
3
+ b: {{ b }}
4
+ c: {{ c }}
5
+ d: {{ d }}
6
+ e: {{ e }}
@@ -0,0 +1,15 @@
1
+ # frozen_string_literal: true
2
+
3
+ Sequel.migration do
4
+ change do
5
+ create_table(:system_log_events) do
6
+ primary_key :id, type: :bigserial
7
+ timestamptz :at, null: false, default: Sequel.function(:now), index: true
8
+ text :title, null: false, default: ""
9
+ text :body, null: false, default: ""
10
+ text :link, null: false, default: ""
11
+ foreign_key :actor_id, :customers
12
+ column :text_search, :tsvector
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ Sequel.migration do
4
+ change do
5
+ alter_table(:service_integrations) do
6
+ add_index :depends_on_id
7
+ add_index :service_name
8
+ end
9
+
10
+ alter_table(:message_deliveries) do
11
+ add_index :soft_deleted_at
12
+ end
13
+ end
14
+ end
@@ -1,4 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Webhookdb::Admin
4
+ module Linked
5
+ protected def _admin_datatype = self.class.dataset.first_source_table
6
+ protected def _admin_id = self.pk
7
+ protected def _admin_display = "show"
8
+ def admin_link = "#{Webhookdb.admin_url}/admin#/#{_admin_datatype}/#{_admin_id}/#{_admin_display}"
9
+ end
4
10
  end
@@ -29,6 +29,7 @@ class Webhookdb::AdminAPI::DataProvider < Webhookdb::AdminAPI::V1
29
29
  service_integrations: [Webhookdb::ServiceIntegration, ServiceIntegration],
30
30
  subscriptions: [Webhookdb::Subscription, Subscription],
31
31
  sync_targets: [Webhookdb::SyncTarget, SyncTarget],
32
+ system_log_events: [Webhookdb::SystemLogEvent, SystemLogEvent],
32
33
  webhook_subscriptions: [Webhookdb::WebhookSubscription, WebhookSubscription],
33
34
  webhook_subscription_deliveries: [Webhookdb::WebhookSubscription::Delivery, WebhookSubscriptionDelivery],
34
35
  }.freeze
@@ -163,6 +163,14 @@ module Webhookdb::AdminAPI::Entities
163
163
  expose :page_size
164
164
  end
165
165
 
166
+ class SystemLogEvent < Base
167
+ expose :at
168
+ expose :title
169
+ expose :body
170
+ expose :link
171
+ expose :actor_id
172
+ end
173
+
166
174
  class WebhookSubscription < Base
167
175
  expose :organization, with: Related
168
176
  expose :service_integration, with: Related
@@ -18,7 +18,7 @@ require "webhookdb" unless defined?(Webhookdb)
18
18
  # end
19
19
  # return ag.finish
20
20
  #
21
- class Webhookdb::AggregateResult < StandardError
21
+ class Webhookdb::AggregateResult < Webhookdb::WebhookdbError
22
22
  attr_reader :successes, :failures, :errors
23
23
 
24
24
  def initialize(existing=nil)
@@ -171,6 +171,23 @@ module Webhookdb::API::Helpers
171
171
  sint = yield
172
172
  return if sint == :pass
173
173
  raise "error instead of return nil if there is no service integration" if sint.nil?
174
+ # If this is a valid GET request (ie, the replicator doesn't do useful auth),
175
+ # and it's from a bot, we want to block it, otherwise we'd process invalid webhooks.
176
+ #
177
+ # This is almost never a problem, because almost all services perform some validation,
178
+ # and the auth info isn't in a GET URL; but in some cases, that won't be the case,
179
+ # and we protect against it here.
180
+ #
181
+ # An example would be a bot getting a link preview.
182
+ if request.get? && !request.GET["skipbotcheck"] && Browser.new(request.user_agent).bot?
183
+ # IMPORTANT: Run the Browser code last since it has to run a bunch of regexes to detect a bot.
184
+ env["api.format"] = :binary
185
+ header "Content-Type", "text/plain"
186
+ body("This route is for receiving webhooks and HTTP calls, not for bots. " \
187
+ "Call this endpoint with the query param 'skipbotcheck=true' to bypass this check.")
188
+ status 403
189
+ return
190
+ end
174
191
  opaque_id = sint.opaque_id
175
192
  organization_id = sint.organization_id
176
193
  svc = Webhookdb::Replicator.create(sint).dispatch_request_to(request)
@@ -206,6 +206,12 @@ class Webhookdb::API::Organizations < Webhookdb::API::V1
206
206
  {title: "Customer", value: "(#{c.id}) #{c.email}", short: false},
207
207
  ],
208
208
  ).emit
209
+ Webhookdb::SystemLogEvent.create(
210
+ title: "Organization Closure Requested",
211
+ body: "#{org.name} (#{org.key})",
212
+ link: org.admin_link,
213
+ actor: c,
214
+ )
209
215
  step = Webhookdb::Replicator::StateMachineStep.new.completed
210
216
  step.output = "Thanks! We've received the request to close your #{org.name} organization. " \
211
217
  "We'll be in touch within 2 business days confirming removal."
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "browser"
3
4
  require "grape"
4
5
  require "oj"
5
6
 
@@ -45,13 +45,16 @@ class Webhookdb::ConnectionCache
45
45
  extend Webhookdb::MethodUtilities
46
46
  include Webhookdb::Dbutil
47
47
 
48
- class ReentranceError < StandardError; end
48
+ class ReentranceError < Webhookdb::ProgrammingError; end
49
49
 
50
50
  configurable(:connection_cache) do
51
51
  # If this many seconds has elapsed since the last connecton was borrowed,
52
52
  # prune connections with no pending borrows.
53
53
  setting :prune_interval, 120
54
54
 
55
+ # If a connection hasn't been used in this long, validate it before reusing it.
56
+ setting :idle_timeout, 20.minutes
57
+
55
58
  # Seconds for the :fast timeout option.
56
59
  setting :timeout_fast, 30
57
60
  # Seconds for the :slow timeout option.
@@ -83,6 +86,25 @@ class Webhookdb::ConnectionCache
83
86
  @last_pruned_at = Time.now
84
87
  end
85
88
 
89
+ Available = Struct.new(:connection, :at) do
90
+ delegate :disconnect, to: :connection
91
+
92
+ # Return +connection+ if it has not been idle long enough,
93
+ # or if it has been idle, then validate it (SELECT 1), and return +connection+
94
+ # if it's valid, or +nil+ if the database disconnected it.
95
+ def validated_connection
96
+ needs_validation_at = self.at + Webhookdb::ConnectionCache.idle_timeout
97
+ return self.connection if needs_validation_at > Time.now
98
+ begin
99
+ self.connection << "SELECT 1"
100
+ return self.connection
101
+ rescue Sequel::DatabaseDisconnectError
102
+ self.connection.disconnect
103
+ return nil
104
+ end
105
+ end
106
+ end
107
+
86
108
  # Connect to the database at the given URL.
87
109
  # borrow is not re-entrant, so if the current thread already owns a connection
88
110
  # to the given url, raise a ReentrantError.
@@ -111,7 +133,11 @@ class Webhookdb::ConnectionCache
111
133
  raise ReentranceError,
112
134
  "ConnectionCache#borrow is not re-entrant for the same database since the connection has stateful config"
113
135
  end
114
- conn = db_loans[:available].pop || take_conn(url, single_threaded: true, extensions: [:pg_json, :pg_streaming])
136
+ if (available = db_loans[:available].pop)
137
+ # If the connection doesn't validate, it won't be in :available at this point, so don't worry about it.
138
+ conn = available.validated_connection
139
+ end
140
+ conn ||= take_conn(url, single_threaded: true, extensions: [:pg_json, :pg_streaming])
115
141
  db_loans[:loaned][t] = conn
116
142
  end
117
143
  conn << "SET statement_timeout TO #{timeout * 1000}" if timeout.present?
@@ -126,7 +152,7 @@ class Webhookdb::ConnectionCache
126
152
  conn << "SET statement_timeout TO 0" if timeout.present?
127
153
  @mutex.synchronize do
128
154
  @dbs_for_urls[url][:loaned].delete(t)
129
- @dbs_for_urls[url][:available] << conn
155
+ @dbs_for_urls[url][:available] << Available.new(conn, Time.now)
130
156
  end
131
157
  end
132
158
  self.prune(url) if now > self.next_prune_at
@@ -5,7 +5,7 @@ require "webhookdb"
5
5
  module Webhookdb::Console
6
6
  extend Webhookdb::MethodUtilities
7
7
 
8
- class Error < StandardError; end
8
+ class Error < Webhookdb::WebhookdbError; end
9
9
 
10
10
  class UnsafeOperation < Error; end
11
11
 
@@ -6,7 +6,7 @@ require "webhookdb/postgres"
6
6
  require "webhookdb/customer"
7
7
 
8
8
  class Webhookdb::Customer::ResetCode < Webhookdb::Postgres::Model(:customer_reset_codes)
9
- class Unusable < StandardError; end
9
+ class Unusable < Webhookdb::WebhookdbError; end
10
10
 
11
11
  TOKEN_LENGTH = 6
12
12
 
@@ -10,9 +10,10 @@ require "webhookdb/demo_mode"
10
10
  class Webhookdb::Customer < Webhookdb::Postgres::Model(:customers)
11
11
  extend Webhookdb::MethodUtilities
12
12
  include Appydays::Configurable
13
+ include Webhookdb::Admin::Linked
13
14
 
14
- class InvalidPassword < StandardError; end
15
- class SignupDisabled < StandardError; end
15
+ class InvalidPassword < Webhookdb::WebhookdbError; end
16
+ class SignupDisabled < Webhookdb::WebhookdbError; end
16
17
 
17
18
  configurable(:customer) do
18
19
  setting :signup_email_allowlist, ["*"], convert: ->(s) { s.split }
@@ -3,7 +3,7 @@
3
3
  class Webhookdb::DBAdapter
4
4
  require "webhookdb/db_adapter/column_types"
5
5
 
6
- class UnsupportedAdapter < StandardError; end
6
+ class UnsupportedAdapter < Webhookdb::ProgrammingError; end
7
7
 
8
8
  VALID_IDENTIFIER = /^[a-zA-Z][a-zA-Z\d_ ]*$/
9
9
  INVALID_IDENTIFIER_PROMPT =
@@ -35,6 +35,7 @@ module Webhookdb::Dbutil
35
35
  4
36
36
  end)
37
37
  setting :pool_timeout, 10
38
+ setting :pool_class, :timed_queue
38
39
  # Set to 'disable' to work around segfault.
39
40
  # See https://github.com/ged/ruby-pg/issues/538
40
41
  setting :gssencmode, ""
@@ -70,6 +71,7 @@ module Webhookdb::Dbutil
70
71
  res[:log_warn_duration] ||= Webhookdb::Dbutil.slow_query_seconds
71
72
  res[:max_connections] ||= Webhookdb::Dbutil.max_connections
72
73
  res[:pool_timeout] ||= Webhookdb::Dbutil.pool_timeout
74
+ res[:pool_class] ||= Webhookdb::Dbutil.pool_class
73
75
  res[:driver_options] = {}
74
76
  (res[:driver_options][:gssencmode] = Webhookdb::Dbutil.gssencmode) if Webhookdb::Dbutil.gssencmode.present?
75
77
  return res
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Webhookdb::Errors
4
+ class << self
5
+ # Call the given block for the given exception, each cause (see +Exception#cause+),
6
+ # and each wrapped errors (see +Amigo::Retry::Error#wrapped+).
7
+ # If the block returns +true+ for any exception, stop walking.
8
+ def each_cause(ex, &)
9
+ raise LocalJumpError unless block_given?
10
+ return true if yield(ex) == true
11
+ caused_got = ex.cause && each_cause(ex.cause, &)
12
+ return true if caused_got == true
13
+ wrapped_got = ex.respond_to?(:wrapped) && ex.wrapped && each_cause(ex.wrapped, &)
14
+ return true if wrapped_got == true
15
+ return nil
16
+ end
17
+
18
+ # Run the given block for each cause (see +each_cause),
19
+ # returning the first exception the block returns +true+ for.
20
+ def find_cause(ex, &)
21
+ raise LocalJumpError unless block_given?
22
+ got = nil
23
+ each_cause(ex) do |cause|
24
+ if yield(cause) == true
25
+ got = cause
26
+ true
27
+ else
28
+ false
29
+ end
30
+ end
31
+ return got
32
+ end
33
+ end
34
+ end
@@ -11,7 +11,7 @@ module Webhookdb::Http
11
11
  end
12
12
 
13
13
  # Error raised when some API has rate limited us.
14
- class BaseError < StandardError; end
14
+ class BaseError < Webhookdb::WebhookdbError; end
15
15
 
16
16
  class Error < BaseError
17
17
  attr_reader :response, :body, :uri, :status, :http_method
@@ -14,6 +14,7 @@ Amigo::DeprecatedJobs.install(
14
14
  "Jobs::ConvertKitBroadcastBackfill",
15
15
  "Jobs::ConvertKitSubscriberBackfill",
16
16
  "Jobs::ConvertKitTagBackfill",
17
+ "Jobs::CustomerCreatedNotifyInternal",
17
18
  "Jobs::RssBackfillPoller",
18
19
  "Jobs::TwilioScheduledBackfill",
19
20
  )
@@ -0,0 +1,105 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "webhookdb/async/job"
4
+ require "webhookdb/messages/invite"
5
+
6
+ class Webhookdb::Jobs::ModelEventSystemLogTracker
7
+ extend Webhookdb::Async::Job
8
+
9
+ on "webhookdb.*"
10
+
11
+ def _perform(event)
12
+ case event.name
13
+ when "webhookdb.customer.created"
14
+ self.alert_customer_created(event)
15
+ when "webhookdb.organization.created"
16
+ self.alert_org_created(event)
17
+ when "webhookdb.serviceintegration.created"
18
+ self.alert_sint_created(event)
19
+ when "webhookdb.serviceintegration.destroyed"
20
+ self.alert_sint_destroyed(event)
21
+ end
22
+ end
23
+
24
+ def create_event(title, body, link)
25
+ Webhookdb::SystemLogEvent.create(
26
+ at: Time.now,
27
+ title:,
28
+ body:,
29
+ link:,
30
+ )
31
+ end
32
+
33
+ def alert_customer_created(event)
34
+ customer = self.lookup_model(Webhookdb::Customer, event)
35
+ Webhookdb::DeveloperAlert.new(
36
+ subsystem: "Customer Created",
37
+ emoji: ":hook:",
38
+ fallback: "New customer created: #{customer.inspect}",
39
+ fields: [
40
+ {title: "Id", value: customer.id, short: true},
41
+ {title: "Email", value: customer.email, short: true},
42
+ {title: "Link", value: customer.admin_link},
43
+ ],
44
+ ).emit
45
+ create_event("Customer Created", customer.email, customer.admin_link)
46
+ end
47
+
48
+ def alert_org_created(event)
49
+ org = self.lookup_model(Webhookdb::Organization, event)
50
+ Webhookdb::DeveloperAlert.new(
51
+ subsystem: "Organization Created",
52
+ emoji: ":office:",
53
+ fallback: "Organization created: #{org.inspect}",
54
+ fields: [
55
+ {title: "Id", value: org.id, short: true},
56
+ {title: "Email", value: org.name, short: true},
57
+ {title: "Link", value: org.admin_link},
58
+ ],
59
+ ).emit
60
+ create_event("Organization Created", "#{org.name} (#{org.key})", org.admin_link)
61
+ end
62
+
63
+ def alert_sint_created(event)
64
+ sint = self.lookup_model(Webhookdb::ServiceIntegration, event)
65
+ Webhookdb::DeveloperAlert.new(
66
+ subsystem: "Integration Created",
67
+ emoji: ":fax:",
68
+ fallback: "Service Integration #{sint.service_name} (#{sint.opaque_id}) created",
69
+ fields: [
70
+ {title: "Id", value: sint.opaque_id, short: true},
71
+ {title: "Service", value: sint.service_name, short: true},
72
+ {title: "Table", value: sint.table_name, short: true},
73
+ {title: "Org Name", value: sint.organization.name, short: true},
74
+ {title: "Link", value: sint.admin_link},
75
+ ],
76
+ ).emit
77
+ create_event(
78
+ "Integration Created",
79
+ "#{sint.service_name} (#{sint.opaque_id}) created in #{sint.organization.name}",
80
+ sint.admin_link,
81
+ )
82
+ end
83
+
84
+ def alert_sint_destroyed(event)
85
+ pl = event.payload[1].symbolize_keys
86
+ org = Webhookdb::Organization[pl[:organization_id]]
87
+ Webhookdb::DeveloperAlert.new(
88
+ subsystem: "Integration Deleted",
89
+ emoji: ":funeral_urn:",
90
+ fallback: "Service Integration #{pl[:service_name]} (#{pl[:opaque_id]}) deleted",
91
+ fields: [
92
+ {title: "Id", value: pl[:opaque_id], short: true},
93
+ {title: "Service", value: pl[:service_name], short: true},
94
+ {title: "Table", value: pl[:table_name], short: true},
95
+ {title: "Org Name", value: org.name, short: true},
96
+ {title: "Link", value: org.admin_link},
97
+ ],
98
+ ).emit
99
+ create_event(
100
+ "Integration Deleted",
101
+ "#{pl[:service_name]} (#{pl[:opaque_id]}) deleted from #{org.name}",
102
+ org.admin_link,
103
+ )
104
+ end
105
+ end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "webhookdb/async/scheduled_job"
4
+ require "webhookdb/jobs"
5
+
6
+ # Log out some metrics every minute.
7
+ class Webhookdb::Jobs::MonitorMetrics
8
+ extend Webhookdb::Async::ScheduledJob
9
+
10
+ cron "* * * * *" # Every 1 minute
11
+ splay 0
12
+
13
+ def _perform
14
+ opts = {}
15
+ max_size = 0
16
+ max_latency = 0
17
+ Sidekiq::Queue.all.each do |q|
18
+ size = q.size
19
+ latency = q.latency
20
+ max_size = [max_size, size].max
21
+ max_latency = [max_latency, latency].max
22
+ opts["#{q.name}_size"] = size
23
+ opts["#{q.name}_latency"] = latency
24
+ end
25
+ opts[:max_size] = max_size
26
+ opts[:max_latency] = max_latency
27
+ self.logger.info("metrics_monitor_queue", **opts)
28
+ end
29
+ end
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "amigo/queue_backoff_job"
3
4
  require "webhookdb/async/job"
4
5
  require "webhookdb/jobs"
5
6
 
@@ -9,8 +10,10 @@ require "webhookdb/jobs"
9
10
  # Calls #renew_watch_channel(row_pk:, expiring_before:).
10
11
  class Webhookdb::Jobs::RenewWatchChannel
11
12
  extend Webhookdb::Async::Job
13
+ include Amigo::QueueBackoffJob
12
14
 
13
15
  on "webhookdb.serviceintegration.renewwatchchannel"
16
+ sidekiq_options queue: "netout"
14
17
 
15
18
  def _perform(event)
16
19
  sint = self.lookup_model(Webhookdb::ServiceIntegration, event)
@@ -5,7 +5,7 @@ require "webhookdb/message"
5
5
  class Webhookdb::Message::Transport
6
6
  extend Webhookdb::MethodUtilities
7
7
 
8
- class Error < StandardError; end
8
+ class Error < Webhookdb::WebhookdbError; end
9
9
  class UndeliverableRecipient < Error; end
10
10
 
11
11
  singleton_attr_reader :transports
@@ -73,6 +73,7 @@ module Webhookdb::Message
73
73
  "environment" => Webhookdb::Message::EnvironmentDrop.new,
74
74
  "app_url" => Webhookdb.app_url,
75
75
  )
76
+ drops = self.unify_drops_encoding(drops)
76
77
 
77
78
  content_tmpl = Liquid::Template.parse(template_file.read)
78
79
  # The 'expose' drop smashes data into the register.
@@ -101,13 +102,63 @@ module Webhookdb::Message
101
102
  return Rendering.new(content, lctx.registers)
102
103
  end
103
104
 
105
+ # Handle encoding in liquid drop string values that would likely crash message rendering.
106
+ #
107
+ # If there is a mixed character encoding of string values in a liquid drop,
108
+ # such as when handling user-supplied values, force all strings into UTF-8.
109
+ #
110
+ # This is needed because the way Ruby does encoding coercion when parsing input
111
+ # which does not declare an encoding, such as a file or especially an HTTP response.
112
+ # Ruby will:
113
+ # - Use ASCII if the values fit into 7 bits
114
+ # - Use ASCII-8BIT if the values fit into 8 bits (128 to 255)
115
+ # - Otherwise, use UTF-8.
116
+ #
117
+ # The actual rules are more complex, but this is common enough.
118
+ #
119
+ # While ASCII encoding can be used as UTF-8, ASCII-8BIT cannot.
120
+ # So adding `(ascii-8bit string) + (utf-8 string)` will error with an
121
+ # `Encoding::CompatibilityError`.
122
+ #
123
+ # Instead, if we see a series of liquid drop string values
124
+ # with different encodings, force them all to be UTF-8.
125
+ # This can result in some unexpected behavior,
126
+ # but it should be fine, since you'd only see it with unexpected input
127
+ # (all valid inputs should be UTF-8).
128
+ #
129
+ # @param [Hash] drops
130
+ # @return [Hash]
131
+ def self.unify_drops_encoding(drops)
132
+ return drops if drops.empty?
133
+ seen_enc = nil
134
+ force_enc = false
135
+ drops.each_value do |v|
136
+ next unless v.respond_to?(:encoding)
137
+ seen_enc ||= v.encoding
138
+ next if seen_enc == v.encoding
139
+ force_enc = true
140
+ break
141
+ end
142
+ return drops unless force_enc
143
+ utf8 = Encoding.find("UTF-8")
144
+ result = drops.each_with_object({}) do |(k, v), memo|
145
+ if v.respond_to?(:encoding) && v.encoding != utf8
146
+ v2 = v.encode(utf8, invalid: :replace, undef: :replace, replace: "?")
147
+ memo[k] = v2
148
+ else
149
+ memo[k] = v
150
+ end
151
+ end
152
+ return result
153
+ end
154
+
104
155
  def self.send_unsent
105
156
  Webhookdb::Message::Delivery.unsent.each(&:send!)
106
157
  end
107
158
 
108
- class InvalidTransportError < StandardError; end
159
+ class InvalidTransportError < Webhookdb::ProgrammingError; end
109
160
 
110
- class MissingTemplateError < StandardError; end
161
+ class MissingTemplateError < Webhookdb::ProgrammingError; end
111
162
 
112
163
  # Presents a homogeneous interface for a given 'to' value (email vs. customer, for example).
113
164
  # .to will always be a plain object, and .customer will be a +Webhookdb::Customer+ if present.