webhookdb 1.3.1 → 1.4.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (63) hide show
  1. checksums.yaml +4 -4
  2. data/admin-dist/assets/{index-6aebf805.js → index-9306dd28.js} +39 -39
  3. data/admin-dist/index.html +1 -1
  4. data/data/messages/templates/errors/generic_backfill.email.liquid +30 -0
  5. data/data/messages/templates/errors/icalendar_fetch.email.liquid +8 -2
  6. data/data/messages/templates/specs/with_fields.email.liquid +6 -0
  7. data/db/migrations/045_system_log.rb +15 -0
  8. data/db/migrations/046_indices.rb +14 -0
  9. data/lib/webhookdb/admin.rb +6 -0
  10. data/lib/webhookdb/admin_api/data_provider.rb +1 -0
  11. data/lib/webhookdb/admin_api/entities.rb +8 -0
  12. data/lib/webhookdb/aggregate_result.rb +1 -1
  13. data/lib/webhookdb/api/helpers.rb +17 -0
  14. data/lib/webhookdb/api/organizations.rb +6 -0
  15. data/lib/webhookdb/api/service_integrations.rb +1 -0
  16. data/lib/webhookdb/connection_cache.rb +29 -3
  17. data/lib/webhookdb/console.rb +1 -1
  18. data/lib/webhookdb/customer/reset_code.rb +1 -1
  19. data/lib/webhookdb/customer.rb +3 -2
  20. data/lib/webhookdb/db_adapter.rb +1 -1
  21. data/lib/webhookdb/dbutil.rb +2 -0
  22. data/lib/webhookdb/errors.rb +34 -0
  23. data/lib/webhookdb/http.rb +1 -1
  24. data/lib/webhookdb/jobs/deprecated_jobs.rb +1 -0
  25. data/lib/webhookdb/jobs/model_event_system_log_tracker.rb +105 -0
  26. data/lib/webhookdb/jobs/monitor_metrics.rb +29 -0
  27. data/lib/webhookdb/jobs/renew_watch_channel.rb +3 -0
  28. data/lib/webhookdb/message/transport.rb +1 -1
  29. data/lib/webhookdb/message.rb +53 -2
  30. data/lib/webhookdb/messages/error_generic_backfill.rb +45 -0
  31. data/lib/webhookdb/messages/error_icalendar_fetch.rb +3 -0
  32. data/lib/webhookdb/messages/specs.rb +16 -0
  33. data/lib/webhookdb/organization/alerting.rb +7 -3
  34. data/lib/webhookdb/organization/database_migration.rb +1 -1
  35. data/lib/webhookdb/organization/db_builder.rb +1 -1
  36. data/lib/webhookdb/organization.rb +14 -1
  37. data/lib/webhookdb/postgres/model.rb +1 -0
  38. data/lib/webhookdb/postgres.rb +2 -1
  39. data/lib/webhookdb/replicator/base.rb +66 -39
  40. data/lib/webhookdb/replicator/column.rb +2 -0
  41. data/lib/webhookdb/replicator/fake.rb +6 -0
  42. data/lib/webhookdb/replicator/front_signalwire_message_channel_app_v1.rb +28 -19
  43. data/lib/webhookdb/replicator/icalendar_calendar_v1.rb +55 -11
  44. data/lib/webhookdb/replicator/intercom_v1_mixin.rb +25 -4
  45. data/lib/webhookdb/replicator/signalwire_message_v1.rb +31 -0
  46. data/lib/webhookdb/replicator/transistor_episode_v1.rb +11 -5
  47. data/lib/webhookdb/replicator/webhook_request.rb +8 -0
  48. data/lib/webhookdb/replicator.rb +2 -2
  49. data/lib/webhookdb/service/view_api.rb +1 -1
  50. data/lib/webhookdb/service.rb +10 -10
  51. data/lib/webhookdb/service_integration.rb +14 -1
  52. data/lib/webhookdb/spec_helpers/shared_examples_for_replicators.rb +153 -64
  53. data/lib/webhookdb/sync_target.rb +7 -5
  54. data/lib/webhookdb/system_log_event.rb +9 -0
  55. data/lib/webhookdb/version.rb +1 -1
  56. data/lib/webhookdb/webhook_subscription.rb +1 -1
  57. data/lib/webhookdb.rb +31 -7
  58. metadata +32 -16
  59. data/lib/webhookdb/jobs/customer_created_notify_internal.rb +0 -22
  60. /data/lib/webhookdb/jobs/{logged_webhook_replay.rb → logged_webhooks_replay.rb} +0 -0
  61. /data/lib/webhookdb/jobs/{logged_webhook_resilient_replay.rb → logged_webhooks_resilient_replay.rb} +0 -0
  62. /data/lib/webhookdb/jobs/{webhook_subscription_delivery_attempt.rb → webhook_subscription_delivery_event.rb} +0 -0
  63. /data/lib/webhookdb/jobs/{webhook_resource_notify_integrations.rb → webhookdb_resource_notify_integrations.rb} +0 -0
@@ -141,11 +141,17 @@ class Webhookdb::Replicator::TransistorEpisodeV1 < Webhookdb::Replicator::Base
141
141
  transcript_url = resource.fetch("attributes").fetch("transcript_url", nil)
142
142
  return nil if transcript_url.blank?
143
143
  (transcript_url += ".txt") unless transcript_url.end_with?(".txt")
144
- resp = Webhookdb::Http.get(
145
- transcript_url,
146
- logger: self.logger,
147
- timeout: Webhookdb::Transistor.http_timeout,
148
- )
144
+ begin
145
+ resp = Webhookdb::Http.get(
146
+ transcript_url,
147
+ logger: self.logger,
148
+ timeout: Webhookdb::Transistor.http_timeout,
149
+ )
150
+ rescue Webhookdb::Http::Error => e
151
+ # Not sure why this happens, but nothing we can do if it does.
152
+ return nil if e.status == 404
153
+ raise e
154
+ end
149
155
  transcript_text = resp.body
150
156
  return {transcript_text:}
151
157
  end
@@ -6,4 +6,12 @@ class Webhookdb::Replicator::WebhookRequest < Webhookdb::TypedStruct
6
6
  # When a webhook is processed synchronously, this will be set to the Rack::Request.
7
7
  # Normal (async) webhook processing does not have this available.
8
8
  attr_accessor :rack_request
9
+
10
+ JSON_KEYS = ["body", "headers", "path", "method"].freeze
11
+ def as_json
12
+ return JSON_KEYS.each_with_object({}) do |k, h|
13
+ v = self.send(k)
14
+ h[k] = v.as_json unless v.nil?
15
+ end
16
+ end
9
17
  end
@@ -15,11 +15,11 @@ class Webhookdb::Replicator
15
15
  PLUGIN_DIR = Pathname(__FILE__).dirname + PLUGIN_DIRNAME
16
16
 
17
17
  # Raised when there is no service registered for a name.
18
- class Invalid < StandardError; end
18
+ class Invalid < Webhookdb::WebhookdbError; end
19
19
 
20
20
  # Raised when credentials to interact with a service are not set up.
21
21
  # Usually this is due to a missing dependency.
22
- class CredentialsMissing < StandardError; end
22
+ class CredentialsMissing < Webhookdb::WebhookdbError; end
23
23
 
24
24
  # Statically describe a replicator.
25
25
  class Descriptor < Webhookdb::TypedStruct
@@ -3,7 +3,7 @@
3
3
  # Mixin for Grape API endpoints that use HTML rendering.
4
4
  # This isn't tested well enough.
5
5
  module Webhookdb::Service::ViewApi
6
- class FormError < StandardError
6
+ class FormError < Webhookdb::WebhookdbError
7
7
  attr_reader :status
8
8
 
9
9
  def initialize(msg, status=400)
@@ -163,6 +163,10 @@ class Webhookdb::Service < Grape::API
163
163
  error!(e.message, 405)
164
164
  end
165
165
 
166
+ rescue_from Webhookdb::ExceptionCarrier do |e|
167
+ merror!(e.status, e.message, code: e.code, more: e.more, headers: e.headers)
168
+ end
169
+
166
170
  rescue_from Webhookdb::LockFailed do |_e|
167
171
  merror!(
168
172
  409,
@@ -191,17 +195,7 @@ class Webhookdb::Service < Grape::API
191
195
  status = e.respond_to?(:status) ? e.status : 500
192
196
  error_id = SecureRandom.uuid
193
197
  error_signature = Digest::MD5.hexdigest("#{e.class}: #{e.message}")
194
-
195
- Webhookdb::Service.logger.error "[%s] Uncaught %p in service: %s" %
196
- [error_id, e.class, e.message]
197
- Webhookdb::Service.logger.debug { e.backtrace.join("\n") }
198
- if ENV["PRINT_API_ERROR"]
199
- puts e
200
- puts e.backtrace
201
- end
202
-
203
198
  more = {error_id:, error_signature:}
204
-
205
199
  if Webhookdb::Service.devmode
206
200
  msg = e.message
207
201
  more[:backtrace] = e.backtrace.join("\n")
@@ -210,6 +204,12 @@ class Webhookdb::Service < Grape::API
210
204
  msg = "An internal error occurred of type #{error_signature}. Error ID: #{error_id}"
211
205
  end
212
206
  Webhookdb::Service.logger.error("api_exception", {error_id:, error_signature:}, e)
207
+ Webhookdb::Service.logger.debug { e.backtrace.join("\n") }
208
+ if ENV["PRINT_API_ERROR"]
209
+ puts e
210
+ puts e.backtrace
211
+ end
212
+
213
213
  merror!(status, msg, code: "api_error", more:)
214
214
  end
215
215
 
@@ -6,6 +6,8 @@ require "webhookdb/postgres/model"
6
6
  require "sequel/plugins/soft_deletes"
7
7
 
8
8
  class Webhookdb::ServiceIntegration < Webhookdb::Postgres::Model(:service_integrations)
9
+ include Webhookdb::Admin::Linked
10
+
9
11
  class TableRenameError < Webhookdb::InvalidInput; end
10
12
 
11
13
  # We limit the information that a user can access through the CLI to these fields.
@@ -70,7 +72,7 @@ class Webhookdb::ServiceIntegration < Webhookdb::Postgres::Model(:service_integr
70
72
  end)
71
73
 
72
74
  many_to_one :depends_on, class: self
73
- one_to_many :dependents, key: :depends_on_id, class: self
75
+ one_to_many :dependents, key: :depends_on_id, class: self, order: :id
74
76
  one_to_many :sync_targets, class: "Webhookdb::SyncTarget"
75
77
 
76
78
  # @return [Webhookdb::ServiceIntegration]
@@ -139,10 +141,21 @@ class Webhookdb::ServiceIntegration < Webhookdb::Postgres::Model(:service_integr
139
141
  select { |si| si.service_name == dep_descr.name }
140
142
  end
141
143
 
144
+ # Return all dependents (integrations that depend on this one), breadth-first
145
+ # (that is, all children before grandchildren).
146
+ # @return [Array<Webhookdb::ServiceIntegration>]
142
147
  def recursive_dependents
143
148
  return self.dependents + self.dependents.flat_map(&:recursive_dependents)
144
149
  end
145
150
 
151
+ # Return all service integrations this one depends on,
152
+ # in closest-ancestor order (that is, parent before grandparent).
153
+ # @return [Array<Webhookdb::ServiceIntegration>]
154
+ def recursive_dependencies
155
+ return [] if self.depends_on.nil?
156
+ return [self.depends_on].concat(self.depends_on.recursive_dependencies)
157
+ end
158
+
146
159
  def destroy_self_and_all_dependents
147
160
  self.dependents.each(&:destroy_self_and_all_dependents)
148
161
 
@@ -4,12 +4,18 @@ require "webhookdb/spec_helpers/whdb"
4
4
 
5
5
  # The basics: these shared examples are among the most commonly used.
6
6
 
7
- RSpec.shared_examples "a replicator" do |name|
8
- let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name: name) }
7
+ # @param supports_rowupsert: If true, this replicator will emit the rowupsert event when rows change.
8
+ # Nearly all replicators support this, but in some cases, replicators may not want to,
9
+ # especially when they otherwise do not adhere to normal replicator design.
10
+ # Usually this is only the case in enterprise intergrations.
11
+ #
12
+ # @param supports_row_diff: If true, test that the rowupsert event is not emitted when the row has not changed.
13
+ RSpec.shared_examples "a replicator" do |supports_rowupsert: true, supports_row_diff: true|
14
+ let(:service_name) { described_class.descriptor.name }
15
+ let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name:) }
9
16
  let(:svc) { Webhookdb::Replicator.create(sint) }
10
17
  let(:body) { raise NotImplementedError }
11
18
  let(:expected_data) { body }
12
- let(:supports_row_diff) { true }
13
19
  let(:expected_row) { nil }
14
20
  Webhookdb::SpecHelpers::Whdb.setup_upsert_webhook_example(self)
15
21
 
@@ -70,48 +76,50 @@ RSpec.shared_examples "a replicator" do |name|
70
76
  end
71
77
  end
72
78
 
73
- it "emits the rowupsert event if the row has changed", :async, :do_not_defer_events, sidekiq: :fake do
74
- Webhookdb::Fixtures.webhook_subscription(service_integration: sint).create
75
- svc.create_table
76
- upsert_webhook(svc, body:)
77
- expect(Sidekiq).to have_queue.consisting_of(
78
- job_hash(
79
- Webhookdb::Jobs::SendWebhook,
80
- args: contain_exactly(
81
- hash_including(
82
- "id",
83
- "name" => "webhookdb.serviceintegration.rowupsert",
84
- "payload" => [
85
- sint.id,
86
- hash_including("external_id", "external_id_column", "row" => hash_including("data")),
87
- ],
79
+ if supports_rowupsert
80
+ it "emits the rowupsert event if the row has changed", :async, :do_not_defer_events, sidekiq: :fake do
81
+ Webhookdb::Fixtures.webhook_subscription(service_integration: sint).create
82
+ svc.create_table
83
+ upsert_webhook(svc, body:)
84
+ expect(Sidekiq).to have_queue.consisting_of(
85
+ job_hash(
86
+ Webhookdb::Jobs::SendWebhook,
87
+ args: contain_exactly(
88
+ hash_including(
89
+ "id",
90
+ "name" => "webhookdb.serviceintegration.rowupsert",
91
+ "payload" => [
92
+ sint.id,
93
+ hash_including("external_id", "external_id_column", "row" => hash_including("data")),
94
+ ],
95
+ ),
88
96
  ),
89
97
  ),
90
- ),
91
- )
92
- end
98
+ )
99
+ end
93
100
 
94
- it "does not emit the rowupsert event if the row has not changed", :async, :do_not_defer_events, sidekiq: :fake do
95
101
  if supports_row_diff
96
- Webhookdb::Fixtures.webhook_subscription(service_integration: sint).create
97
- expect(Webhookdb::Jobs::SendWebhook).to receive(:perform_async).once
102
+ it "does not emit the rowupsert event if the row has not changed", :async, :do_not_defer_events, sidekiq: :fake do
103
+ Webhookdb::Fixtures.webhook_subscription(service_integration: sint).create
104
+ expect(Webhookdb::Jobs::SendWebhook).to receive(:perform_async).once
105
+ svc.create_table
106
+ upsert_webhook(svc, body:) # Upsert and make sure the next does not run
107
+ expect do
108
+ upsert_webhook(svc, body:)
109
+ end.to_not publish("webhookdb.serviceintegration.rowupsert")
110
+ expect(Sidekiq).to have_empty_queues
111
+ end
112
+ end
113
+
114
+ it "does not emit the rowupsert event if there are no subscriptions", :async, :do_not_defer_events do
115
+ # No subscription is created so should not publish
98
116
  svc.create_table
99
- upsert_webhook(svc, body:) # Upsert and make sure the next does not run
100
117
  expect do
101
118
  upsert_webhook(svc, body:)
102
119
  end.to_not publish("webhookdb.serviceintegration.rowupsert")
103
- expect(Sidekiq).to have_empty_queues
104
120
  end
105
121
  end
106
122
 
107
- it "does not emit the rowupsert event if there are no subscriptions", :async, :do_not_defer_events do
108
- # No subscription is created so should not publish
109
- svc.create_table
110
- expect do
111
- upsert_webhook(svc, body:)
112
- end.to_not publish("webhookdb.serviceintegration.rowupsert")
113
- end
114
-
115
123
  it "can serve a webhook response" do
116
124
  create_all_dependencies(sint)
117
125
  request = fake_request
@@ -174,7 +182,8 @@ RSpec.shared_examples "a replicator" do |name|
174
182
  end
175
183
  end
176
184
 
177
- RSpec.shared_examples "a replicator with dependents" do |service_name, dependent_service_name|
185
+ RSpec.shared_examples "a replicator with dependents" do |dependent_service_name|
186
+ let(:service_name) { described_class.descriptor.name }
178
187
  let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name:) }
179
188
  let(:svc) { Webhookdb::Replicator.create(sint) }
180
189
  let(:body) { raise NotImplementedError }
@@ -218,7 +227,8 @@ RSpec.shared_examples "a replicator with dependents" do |service_name, dependent
218
227
  end
219
228
  end
220
229
 
221
- RSpec.shared_examples "a replicator dependent on another" do |service_name, dependency_service_name|
230
+ RSpec.shared_examples "a replicator dependent on another" do |dependency_service_name|
231
+ let(:service_name) { described_class.descriptor.name }
222
232
  let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name:) }
223
233
  let(:svc) { Webhookdb::Replicator.create(sint) }
224
234
 
@@ -250,8 +260,9 @@ RSpec.shared_examples "a replicator dependent on another" do |service_name, depe
250
260
  end
251
261
  end
252
262
 
253
- RSpec.shared_examples "a replicator that prevents overwriting new data with old" do |name|
254
- let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name: name) }
263
+ RSpec.shared_examples "a replicator that prevents overwriting new data with old" do
264
+ let(:service_name) { described_class.descriptor.name }
265
+ let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name:) }
255
266
  let(:svc) { Webhookdb::Replicator.create(sint) }
256
267
  let(:old_body) { raise NotImplementedError }
257
268
  let(:new_body) { raise NotImplementedError }
@@ -313,11 +324,12 @@ RSpec.shared_examples "a replicator that prevents overwriting new data with old"
313
324
  end
314
325
  end
315
326
 
316
- RSpec.shared_examples "a replicator that can backfill" do |name|
327
+ RSpec.shared_examples "a replicator that can backfill" do
317
328
  let(:api_url) { "https://fake-url.com" }
329
+ let(:service_name) { described_class.descriptor.name }
318
330
  let(:sint) do
319
331
  Webhookdb::Fixtures.service_integration.create(
320
- service_name: name,
332
+ service_name:,
321
333
  backfill_key: "bfkey",
322
334
  backfill_secret: "bfsek",
323
335
  api_url:,
@@ -427,8 +439,9 @@ end
427
439
 
428
440
  # These shared examples test the way a replicator synthesizes and retrieves information from the API.
429
441
 
430
- RSpec.shared_examples "a replicator that may have a minimal body" do |name|
431
- let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name: name) }
442
+ RSpec.shared_examples "a replicator that may have a minimal body" do
443
+ let(:service_name) { described_class.descriptor.name }
444
+ let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name:) }
432
445
  let(:svc) { Webhookdb::Replicator.create(sint) }
433
446
  let(:body) { raise NotImplementedError }
434
447
  let(:other_bodies) { [] }
@@ -453,8 +466,9 @@ RSpec.shared_examples "a replicator that may have a minimal body" do |name|
453
466
  end
454
467
  end
455
468
 
456
- RSpec.shared_examples "a replicator that deals with resources and wrapped events" do |name|
457
- let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name: name) }
469
+ RSpec.shared_examples "a replicator that deals with resources and wrapped events" do
470
+ let(:service_name) { described_class.descriptor.name }
471
+ let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name:) }
458
472
  let(:svc) { Webhookdb::Replicator.create(sint) }
459
473
  let(:resource_json) { raise NotImplementedError }
460
474
  let(:resource_in_envelope_json) { raise NotImplementedError }
@@ -489,8 +503,9 @@ RSpec.shared_examples "a replicator that deals with resources and wrapped events
489
503
  end
490
504
  end
491
505
 
492
- RSpec.shared_examples "a replicator that uses enrichments" do |name, stores_enrichment_column: true|
493
- let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name: name) }
506
+ RSpec.shared_examples "a replicator that uses enrichments" do |stores_enrichment_column: true|
507
+ let(:service_name) { described_class.descriptor.name }
508
+ let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name:) }
494
509
  let(:svc) { Webhookdb::Replicator.create(sint) }
495
510
  let(:body) { raise NotImplementedError }
496
511
  # Needed if stores_enrichment_column is true
@@ -549,8 +564,9 @@ RSpec.shared_examples "a replicator that uses enrichments" do |name, stores_enri
549
564
  end
550
565
  end
551
566
 
552
- RSpec.shared_examples "a replicator that upserts webhooks only under specific conditions" do |name|
553
- let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name: name) }
567
+ RSpec.shared_examples "a replicator that upserts webhooks only under specific conditions" do
568
+ let(:service_name) { described_class.descriptor.name }
569
+ let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name:) }
554
570
  let(:svc) { Webhookdb::Replicator.create(sint) }
555
571
  let(:incorrect_webhook) { raise NotImplementedError }
556
572
  Webhookdb::SpecHelpers::Whdb.setup_upsert_webhook_example(self)
@@ -574,8 +590,9 @@ end
574
590
 
575
591
  # These shared examples can be used to test replicators that support webhooks.
576
592
 
577
- RSpec.shared_examples "a webhook validating replicator that uses credentials from a dependency" do |name|
578
- let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name: name) }
593
+ RSpec.shared_examples "a webhook validating replicator that uses credentials from a dependency" do
594
+ let(:service_name) { described_class.descriptor.name }
595
+ let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name:) }
579
596
 
580
597
  before(:each) do
581
598
  create_all_dependencies(sint)
@@ -597,8 +614,9 @@ RSpec.shared_examples "a webhook validating replicator that uses credentials fro
597
614
  end
598
615
  end
599
616
 
600
- RSpec.shared_examples "a replicator that processes webhooks synchronously" do |name|
601
- let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name: name) }
617
+ RSpec.shared_examples "a replicator that processes webhooks synchronously" do
618
+ let(:service_name) { described_class.descriptor.name }
619
+ let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name:) }
602
620
  let(:svc) { Webhookdb::Replicator.create(sint) }
603
621
  let(:expected_synchronous_response) { raise NotImplementedError }
604
622
  Webhookdb::SpecHelpers::Whdb.setup_upsert_webhook_example(self)
@@ -620,8 +638,9 @@ end
620
638
 
621
639
  # These shared examples test the intricacies of backfill logic.
622
640
 
623
- RSpec.shared_examples "a backfill replicator that requires credentials from a dependency" do |name|
624
- let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name: name) }
641
+ RSpec.shared_examples "a backfill replicator that requires credentials from a dependency" do
642
+ let(:service_name) { described_class.descriptor.name }
643
+ let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name:) }
625
644
  let(:error_message) { raise NotImplementedError }
626
645
 
627
646
  before(:each) do
@@ -640,12 +659,13 @@ RSpec.shared_examples "a backfill replicator that requires credentials from a de
640
659
  end
641
660
  end
642
661
 
643
- RSpec.shared_examples "a replicator that can backfill incrementally" do |name|
662
+ RSpec.shared_examples "a replicator that can backfill incrementally" do
663
+ let(:service_name) { described_class.descriptor.name }
644
664
  let(:last_backfilled) { raise NotImplementedError, "what should the last_backfilled_at value be to start?" }
645
665
  let(:api_url) { "https://fake-url.com" }
646
666
  let(:sint) do
647
667
  Webhookdb::Fixtures.service_integration.create(
648
- service_name: name,
668
+ service_name:,
649
669
  backfill_key: "bfkey",
650
670
  backfill_secret: "bfsek",
651
671
  api_url:,
@@ -707,6 +727,71 @@ RSpec.shared_examples "a replicator that can backfill incrementally" do |name|
707
727
  end
708
728
  end
709
729
 
730
+ RSpec.shared_examples "a replicator that alerts on backfill auth errors" do
731
+ let(:service_name) { described_class.descriptor.name }
732
+ let(:sint_params) { {} }
733
+ let(:sint) do
734
+ Webhookdb::Fixtures.service_integration.create(
735
+ service_name:,
736
+ backfill_key: "bfkey",
737
+ backfill_secret: "bfsek",
738
+ api_url: "https://fake-url.com",
739
+ **sint_params,
740
+ )
741
+ end
742
+ let(:svc) { Webhookdb::Replicator.create(sint) }
743
+ let(:template_name) { raise NotImplementedError }
744
+
745
+ def stub_service_request
746
+ raise NotImplementedError, "stub the request without setting the return response"
747
+ end
748
+
749
+ def handled_responses
750
+ raise NotImplementedError, "Something like: [[:and_return, {status: 401}], [:and_raise, SocketError.new('hi')]]"
751
+ end
752
+
753
+ def unhandled_response
754
+ raise NotImplementedError, "Something like: [:and_return, {status: 500}]"
755
+ end
756
+
757
+ def insert_required_data_callback
758
+ # See backfiller example
759
+ return ->(*) { return }
760
+ end
761
+
762
+ before(:each) do
763
+ sint.organization.prepare_database_connections
764
+ end
765
+
766
+ after(:each) do
767
+ sint.organization.remove_related_database
768
+ end
769
+
770
+ it "dispatches an alert and returns true for handled errors" do
771
+ create_all_dependencies(sint)
772
+ setup_dependencies(sint, insert_required_data_callback)
773
+ Webhookdb::Fixtures.organization_membership.org(sint.organization).verified.admin.create
774
+ req = stub_service_request
775
+ handled_responses.each { |(m, arg)| req.send(m, arg) }
776
+ handled_responses.count.times do
777
+ backfill(sint)
778
+ end
779
+ expect(req).to have_been_made.times(handled_responses.count)
780
+ expect(Webhookdb::Message::Delivery.all).to contain_exactly(
781
+ have_attributes(template: template_name),
782
+ )
783
+ end
784
+
785
+ it "does not dispatch an alert, and raises the original error, if unhandled" do
786
+ create_all_dependencies(sint)
787
+ setup_dependencies(sint, insert_required_data_callback)
788
+ Webhookdb::Fixtures.organization_membership.org(sint.organization).verified.admin.create
789
+ req = stub_service_request.send(*unhandled_response)
790
+ expect { backfill(sint) }.to raise_error(Amigo::Retry::OrDie)
791
+ expect(req).to have_been_made
792
+ end
793
+ end
794
+
710
795
  RSpec.shared_examples "a replicator that verifies backfill secrets" do
711
796
  let(:correct_creds_sint) { raise NotImplementedError, "what sint should we use to test correct creds?" }
712
797
  let(:incorrect_creds_sint) { raise NotImplementedError, "what sint should we use to test incorrect creds?" }
@@ -748,19 +833,21 @@ RSpec.shared_examples "a replicator that verifies backfill secrets" do
748
833
  end
749
834
  end
750
835
 
751
- RSpec.shared_examples "a replicator with a custom backfill not supported message" do |name|
836
+ RSpec.shared_examples "a replicator with a custom backfill not supported message" do
837
+ let(:service_name) { described_class.descriptor.name }
752
838
  it "has a custom message" do
753
- sint = Webhookdb::Fixtures.service_integration.create(service_name: name)
839
+ sint = Webhookdb::Fixtures.service_integration.create(service_name:)
754
840
  expect(sint.replicator.backfill_not_supported_message).to_not include("You may be looking for one of the following")
755
841
  end
756
842
  end
757
843
 
758
- RSpec.shared_examples "a backfill replicator that marks missing rows as deleted" do |name|
844
+ RSpec.shared_examples "a backfill replicator that marks missing rows as deleted" do
845
+ let(:service_name) { described_class.descriptor.name }
759
846
  let(:deleted_column_name) { raise NotImplementedError }
760
847
  let(:api_url) { "https://fake-url.com" }
761
848
  let(:sint) do
762
849
  Webhookdb::Fixtures.service_integration.create(
763
- service_name: name,
850
+ service_name:,
764
851
  backfill_key: "bfkey",
765
852
  backfill_secret: "bfsek",
766
853
  api_url:,
@@ -813,11 +900,12 @@ RSpec.shared_examples "a backfill replicator that marks missing rows as deleted"
813
900
  end
814
901
  end
815
902
 
816
- RSpec.shared_examples "a replicator that ignores HTTP errors during backfill" do |name|
903
+ RSpec.shared_examples "a replicator that ignores HTTP errors during backfill" do
904
+ let(:service_name) { described_class.descriptor.name }
817
905
  let(:api_url) { "https://fake-url.com" }
818
906
  let(:sint) do
819
907
  Webhookdb::Fixtures.service_integration.create(
820
- service_name: name,
908
+ service_name:,
821
909
  backfill_key: "bfkey",
822
910
  backfill_secret: "bfsek",
823
911
  api_url:,
@@ -856,8 +944,9 @@ RSpec.shared_examples "a replicator that ignores HTTP errors during backfill" do
856
944
  end
857
945
  end
858
946
 
859
- RSpec.shared_examples "a replicator backfilling against the table of its dependency" do |name|
860
- let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name: name) }
947
+ RSpec.shared_examples "a replicator backfilling against the table of its dependency" do
948
+ let(:service_name) { described_class.descriptor.name }
949
+ let(:sint) { Webhookdb::Fixtures.service_integration.create(service_name:) }
861
950
  let(:svc) { Webhookdb::Replicator.create(sint) }
862
951
  let(:dep_svc) { @dep_svc }
863
952
  let(:external_id_col) { raise NotImplementedError }
@@ -21,9 +21,9 @@ class Webhookdb::SyncTarget < Webhookdb::Postgres::Model(:sync_targets)
21
21
  include Appydays::Configurable
22
22
  include Webhookdb::Dbutil
23
23
 
24
- class Deleted < StandardError; end
25
- class InvalidConnection < StandardError; end
26
- class SyncInProgress < StandardError; end
24
+ class Deleted < Webhookdb::WebhookdbError; end
25
+ class InvalidConnection < Webhookdb::WebhookdbError; end
26
+ class SyncInProgress < Webhookdb::WebhookdbError; end
27
27
 
28
28
  # Advisory locks for sync targets use this as the first int, and the id as the second.
29
29
  ADVISORY_LOCK_KEYSPACE = 2_000_000_000
@@ -87,8 +87,10 @@ class Webhookdb::SyncTarget < Webhookdb::Postgres::Model(:sync_targets)
87
87
  dataset_module do
88
88
  def due_for_sync(as_of:)
89
89
  never_synced = Sequel[last_synced_at: nil]
90
- next_due_at = Sequel[:last_synced_at] + (Sequel.lit("INTERVAL '1 second'") * Sequel[:period_seconds])
91
- due_before_now = next_due_at <= as_of
90
+ # Use 'last_synced_at <= (now - internal)' rather than 'last_synced_at + interval <= now'
91
+ # so we can use the last_synced_at index.
92
+ cutoff = (Sequel[as_of].cast("TIMESTAMPTZ") - (Sequel.lit("INTERVAL '1 second'") * Sequel[:period_seconds]))
93
+ due_before_now = Sequel[:last_synced_at] <= cutoff
92
94
  return self.where(never_synced | due_before_now)
93
95
  end
94
96
  end
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "webhookdb/postgres/model"
4
+
5
+ class Webhookdb::SystemLogEvent < Webhookdb::Postgres::Model(:system_log_events)
6
+ plugin :text_searchable, terms: [:title, :body]
7
+
8
+ many_to_one :actor, class: "Webhookdb::Customer"
9
+ end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Webhookdb
4
- VERSION = "1.3.1"
4
+ VERSION = "1.4.0"
5
5
  end
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "webhookdb/jobs/webhook_subscription_delivery_attempt"
3
+ require "webhookdb/jobs/webhook_subscription_delivery_event"
4
4
 
5
5
  # Webhook subscriptions have a few parts:
6
6
  #
data/lib/webhookdb.rb CHANGED
@@ -35,27 +35,33 @@ module Webhookdb
35
35
  include Appydays::Configurable
36
36
  extend Webhookdb::MethodUtilities
37
37
 
38
+ # Base class for all WebhookDB errors.
39
+ class WebhookdbError < StandardError; end
40
+
41
+ # Class for errors that usually should not be rescued from.
42
+ class ProgrammingError < WebhookdbError; end
43
+
38
44
  # Error raised when we cannot take an action
39
45
  # because some condition has not been set up right.
40
- class InvalidPrecondition < StandardError; end
46
+ class InvalidPrecondition < ProgrammingError; end
41
47
 
42
48
  # Error raised when, after we take an action,
43
49
  # something we expect to have changed has not changed.
44
- class InvalidPostcondition < StandardError; end
50
+ class InvalidPostcondition < ProgrammingError; end
45
51
 
46
52
  # Some invariant has been violated, which we never expect to see.
47
- class InvariantViolation < StandardError; end
53
+ class InvariantViolation < ProgrammingError; end
48
54
 
49
55
  # Error raised when a customer gives us some invalid input.
50
56
  # Allows the library to raise the error with the message,
51
57
  # and is caught automatically by the service as a 400.
52
- class InvalidInput < StandardError; end
58
+ class InvalidInput < WebhookdbError; end
53
59
 
54
60
  # Raised when an organization's database cannot be modified.
55
- class DatabaseLocked < StandardError; end
61
+ class DatabaseLocked < WebhookdbError; end
56
62
 
57
63
  # Used in various places that need to short-circuit code in regression mode.
58
- class RegressionModeSkip < StandardError; end
64
+ class RegressionModeSkip < WebhookdbError; end
59
65
 
60
66
  APPLICATION_NAME = "Webhookdb"
61
67
  RACK_ENV = Appydays::Configurable.fetch_env(["RACK_ENV", "RUBY_ENV"], "development")
@@ -93,6 +99,8 @@ module Webhookdb
93
99
  end
94
100
  end
95
101
 
102
+ def self.admin_url = self.api_url
103
+
96
104
  # Regression mode is true when we re replaying webhooks locally,
97
105
  # or for some other reason, want to disable certain checks we use in production.
98
106
  # For example, we may want to ignore certain errors (like if integrations are missing dependency rows),
@@ -134,7 +142,23 @@ module Webhookdb
134
142
  # :section: Errors
135
143
  #
136
144
 
137
- class LockFailed < StandardError; end
145
+ class LockFailed < WebhookdbError; end
146
+
147
+ # This exception is rescued in the API and returned to the caller via +merror!+.
148
+ # This is mostly used in synchronously-processed replicators that need to return
149
+ # very specific types of errors during processing.
150
+ # This is very rare.
151
+ class ExceptionCarrier < WebhookdbError
152
+ attr_reader :status, :code, :more, :headers
153
+
154
+ def initialize(status, message, code: nil, more: {}, headers: {})
155
+ super(message)
156
+ @status = status
157
+ @code = code
158
+ @more = more
159
+ @headers = headers
160
+ end
161
+ end
138
162
 
139
163
  ### Generate a key for the specified Sequel model +instance+ and
140
164
  ### any additional +parts+ that can be used for idempotent requests.