webhookdb 1.3.1 → 1.5.0

data/lib/webhookdb/replicator/partitionable_mixin.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+# Mixin for replicators that support partitioning.
+# Partitioning is currently in beta,
+# with the following limitations/context:
+#
+# - They cannot be created from the CLI.
+#   Because the partitions must be created during the CREATE TABLE call,
+#   the partition_value must be set immediately on creation,
+#   or CREATE TABLE must be deferred.
+# - CLI support would also require making sure this field isn't edited.
+#   This is an annoying change, so we're putting it off for now.
+# - Instead, partitioned replicators must be created in the console.
+# - The number of HASH partitions cannot be changed;
+#   there is no good way to handle this in Postgres so we don't bother here.
+# - RANGE partitions are not supported.
+#   We need to support creating the partition when the INSERT fails.
+#   But creating the partitioned table definition itself does work/has a shared behavior at least.
+# - Existing replicators cannot be converted to partitioned.
+#   This is theoretically possible, but it seems easier to just start over
+#   with a new replicator.
+# - Instead:
+#   - If this is a 'child' replicator, then create a new parent and this child,
+#     then copy over the parent data, either directly (for icalendar)
+#     or using HTTP requests (like with Plaid or Google) where more logic is required.
+#   - Otherwise, it'll depend on the replicator.
+#   - Then to switch clients using the old replicator, to the new replicator, you can:
+#     - Then turn off all workers.
+#     - Rename the new table to the old, and old table to the new.
+#     - Update the service integrations, so the old one points to the new table name and opaque id,
+#       and the new one points to the old table name and opaque id.
+#
+module Webhookdb::Replicator::PartitionableMixin
+  # The partition method, like Webhookdb::DBAdapter::Partitioning::HASH
+  def partition_method = raise NotImplementedError
+  # The partition column name.
+  # Must be present in +_denormalized_columns+.
+  # @return [Symbol]
+  def partition_column_name = raise NotImplementedError
+  # The value for the denormalized column. For HASH partitioning this would be an integer,
+  # for RANGE partitioning this could be a timestamp, etc.
+  # Takes the resource and returns the value.
+  def partition_value(_resource) = raise NotImplementedError
+
+  def partition? = true
+
+  def partitioning
+    return Webhookdb::DBAdapter::Partitioning.new(by: self.partition_method, column: self.partition_column_name)
+  end
+
+  def _prepare_for_insert(resource, event, request, enrichment)
+    h = super
+    h[self.partition_column_name] = self.partition_value(resource)
+    return h
+  end
+
+  def _upsert_conflict_target
+    return [self.partition_column_name, self._remote_key_column.name]
+  end
+
+  # Convert the given string into a stable MD5-derived hash
+  # that can be stored in a (signed, 4 bit) INTEGER column.
+  def _str2inthash(s)
+    # MD5 is 128 bits/16 bytes/32 hex chars (2 chars per byte).
+    # Integers are 32 bits/4 bytes/8 hex chars.
+    # Grab the first 8 chars and convert it to an integer.
+    unsigned_md5int = Digest::MD5.hexdigest(s)[..8].to_i(16)
+    # Then AND it with a 32 bit bitmask to make sure it fits in 32 bits
+    # (though I'm not entirely sure why the above doesn't result in 32 bits always).
+    unsigned_int32 = unsigned_md5int & 0xFFFFFFFF
+    # Convert it from unsigned (0 to 4.2B) to signed (-2.1B to 2.1B) by subtracting 2.1B
+    # (the max 2 byte integer), as opposed to a 4 byte integer which we're dealing with here.
+    signed_md5int = unsigned_int32 - MAX_16BIT_INT
+    return signed_md5int
+  end
+
+  MAX_16BIT_INT = 2**31
+
+  # Return the partitions belonging to the table.
+  # @param db The organization connection.
+  # @return [Array<Webhookdb::DBAdapter::Partition>]
+  def existing_partitions(db)
+    # SELECT inhrelid::regclass AS child
+    # FROM   pg_catalog.pg_inherits
+    # WHERE  inhparent = 'my_schema.foo'::regclass;
+    parent = self.schema_and_table_symbols.map(&:to_s).join(".")
+    partnames = db[Sequel[:pg_catalog][:pg_inherits]].
+      where(inhparent: Sequel[parent].cast(:regclass)).
+      select_map(Sequel[:inhrelid].cast(:regclass))
+    parent_table = self.dbadapter_table
+    result = partnames.map do |part|
+      suffix = self.partition_suffix(part)
+      Webhookdb::DBAdapter::Partition.new(parent_table:, partition_name: part.to_sym, suffix:)
+    end
+    return result
+  end
+
+  def partition_suffix(partname)
+    return partname[/_[a-zA-Z\d]+$/].to_sym
+  end
+
+  def partition_align_name
+    tblname = self.service_integration.table_name
+    self.service_integration.organization.admin_connection do |db|
+      partitions = self.existing_partitions(db)
+      db.transaction do
+        partitions.each do |partition|
+          next if partition.partition_name.to_s.start_with?(tblname)
+          schema = partition.parent_table.schema.name
+          new_partname = "#{tblname}#{partition.suffix}"
+          db << "ALTER TABLE #{schema}.#{partition.partition_name} RENAME TO #{new_partname}"
+        end
+      end
+    end
+  end
+end

data/lib/webhookdb/replicator/shopify_v1_mixin.rb

@@ -45,7 +45,7 @@ module Webhookdb::Replicator::ShopifyV1Mixin
       field = "api_url"
       value = "https://#{value}.myshopify.com"
     end
-    return super(field, value)
+    return super
   end
 
   def calculate_webhook_state_machine

data/lib/webhookdb/replicator/signalwire_message_v1.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
+require "webhookdb/errors"
 require "webhookdb/signalwire"
+require "webhookdb/messages/error_generic_backfill"
 
 class Webhookdb::Replicator::SignalwireMessageV1 < Webhookdb::Replicator::Base
   include Appydays::Loggable
@@ -50,7 +52,7 @@ class Webhookdb::Replicator::SignalwireMessageV1 < Webhookdb::Replicator::Base
       h = u.host.gsub(/\.signalwire\.com$/, "")
       value = h
     end
-    return super(field, value, attr:)
+    return super
   end
 
   def calculate_backfill_state_machine
@@ -180,4 +182,32 @@ Press 'Show' next to the newly-created API token, and copy it.)
 
     return messages, data["next_page_uri"]
   end
+
+  def on_backfill_error(be)
+    e = Webhookdb::Errors.find_cause(be) do |ex|
+      next true if ex.is_a?(Webhookdb::Http::Error) && ex.status == 401
+      next true if ex.is_a?(::SocketError)
+    end
+    return unless e
+    if e.is_a?(::SocketError)
+      response_status = 0
+      response_body = e.message
+      request_url = "<unknown>"
+      request_method = "<unknown>"
+    else
+      response_status = e.status
+      response_body = e.body
+      request_url = e.uri.to_s
+      request_method = e.http_method
+    end
+    message = Webhookdb::Messages::ErrorGenericBackfill.new(
+      self.service_integration,
+      response_status:,
+      response_body:,
+      request_url:,
+      request_method:,
+    )
+    self.service_integration.organization.alerting.dispatch_alert(message)
+    return true
+  end
 end

data/lib/webhookdb/replicator/sponsy_v1_mixin.rb

@@ -99,7 +99,7 @@ module Webhookdb::Replicator::SponsyV1Mixin
       )
     rescue Webhookdb::Http::Error => e
       raise e unless e.status == 404
-      self.logger.warn("sponsy_404", error: e)
+      self.logger.warn("sponsy_404", e)
       return [], nil
     end
 

data/lib/webhookdb/replicator/transistor_episode_stats_v1.rb

@@ -40,7 +40,6 @@ class Webhookdb::Replicator::TransistorEpisodeStatsV1 < Webhookdb::Replicator::B
       :compound_identity,
       TEXT,
       data_key: "<compound key, see converter>",
-      index: true,
       optional: true,
       converter: CONV_REMOTE_KEY,
     )

data/lib/webhookdb/replicator/transistor_episode_v1.rb

@@ -141,11 +141,17 @@ class Webhookdb::Replicator::TransistorEpisodeV1 < Webhookdb::Replicator::Base
     transcript_url = resource.fetch("attributes").fetch("transcript_url", nil)
     return nil if transcript_url.blank?
     (transcript_url += ".txt") unless transcript_url.end_with?(".txt")
-    resp = Webhookdb::Http.get(
-      transcript_url,
-      logger: self.logger,
-      timeout: Webhookdb::Transistor.http_timeout,
-    )
+    begin
+      resp = Webhookdb::Http.get(
+        transcript_url,
+        logger: self.logger,
+        timeout: Webhookdb::Transistor.http_timeout,
+      )
+    rescue Webhookdb::Http::Error => e
+      # Not sure why this happens, but nothing we can do if it does.
+      return nil if e.status == 404
+      raise e
+    end
     transcript_text = resp.body
     return {transcript_text:}
   end

data/lib/webhookdb/replicator/webhook_request.rb

@@ -6,4 +6,12 @@ class Webhookdb::Replicator::WebhookRequest < Webhookdb::TypedStruct
   # When a webhook is processed synchronously, this will be set to the Rack::Request.
   # Normal (async) webhook processing does not have this available.
   attr_accessor :rack_request
+
+  JSON_KEYS = ["body", "headers", "path", "method"].freeze
+  def as_json
+    return JSON_KEYS.each_with_object({}) do |k, h|
+      v = self.send(k)
+      h[k] = v.as_json unless v.nil?
+    end
+  end
 end

data/lib/webhookdb/replicator.rb

@@ -15,11 +15,14 @@ class Webhookdb::Replicator
   PLUGIN_DIR = Pathname(__FILE__).dirname + PLUGIN_DIRNAME
 
   # Raised when there is no service registered for a name.
-  class Invalid < StandardError; end
+  class Invalid < Webhookdb::WebhookdbError; end
 
   # Raised when credentials to interact with a service are not set up.
   # Usually this is due to a missing dependency.
-  class CredentialsMissing < StandardError; end
+  class CredentialsMissing < Webhookdb::WebhookdbError; end
+
+  # Raised when the columns or indices for a replicator are invalid.
+  class BrokenSpecification < Webhookdb::WebhookdbError; end
 
   # Statically describe a replicator.
   class Descriptor < Webhookdb::TypedStruct
@@ -142,7 +145,7 @@ class Webhookdb::Replicator
   end
 
   class IndexSpec < Webhookdb::TypedStruct
-    attr_reader :columns, :where
+    attr_reader :columns, :where, :identifier
   end
 
   class << self

data/lib/webhookdb/service/helpers.rb

@@ -16,6 +16,10 @@ module Webhookdb::Service::Helpers
     return Webhookdb::Service.logger
   end
 
+  def set_request_tags(tags)
+    Webhookdb::Service::Middleware::RequestLogger.set_request_tags(tags)
+  end
+
   # Return the currently-authenticated user,
   # or respond with a 401 if there is no authenticated user.
   def current_customer

data/lib/webhookdb/service/middleware.rb

@@ -128,12 +128,16 @@ module Webhookdb::Service::Middleware
     def request_tags(env)
       tags = super
       begin
-        tags[:customer_id] = env["warden"].user(:customer)&.id || 0
+        c = env["warden"].user(:customer)
       rescue Sequel::DatabaseError
         # If we cant hit the database, ignore this for now.
         # We run this code on all code paths, including those that don't need the customer,
         # and we want those to run even if the DB is down (like health checks, for example).
-        nil
+        c = nil
+      end
+      if c
+        tags[:customer_id] = c.id || 0
+        tags[:customer] = c.email
       end
       return tags
     end

data/lib/webhookdb/service/view_api.rb

@@ -3,7 +3,7 @@
 # Mixin for Grape API endpoints that use HTML rendering.
 # This isn't tested well enough.
 module Webhookdb::Service::ViewApi
-  class FormError < StandardError
+  class FormError < Webhookdb::WebhookdbError
     attr_reader :status
 
     def initialize(msg, status=400)

data/lib/webhookdb/service.rb

@@ -163,6 +163,10 @@ class Webhookdb::Service < Grape::API
     error!(e.message, 405)
   end
 
+  rescue_from Webhookdb::ExceptionCarrier do |e|
+    merror!(e.status, e.message, code: e.code, more: e.more, headers: e.headers)
+  end
+
   rescue_from Webhookdb::LockFailed do |_e|
     merror!(
       409,
@@ -191,17 +195,7 @@ class Webhookdb::Service < Grape::API
     status = e.respond_to?(:status) ? e.status : 500
     error_id = SecureRandom.uuid
     error_signature = Digest::MD5.hexdigest("#{e.class}: #{e.message}")
-
-    Webhookdb::Service.logger.error "[%s] Uncaught %p in service: %s" %
-      [error_id, e.class, e.message]
-    Webhookdb::Service.logger.debug { e.backtrace.join("\n") }
-    if ENV["PRINT_API_ERROR"]
-      puts e
-      puts e.backtrace
-    end
-
     more = {error_id:, error_signature:}
-
     if Webhookdb::Service.devmode
       msg = e.message
       more[:backtrace] = e.backtrace.join("\n")
@@ -210,6 +204,12 @@ class Webhookdb::Service < Grape::API
       msg = "An internal error occurred of type #{error_signature}. Error ID: #{error_id}"
     end
     Webhookdb::Service.logger.error("api_exception", {error_id:, error_signature:}, e)
+    Webhookdb::Service.logger.debug { e.backtrace.join("\n") }
+    if ENV["PRINT_API_ERROR"]
+      puts e
+      puts e.backtrace
+    end
+
     merror!(status, msg, code: "api_error", more:)
   end
 

data/lib/webhookdb/service_integration.rb

@@ -6,6 +6,8 @@ require "webhookdb/postgres/model"
 require "sequel/plugins/soft_deletes"
 
 class Webhookdb::ServiceIntegration < Webhookdb::Postgres::Model(:service_integrations)
+  include Webhookdb::Admin::Linked
+
   class TableRenameError < Webhookdb::InvalidInput; end
 
   # We limit the information that a user can access through the CLI to these fields.
@@ -70,7 +72,7 @@ class Webhookdb::ServiceIntegration < Webhookdb::Postgres::Model(:service_integr
                 end)
 
   many_to_one :depends_on, class: self
-  one_to_many :dependents, key: :depends_on_id, class: self
+  one_to_many :dependents, key: :depends_on_id, class: self, order: :id
   one_to_many :sync_targets, class: "Webhookdb::SyncTarget"
 
   # @return [Webhookdb::ServiceIntegration]
@@ -139,10 +141,21 @@ class Webhookdb::ServiceIntegration < Webhookdb::Postgres::Model(:service_integr
         select { |si| si.service_name == dep_descr.name }
   end
 
+  # Return all dependents (integrations that depend on this one), breadth-first
+  # (that is, all children before grandchildren).
+  # @return [Array<Webhookdb::ServiceIntegration>]
   def recursive_dependents
     return self.dependents + self.dependents.flat_map(&:recursive_dependents)
   end
 
+  # Return all service integrations this one depends on,
+  # in closest-ancestor order (that is, parent before grandparent).
+  # @return [Array<Webhookdb::ServiceIntegration>]
+  def recursive_dependencies
+    return [] if self.depends_on.nil?
+    return [self.depends_on].concat(self.depends_on.recursive_dependencies)
+  end
+
   def destroy_self_and_all_dependents
     self.dependents.each(&:destroy_self_and_all_dependents)
 
@@ -329,6 +342,11 @@ class Webhookdb::ServiceIntegration < Webhookdb::Postgres::Model(:service_integr
   # @!attribute skip_webhook_verification
   #   @return [Boolean] Set this to disable webhook verification on this integration.
   #                     Useful when replaying logged webhooks.
+
+  # @!attribute partition_value
+  #   @return [Integer] Value to control partitioning. For replicators that use hash partitioning,
+  #                     this defines the number of partitions. For other partition types, like range,
+  #                     the meaning of this value depends on the replicator itself.
 end
 
 # Table: service_integrations

data/lib/webhookdb/signalwire.rb

@@ -9,7 +9,7 @@ module Webhookdb::Signalwire
 
   configurable(:signalwire) do
     setting :http_timeout, 30
-    setting :sms_allowlist, [], convert: ->(s) { s.split }
+    setting :sms_allowlist, [], convert: lambda(&:split)
   end
 
   def self.send_sms(from:, to:, body:, project_id:, **kw)

data/lib/webhookdb/spec_helpers/async.rb

@@ -22,10 +22,6 @@ module Webhookdb::SpecHelpers::Async
         Webhookdb::Slack.http_client = Webhookdb::Slack::NoOpHttpClient.new
         Webhookdb::Slack.suppress_all = false
       end
-      if example.metadata[:sentry]
-        Webhookdb::Sentry.dsn = "http://public:secret@not-really-sentry.nope/someproject"
-        Webhookdb::Sentry.run_after_configured_hooks
-      end
     end
 
     context.after(:each) do |example|

data/lib/webhookdb/spec_helpers/sentry.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "webhookdb/spec_helpers"
+require "webhookdb/sentry"
+
+module Webhookdb::SpecHelpers::Sentry
+  def self.included(context)
+    context.before(:each) do |example|
+      if example.metadata[:sentry]
+        # We need to fake doing what Sentry would be doing for initialization,
+        # so we can assert it has the right data in its scope.
+        Webhookdb::Sentry.dsn = "https://public:secret@test-sentry.webhookdb.com/whdb"
+        hub = Sentry::Hub.new(
+          Sentry::Client.new(Sentry::Configuration.new),
+          Sentry::Scope.new,
+        )
+        expect(Sentry).to_not be_initialized
+        Sentry.instance_variable_set(:@main_hub, hub)
+        expect(Sentry).to be_initialized
+      end
+    end
+
+    context.after(:each) do |example|
+      if example.metadata[:sentry]
+        Webhookdb::Sentry.reset_configuration
+        expect(Sentry).to_not be_initialized
+      end
+    end
+
+    super
+  end
+end