webhookdb 0.1.0

data/lib/webhookdb/jobs/webhook_resource_notify_integrations.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+require "stripe"
+require "webhookdb/stripe"
+
+# All the 'webhookdb' integrations are internal, so this endpoint is not hardened thoroughly.
+# If we were to open this up or use it on prod, we need to harden it to support
+# more and slower endpoints (at least by fanning out the work).
+class Webhookdb::Jobs::WebhookdbResourceNotifyIntegrations
+  extend Webhookdb::Async::Job
+
+  # As we add more resources, modify this wildcard
+  on "webhookdb.customer.created"
+
+  sidekiq_options queue: "netout"
+
+  def _perform(event)
+    cu = self.lookup_model(Webhookdb::Customer, event)
+    Webhookdb::ServiceIntegration.where(service_name: "webhookdb_customer_v1").each do |sint|
+      Webhookdb::Http.post(
+        sint.replicator.webhook_endpoint,
+        cu.values,
+        headers: {"Whdb-Secret" => sint.webhook_secret},
+        logger: self.logger,
+        timeout: nil,
+      )
+    end
+  end
+end

data/lib/webhookdb/jobs/webhook_subscription_delivery_attempt.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "amigo/queue_backoff_job"
+require "amigo/durable_job"
+require "webhookdb/async/job"
+require "webhookdb/jobs"
+
+class Webhookdb::Jobs::WebhookSubscriptionDeliveryEvent
+  include Sidekiq::Worker
+  include Amigo::DurableJob
+  include Amigo::QueueBackoffJob
+
+  sidekiq_options queue: "netout"
+
+  def dependent_queues
+    return ["critical"]
+  end
+
+  def perform(delivery_id)
+    delivery = Webhookdb::WebhookSubscription::Delivery[delivery_id]
+    Webhookdb::Async::JobLogger.with_log_tags(
+      webhook_subscription_delivery_id: delivery.id,
+      webhook_subscription_id: delivery.webhook_subscription_id,
+      organization_key: delivery.webhook_subscription.fetch_organization,
+    ) do
+      delivery.attempt_delivery
+    end
+  end
+end

data/lib/webhookdb/jobs.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+module Webhookdb::Jobs
+end

data/lib/webhookdb/json.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require "json"
+require "oj"
+require "active_support/json"
+
+module Webhookdb
+  # Helpers for JSON encoding and (to a lesser degree) decoding.
+  # In general, decoding JSON in Ruby is simple, as long as we
+  # don't try and deal with automatically converting types (which we don't).
+  #
+  # But encoding JSON is complex, like for example encoding a Time.
+  # ActiveSupport uses #as_json to convert Ruby types to JSON-native types.
+  # In order to use this, we must call Webhookdb::Json.encode,
+  # or go through ActiveSupport (which we patch here).
+  #
+  # That is, using JSON.dump(Time.now) would give you '"1970-01-01 02:00:00 +0200"'
+  # but using Webhookdb::Json.encode(Time.now) gives you '"1970-01-01T02:00:00.123+02:00"'
+  #
+  # Anyway, this is all largely under the hood and handled for us using ActiveSupport,
+  # but while we switched Yajl to Oj, we went ahead and added this shim to bypass
+  # the worst part of ActiveSupport (its still very tied to Rails needs).
+  module Json
+    DEFAULT_OPTIONS = {}.freeze
+    PRETTY_OPTIONS = {indent: "  ", space: " ", object_nl: "\n", array_nl: "\n"}.freeze
+
+    # Based on ActiveSupport::JSON::Encoding::JSONGemEncoder.
+    # Removes the HTML escaping code (which is slow), but otherwise continues to depend
+    # on the as_json approach.
+    class Encoder
+      attr_reader :options
+
+      def initialize(options=nil)
+        @options = options || DEFAULT_OPTIONS
+      end
+
+      def encode(value)
+        return stringify(jsonify(value.as_json(@options.dup)))
+      end
+
+      # Convert an object into a "JSON-ready" representation composed of
+      # primitives like Hash, Array, String, Numeric,
+      # and +true+/+false+/+nil+.
+      # Recursively calls #as_json to the object to recursively build a
+      # fully JSON-ready object.
+      #
+      # This allows developers to implement #as_json without having to
+      # worry about what base types of objects they are allowed to return
+      # or having to remember to call #as_json recursively.
+      #
+      # Note: the +options+ hash passed to +object.to_json+ is only passed
+      # to +object.as_json+, not any of this method's recursive +#as_json+
+      # calls.
+      def jsonify(value)
+        case value
+          when Rational
+            value.to_s
+          when String, Numeric, NilClass, TrueClass, FalseClass
+            value.as_json
+          when Hash
+            result = {}
+            value.each do |k, v|
+              result[jsonify(k)] = jsonify(v)
+            end
+            result
+          when Array
+            value.map { |v| jsonify(v) }
+          else
+            jsonify value.as_json
+        end
+      end
+
+      def stringify(jsonified)
+        # If this breaks because we actually need to handle more types,
+        # add them to jsonify, like Rational.
+        #
+        # We can't use `mode: :rails` here since we can't control
+        # whether to use html-escaping (we don't want to use it) without using `Oj.optimize_rails`,
+        # which causes other problems- we want to use ISO8601 encoding with millsecond precision,
+        # but we get JSON-gem-style formatting (even if setting flags to modify ActiveSupport params)
+        # when we use `optimize-rails` (but not mode: :rails, since we stringify the time beforehand).
+        return ::Oj.dump(jsonified, mode: :strict, **@options)
+      end
+    end
+
+    class << self
+      # Dump as compact, standard JSON, using iso8601 for times.
+      def encode(value, options=nil)
+        Encoder.new(options).encode(value)
+      end
+
+      # Dump as pretty JSON, similar to JSON.pretty_generate but with iso8601 times.
+      def pretty_generate(value)
+        Webhookdb::Json.encode(value, PRETTY_OPTIONS)
+      end
+    end
+  end
+end
+
+# This stomps the JSON.load, etc., methods, to use the faster (and still compatible) Oj version.
+Oj.mimic_JSON
+
+# Use the shim encoder rather than the ActiveSupport one.
+# This mostly handles #to_json calls.
+ActiveSupport::JSON::Encoding.json_encoder = Webhookdb::Json::Encoder
+
+# Replace ActiveSupport::JSON.encode so it calls Oj directly.
+# This isn't strictly necessary (since we set json_encoder), but skips an extra method call.
+module ActiveSupport::JSON
+  def self.encode(value, options=nil)
+    return Webhookdb::Json.encode(value, options)
+  end
+end

data/lib/webhookdb/liquid/expose.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "liquid"
+
+require "webhookdb"
+
+# Allow customers to expose variables from a template
+# using custom blocks and registers.
+# See https://github.com/Shopify/liquid/wiki/Liquid-for-Programmers#create-your-own-tag-blocks
+# for more info about blocks.
+# See https://github.com/Shopify/liquid/wiki/Liquid-for-Programmers#difference-between-assigns-and-registers
+# for info about "registers", which are used as template-render-specific mutable state
+# (so we can mutate it in the tag/block, then inspect the mutated value after-the-fact).
+class Webhookdb::Liquid::Expose < Liquid::Block
+  def initialize(tag_name, var_name, options)
+    super
+    @var_name = var_name.strip.to_sym
+  end
+
+  def render(context)
+    content = super
+    context.registers[@var_name] = content
+    ""
+  end
+end
+
+Liquid::Template.register_tag("expose", Webhookdb::Liquid::Expose)

data/lib/webhookdb/liquid/filters.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "liquid"
+require "webhookdb"
+
+module Webhookdb::Liquid::Filters
+  def humanize(input)
+    return input.humanize
+  end
+
+  def money(input)
+    return input.format
+  end
+end
+
+Liquid::Template.register_filter(Webhookdb::Liquid::Filters)

data/lib/webhookdb/liquid/liquification.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# Liquification provides a to_liquid method that returns a wrapped object,
+# so that any object can be used in a liquid template
+# (normally only things like basic types, and objects with to_liquid,
+# can be used in templates). This allows us to use Liquid filters for rendering
+# custom types, rather than presenters. We prefer filters because
+# it keeps display logic in the view, rather than the backing 'template'
+# having to choose the presenter.
+class Webhookdb::Liquid::Liquification
+  def initialize(wrapped)
+    @wrapped = wrapped
+  end
+
+  def method_missing(m, *, &)
+    return @wrapped.respond_to?(m) ? @wrapped.send(m, *, &) : super
+  end
+
+  def respond_to_missing?(m, include_private: false)
+    return super || @wrapped.respond_to?(m)
+  end
+
+  def to_liquid
+    return self
+  end
+end

data/lib/webhookdb/liquid/partial.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "webhookdb"
+require "liquid"
+
+class Webhookdb::Liquid::Partial < Liquid::Include
+  def initialize(tag_name, name, options)
+    name = "'partials/#{Regexp.last_match(1)}'" if name =~ /['"]([a-z0-9_]+)['"]/
+    super(tag_name, name, options)
+  end
+end
+Liquid::Template.register_tag("partial", Webhookdb::Liquid::Partial)

data/lib/webhookdb/logged_webhook/resilient.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+class Webhookdb::LoggedWebhook::Resilient
+  def logger = Webhookdb::LoggedWebhook.logger
+
+  def database_urls = Webhookdb::LoggedWebhook.available_resilient_database_urls
+
+  def insert(kwargs)
+    return Webhookdb::LoggedWebhook.dataset.insert(kwargs)
+  rescue Sequel::DatabaseError => e
+    service_integration_opaque_id = kwargs.fetch(:service_integration_opaque_id)
+    str_payload = JSON.dump(kwargs)
+    self.database_urls.each do |url|
+      next unless self.write_to(url, service_integration_opaque_id, str_payload)
+      self.logger.warn "resilient_insert_handled", error: e, **self._dburl_log_kwargs(url)
+      return true
+    end
+    self.logger.error "resilient_insert_unhandled", error: e, logged_webhook_kwargs: kwargs
+    raise
+  end
+
+  def write_to(dburl, service_integration_opaque_id, str_payload)
+    tblname = Webhookdb::LoggedWebhook.resilient_table_name
+    Sequel.connect(dburl, single_threaded: true) do |db|
+      begin
+        db.create_table?(tblname.to_sym) do
+          primary_key :pk
+          text :service_integration_opaque_id
+          text :json_payload
+        end
+      rescue Sequel::UniqueConstraintViolation
+        # We cannot avoid this race condition. If needed, we can optimize this, but it's a pain
+        # so don't worry about it for now.
+        nil
+      end
+      db.from(tblname).insert(service_integration_opaque_id:, json_payload: str_payload)
+    end
+    return true
+  rescue StandardError => e
+    self.logger.debug "resilient_insert_failure", error: e, **self._dburl_log_kwargs(dburl)
+    return false
+  end
+
+  def _dburl_log_kwargs(dburl)
+    u = URI(dburl)
+    return {fallback_database_host: u.host, fallback_database_name: u.path}
+  end
+
+  # - For each (reachable) database:
+  # - Select 1 row, with a lock
+  # - Replay the webhook
+  # - On success, delete the row
+  # - On failure, process the next row
+  def replay
+    tblname = Webhookdb::LoggedWebhook.resilient_table_name
+    begin
+      Webhookdb::LoggedWebhook.db.execute("SELECT 1=1")
+    rescue Sequel::DatabaseError
+      self.logger.debug("resilient_replay_primary_db_not_ready")
+      return nil
+    end
+    replayed = 0
+    self.database_urls.each do |url|
+      Sequel.connect(url) do |rdb|
+        has_more = true
+        # Keep track of the last pk we've replayed, so we can grab the next available one.
+        # Otherwise we can end up spinning on the same one, especially with other threads.
+        seen_pk = 0
+        while has_more
+          # Each row must be processed in a transaction
+          rdb.transaction do
+            row = rdb.from(tblname).where { pk > seen_pk }.for_update.skip_locked.order(:pk).limit(1).first
+            if row.nil?
+              has_more = false
+              break # The break only works for the transaction
+            end
+            pk = row.fetch(:pk)
+            seen_pk = pk
+            payload = JSON.parse(row.fetch(:json_payload))
+            # We replay the webhook from a separate job
+            # so it can be done idempotently/exclusively.
+            lwh = Webhookdb::LoggedWebhook.create(payload)
+            lwh.replay_async
+            replayed += 1
+            rdb.from(tblname).where(pk:).delete
+          end
+        end
+      end
+      return replayed
+    rescue Sequel::DatabaseError
+      self.logger.debug("resilient_replay_fallback_unavailable", **self._dburl_log_kwargs(url))
+      return nil
+    end
+  end
+end

data/lib/webhookdb/logged_webhook.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require "appydays/configurable"
+
+require "webhookdb/postgres/model"
+
+class Webhookdb::LoggedWebhook < Webhookdb::Postgres::Model(:logged_webhooks)
+  include Appydays::Configurable
+
+  class << self
+    attr_accessor :available_resilient_database_urls
+  end
+
+  configurable(:logged_webhooks) do
+    # Space-separated URLs to use for resilient/high availability writes.
+    setting :resilient_database_urls, [], convert: ->(s) { s.split.map(&:strip) }
+    setting :resilient_database_env_vars, [], convert: ->(s) { s.split.map(&:strip) }
+    setting :resilient_table_name, "_logged_webhooks_resilient_writes"
+    # Using /replay can send this many webhooks in one request.
+    # Making this too high can cause timeouts- the caller may have to make multiple calls instead.
+    setting :maximum_replay_interval_hours, 4
+    # Webhooks this old cannot be replayed; they are probably truncated anyway.
+    setting :maximum_replay_history_hours, 7 * 24
+
+    after_configured do
+      self.available_resilient_database_urls = self.resilient_database_urls.dup
+      self.available_resilient_database_urls.concat(self.resilient_database_env_vars.map { |e| ENV.fetch(e, nil) })
+    end
+  end
+
+  many_to_one :organization, class: "Webhookdb::Organization"
+
+  many_to_one :service_integration,
+              class: "Webhookdb::ServiceIntegration",
+              key: :service_integration_opaque_id,
+              primary_key: :opaque_id
+
+  DELETE_UNOWNED = 14.days
+  DELETE_SUCCESSES = 90.days
+  TRUNCATE_SUCCESSES = 7.days
+  DELETE_FAILURES = 90.days
+  TRUNCATE_FAILURES = 30.days
+  # When we retry a request, set this so we know not to re-log it.
+  RETRY_HEADER = "Whdb-Logged-Webhook-Retry"
+  # When we retry a request, these headers
+  # must come from the Ruby client, NOT the original request.
+  NONOVERRIDABLE_HEADERS = [
+    "Accept-Encoding",
+    "Accept",
+    "Host",
+    "Version",
+  ].to_set
+  # These headers have been added by Heroku/our web host,
+  # so should not be part of the retry.
+  WEBHOST_HEADERS = [
+    "Connection",
+    "Connect-Time",
+    "X-Request-Id",
+    "X-Forwarded-For",
+    "X-Request-Start",
+    "Total-Route-Time",
+    "X-Forwarded-Port",
+    "X-Forwarded-Proto",
+    "Via",
+  ].to_set
+
+  # Trim logged webhooks to keep this table to a reasonable size.
+  # The current trim algorithm and rationale is:
+  #
+  # - Logs that belong to inserts that were not part of an org are for our internal use only.
+  #   They usually indicate an integration that was misconfigured, or is for an org that doesn't exist.
+  #   We keep these around for 2 weeks (they are always errors since they have no org).
+  #   Ideally we investigate and remove them before that.
+  #   We may need to 'block' certain opaque ids from being logged in the future,
+  #   if for example we cannot get a client to turn off a misconfigured webhook.
+  # - Successful webhooks get their contents (request body and headers)
+  #   _truncated_ after 7 days (but the webhook row remains).
+  #   Usually we don't need to worry about these so in theory we can avoid logging verbose info at all.
+  # - Successful webhooks are deleted entirely after 90 days.
+  #   Truncated webhooks are useful for statistics,
+  #   but we can remove them earlier in the future.
+  # - Failed webhooks get their contents truncated after 30 days,
+  #   but the webhook row remains. We have a longer truncation date
+  #   so we have more time to investigate.
+  # - Error webhooks are deleted entirely after 90 days.
+  def self.trim(now: Time.now)
+    owned = self.exclude(organization_id: nil)
+    unowned = self.where(organization_id: nil)
+    successes = owned.where { response_status < 400 }
+    failures = owned.where { response_status >= 400 }
+    # Delete old unowned
+    unowned.where { inserted_at < now - DELETE_UNOWNED }.delete
+    # Delete successes first so they don't have to be truncated
+    successes.where { inserted_at < now - DELETE_SUCCESSES }.delete
+    self.truncate_dataset(successes.where { inserted_at < now - TRUNCATE_SUCCESSES })
+    # Delete failures
+    failures.where { inserted_at < now - DELETE_FAILURES }.delete
+    self.truncate_dataset(failures.where { inserted_at < now - TRUNCATE_FAILURES })
+  end
+
+  # Send instances back in 'through the front door' of this API.
+  # Return is a partition of [logs with 2xx responses, others].
+  # Generally you can safely call `truncate_logs(result[0])`,
+  # or pass in (truncate_successful: true).
+  def self.retry_logs(instances, truncate_successful: false)
+    successes, failures = instances.partition do |lw|
+      uri = URI(Webhookdb.api_url + lw.request_path)
+      req = Net::HTTP::Post.new(uri.path, {"Content-Type" => "application/json"})
+      req.body = lw.request_body
+      # This is going to have these headers:
+      # ["content-type", "accept-encoding", "accept", "user-agent", "host"]
+      # We want to keep all of these, except if user-agent or content-type were set
+      # in the original request; then we want to use those.
+      # Additionally, there are a whole set of headers we'll find on our webserver
+      # that are added by our web platform, which we do NOT want to include.
+      lw.request_headers.each do |k, v|
+        next if Webhookdb::LoggedWebhook::WEBHOST_HEADERS.include?(k)
+        next if Webhookdb::LoggedWebhook::NONOVERRIDABLE_HEADERS.include?(k)
+        req[k] = v
+      end
+      req[Webhookdb::LoggedWebhook::RETRY_HEADER] = lw.id
+      resp = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https") do |http|
+        http.request(req)
+      end
+      resp.code.to_i < 400
+    end
+    self.truncate_logs(*successes) if truncate_successful
+    return successes, failures
+  end
+
+  def retry_one(truncate_successful: false)
+    _, bad = self.class.retry_logs([self], truncate_successful:)
+    return bad.empty?
+  end
+
+  def replay_async
+    return self.publish_immediate("replay", self.id)
+  end
+
+  # Truncate the logs id'ed by the given instances.
+  # Instances are NOT modified; you need to .refresh to see truncated values.
+  def self.truncate_logs(*instances)
+    ds = self.where(id: instances.map(&:id))
+    return self.truncate_dataset(ds)
+  end
+
+  def self.truncate_dataset(ds)
+    return ds.update(request_body: "", request_headers: "{}", truncated_at: Time.now)
+  end
+
+  def truncated?
+    return self.truncated_at ? true : false
+  end
+
+  # Insert the logged webhook, and fall back to inserting into the configured
+  # available_resilient_database_urls. If none are inserted successfully, raise the error;
+  # otherwise, swallow the insert error and more on.
+  #
+  # Note that these resilient inserts are MUCH slower than normal inserts;
+  # they require a separate database connection, CREATE TABLE call, etc.
+  # But it's a reasonable way to handle when the database is down.
+  def self.resilient_insert(**kwargs)
+    Resilient.new.insert(kwargs)
+  end
+
+  # Replay and delete all rows in the resilient database tables.
+  def self.resilient_replay
+    Resilient.new.replay
+  end
+end
+
+require "webhookdb/logged_webhook/resilient"
+
+# Table: logged_webhooks
+# ---------------------------------------------------------------------------------------------------------------------------
+# Columns:
+#  id                            | bigint                   | PRIMARY KEY DEFAULT nextval('logged_webhooks_id_seq'::regclass)
+#  inserted_at                   | timestamp with time zone | NOT NULL DEFAULT now()
+#  truncated_at                  | timestamp with time zone |
+#  request_body                  | text                     | NOT NULL
+#  request_headers               | jsonb                    | NOT NULL
+#  response_status               | smallint                 | NOT NULL
+#  service_integration_opaque_id | text                     | NOT NULL
+#  organization_id               | integer                  |
+#  request_method                | text                     | NOT NULL
+#  request_path                  | text                     | NOT NULL
+# Indexes:
+#  logged_webhooks_pkey                                | PRIMARY KEY btree (id)
+#  logged_webhooks_inserted_at_index                   | btree (inserted_at)
+#  logged_webhooks_organization_id_index               | btree (organization_id)
+#  logged_webhooks_service_integration_opaque_id_index | btree (service_integration_opaque_id)
+# Foreign key constraints:
+#  logged_webhooks_organization_id_fkey | (organization_id) REFERENCES organizations(id) ON DELETE CASCADE
+# ---------------------------------------------------------------------------------------------------------------------------

data/lib/webhookdb/message/body.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "webhookdb/postgres/model"
+
+require "webhookdb/message"
+
+class Webhookdb::Message::Body < Webhookdb::Postgres::Model(:message_bodies)
+  plugin :timestamps
+
+  many_to_one :delivery, class: "Webhookdb::Message::Delivery"
+end
+
+# Table: message_bodies
+# ----------------------------------------------------------------------------------------------------
+# Columns:
+#  id          | integer | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY
+#  content     | text    | NOT NULL
+#  mediatype   | text    | NOT NULL
+#  delivery_id | integer | NOT NULL
+# Indexes:
+#  message_bodies_pkey              | PRIMARY KEY btree (id)
+#  message_bodies_delivery_id_index | btree (delivery_id)
+# Foreign key constraints:
+#  message_bodies_delivery_id_fkey | (delivery_id) REFERENCES message_deliveries(id) ON DELETE CASCADE
+# ----------------------------------------------------------------------------------------------------