webhukhs 0.5.0

data/lib/webhukhs/controllers/receive_webhooks_controller.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Webhukhs
+  class ReceiveWebhooksController < ActionController::API
+    class HandlerInactive < StandardError
+    end
+
+    class UnknownHandler < StandardError
+    end
+
+    def create
+      Rails.error.set_context(**Webhukhs.configuration.error_context)
+      handler = lookup_handler(service_id)
+      raise HandlerInactive unless handler.active?
+      handler.handle(request)
+      render(json: {ok: true, error: nil})
+    rescue UnknownHandler => e
+      Rails.error.report(e, handled: true, severity: :error)
+      render_error_with_status("No handler found for #{service_id.inspect}", status: :not_found)
+    rescue HandlerInactive => e
+      Rails.error.report(e, handled: true, severity: :error)
+      render_error_with_status("Webhook handler #{service_id.inspect} is inactive", status: :service_unavailable)
+    rescue => e
+      raise e unless handler
+      raise e if handler.expose_errors_to_sender?
+      Rails.error.report(e, handled: true, severity: :error)
+      render_error_with_status("Internal error (#{e})")
+    end
+
+    def service_id
+      params.require(:service_id)
+    end
+
+    def render_error_with_status(message_str, status: :ok)
+      json = {ok: false, error: message_str}.to_json
+      render(json: json, status: status)
+    end
+
+    def lookup_handler(service_id_str)
+      active_handlers = Webhukhs.configuration.active_handlers.with_indifferent_access
+      # The config can specify a mapping of:
+      # {"service-1" => MyHandler }
+      # or
+      # {"service-2" => "MyOtherHandler"}
+      # We need to support both, because `MyHandler` is not loaded yet when Rails initializers run.
+      # Zeitwerk takes over after the initializers. So we can't really use a module in the init cycle just yet.
+      # We can, however, use the module name - and resolve it lazily, later.
+      handler_class_or_class_name = active_handlers.fetch(service_id_str)
+      handler_class = handler_class_or_class_name.respond_to?(:constantize) ? handler_class_or_class_name.constantize : handler_class_or_class_name
+      handler_class.new
+    rescue KeyError
+      raise UnknownHandler
+    end
+  end
+end

data/lib/webhukhs/engine.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require_relative "controllers/receive_webhooks_controller"
+require_relative "jobs/processing_job"
+require_relative "models/received_webhook"
+require_relative "base_handler"
+
+module Webhukhs
+  class Engine < ::Rails::Engine
+    isolate_namespace Webhukhs
+
+    autoload :ReceiveWebhooksController, "webhukhs/controllers/receive_webhooks_controller"
+    autoload :ProcessingJob, "webhukhs/jobs/processing_job"
+    autoload :BaseHandler, "webhukhs/base_handler"
+
+    generators do
+      require_relative "install_generator"
+    end
+
+    routes do
+      post "/:service_id" => "received_webhooks#create"
+    end
+  end
+end

data/lib/webhukhs/install_generator.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module Webhukhs
+  #
+  # Rails generator used for setting up Webhukhs in a Rails application.
+  # Run it with +bin/rails g webhukhs:install+ in your console.
+  #
+  class InstallGenerator < Rails::Generators::Base
+    include ActiveRecord::Generators::Migration
+
+    source_root File.expand_path("../templates", __FILE__)
+
+    def create_migration_file
+      migration_template "create_webhukhs_tables.rb.erb", File.join(db_migrate_path, "create_webhukhs_tables.rb")
+      migration_template "add_headers_to_webhukhs_webhooks.rb.erb", File.join(db_migrate_path, "add_headers_to_webhukhs_webhooks.rb")
+    end
+
+    def copy_files
+      template "webhukhs.rb", File.join("config", "initializers", "webhukhs.rb")
+    end
+
+    private
+
+    def migration_version
+      "[#{ActiveRecord::VERSION::STRING.to_f}]"
+    end
+  end
+end

data/lib/webhukhs/jobs/processing_job.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "active_job/railtie"
+
+module Webhukhs
+  class ProcessingJob < ActiveJob::Base
+    def perform(webhook)
+      Rails.error.set_context(webhukhs_handler_module_name: webhook.handler_module_name, **Webhukhs.configuration.error_context)
+
+      webhook_details_for_logs = "Webhukhs::ReceivedWebhook#%s (handler: %s)" % [webhook.id, webhook.handler]
+      webhook.with_lock do
+        unless webhook.received?
+          logger.info { "#{webhook_details_for_logs} is being processed in a different job or has been processed already, skipping." }
+          return
+        end
+        webhook.processing!
+      end
+
+      if webhook.handler.valid?(webhook.request)
+        logger.info { "#{webhook_details_for_logs} starting to process" }
+        webhook.handler.process(webhook)
+        webhook.processed! if webhook.processing?
+        logger.info { "#{webhook_details_for_logs} processed" }
+      else
+        logger.info { "#{webhook_details_for_logs} did not pass validation by the handler. Marking it `failed_validation`." }
+        webhook.failed_validation!
+      end
+    rescue => e
+      webhook.error!
+      raise e
+    end
+  end
+end

data/lib/webhukhs/models/received_webhook.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require "state_machine_enum"
+
+module Webhukhs
+  class ReceivedWebhook < ActiveRecord::Base
+    self.implicit_order_column = "created_at"
+    self.table_name = "received_webhooks"
+
+    include StateMachineEnum
+
+    state_machine_enum :status do |s|
+      s.permit_transition(:received, :processing)
+      s.permit_transition(:processing, :failed_validation)
+      s.permit_transition(:processing, :skipped)
+      s.permit_transition(:processing, :processed)
+      s.permit_transition(:processing, :error)
+      s.permit_transition(:error, :received)
+
+      s.after_committed_transition_to(:received) do |webhook|
+        webhook.handler.enqueue(webhook)
+      end
+    end
+
+    # Store the pertinent data from an ActionDispatch::Request into the webhook.
+    # @param [ActionDispatch::Request]
+    def request=(action_dispatch_request)
+      # Filter out all Rack-specific headers such as "rack.input" and the like. We are
+      # only interested in the actual HTTP headers presented by the webserver. Mostly...
+      headers = action_dispatch_request.env.filter_map do |(header_name, header_value)|
+        if header_name.is_a?(String) && header_name.upcase == header_name && header_value.is_a?(String)
+          [header_name, header_value]
+        end
+      end.to_h
+
+      # ...except the path parameters - they do not get parsed from the headers, but instead get set by Journey - the Rails
+      # router - when the ActionDispatch::Request object gets instantiated. They need to be preserved separately in case the Webhukhs
+      # controller gets mounted under a parametrized path - and the path component actually is a parameter that the webhook
+      # handler either needs for validation or for processing
+      headers["action_dispatch.request.path_parameters"] = action_dispatch_request.env.fetch("action_dispatch.request.path_parameters")
+
+      # ...and the raw request body - because we already save it separately
+      headers.delete("RAW_POST_DATA")
+
+      # Verify the request body is not too large
+      request_body_io = action_dispatch_request.env.fetch("rack.input")
+      if request_body_io.size > Webhukhs.configuration.request_body_size_limit
+        raise "Cannot accept the webhook as the request body is larger than #{limit} bytes"
+      end
+
+      write_attribute("body", request_body_io.read.force_encoding(Encoding::BINARY))
+      write_attribute("request_headers", headers)
+    ensure
+      request_body_io.rewind
+    end
+
+    # A Webhukhs handler is, in a way, a tiny Rails controller which runs in a background job. To allow this,
+    # we need to provide access not only to the webhook payload (the HTTP request body, usually), but also
+    # to the rest of the HTTP request - such as headers and route params. For example, imagine you use
+    # a system where your multiple tenants (users) may receive webhooks from the same sender. However, you
+    # need to dispatch those webhooks to those particular tenants of your application. Instead of mounting
+    # Webhukhs as an engine under a common route, you mount it like so:
+    #
+    #    post "/incoming-webhooks/:user_id/:service_id" => "webhukhs/receive_webhooks#create"
+    #
+    # This way, the tenant ID (the `user_id`) parameter is not going to be provided to you inside the webhook
+    # payload, as the sender is not sending it to you at all. However, you do have that parameter in your
+    # route. When processing the webhook, it is important for you to know which tenant has received the
+    # webhook - so that you can manipulate their data, and not the data belonging to another tenant. With
+    # validation, it is important too - in such a multitenant setup every user is likely to have their own,
+    # specific signing secret that they have set up. To find that secret and compare the signature, you
+    # need access to that `user_id` parameter.
+    #
+    # To allow access to these, Webhukhs allows the ActionDispatch::Request object to be persisted. The
+    # persistence is not 1:1 - the Request is a fairly complex object, with lots of things injected into it
+    # by the Rails stack. Not all of those injected properties (Rack headers) are marshalable, some of them
+    # depend on the Rails application configuration, etc. However, we do retain the most important things
+    # for webhooks to be correctly handled.
+    #
+    # * The HTTP request body
+    # * The headers set by the webserver and the downstream proxies
+    # * The request body and query string params, depending on the MIME type
+    # * The route params. These are set by Journey (the Rails router) and cannot be reconstructed from a "bare" request
+    #
+    # While this reconstruction is best-effort it might not be lossless. For example, there might be no access
+    # to Rack hijack, streaming APIs, the cookie jar or other more high-level Rails request access features.
+    # You will, however, have the basics in place - such as the params, the request body, the path params
+    # (as were decoded by your routes) etc. But it should be sufficient to do the basic tasks to process a webhook.
+    #
+    # @return [ActionDispatch::Request]
+    def request
+      ActionDispatch::Request.new(request_headers.merge!("rack.input" => StringIO.new(body.to_s.b)))
+    end
+
+    def handler
+      handler_module_name.constantize.new
+    end
+  end
+end

data/lib/webhukhs/templates/add_headers_to_webhukhs_webhooks.rb.erb

@@ -0,0 +1,5 @@
+class AddHeadersToWebhukhsWebhooks < ActiveRecord::Migration<%= migration_version %>
+  def change
+    add_column :received_webhooks, :request_headers, :json, null: true
+  end
+end

data/lib/webhukhs/templates/create_webhukhs_tables.rb.erb

@@ -0,0 +1,23 @@
+class CreateWebhukhsTables < ActiveRecord::Migration<%= migration_version %>
+<% id_type = Rails.application.config.generators.options[:active_record][:primary_key_type] rescue nil %>
+  def change
+    create_table :received_webhooks <%= ", id: :#{id_type}" if id_type %> do |t|
+      t.string :handler_event_id, null: false
+      t.string :handler_module_name, null: false
+      t.string :status, default: "received", null: false
+
+      # We don't assume that we can always parse the received body as JSON. Body could be invalid or partly missing,
+      # we can argue how we can handle that for different integrations, but we still should be able to save this data
+      # if it's required. Hence, we don't use :jsonb, but :binary type column here.
+      t.binary :body, null: false
+
+      t.timestamps
+    end
+
+    # For deduplication at ingress. UNIQUE indices in Postgres are case-sensitive
+    # which is what we want, as these are externally-generated IDs
+    add_index :received_webhooks, [:handler_module_name, :handler_event_id], unique: true, name: "webhook_dedup_idx"
+    # For backfill processing (to know how many skipped etc. payloads we have)
+    add_index :received_webhooks, [:status]
+  end
+end

data/lib/webhukhs/templates/webhukhs.rb

@@ -0,0 +1,40 @@
+Webhukhs.configure do |config|
+  # Active Handlers are defined as hash with key as a service_id and handler class  that would handle webhook request.
+  # A Handler must respond to `.new` and return an object roughly matching `Webhukhs::BaseHandler` in terms of interface.
+  # Use module names (strings) here to allow the handler modules to be lazy-loaded by Rails.
+  #
+  # Example:
+  #   {:test => "TestHandler", :inactive => "InactiveHandler"}
+  config.active_handlers = {}
+
+  # It's possible to overwrite default processing job to enahance it. As example if you want to add custom
+  # locking or retry mechanism. You want to inherit that job from Webhukhs::ProcessingJob because the background
+  # job also manages the webhook state.
+  #
+  # Example:
+  #
+  # class WebhookProcessingJob < Webhukhs::ProcessingJob
+  #   def perform(webhook)
+  #     TokenLock.with(name: "webhook-processing-#{webhook.id}") do
+  #       super(webhook)
+  #     end
+  #   end
+  #
+  # In the config a string with your job' class name can be used so that the job can be lazy-loaded by Rails:
+  #
+  # config.processing_job_class = "WebhookProcessingJob"
+
+  # We're using a common interface for error reporting provided by Rails, e.g Rails.error.report. In some cases
+  # you want to enhance those errors with additional context. As example to provide a namespace:
+  #
+  # { appsignal: { namespace: "webhooks" } }
+  #
+  # config.error_context = { appsignal: { namespace: "webhooks" } }
+
+  # Incoming webhooks will be written into your DB without any prior validation. By default, Webhukhs limits the
+  # request body size for webhooks to 512 KiB, so that it would not be too easy for an attacker to fill your
+  # database with junk. However, if you are receiving very large webhook payloads you might need to increase
+  # that limit (or make it even smaller for extra security)
+  #
+  # config.request_body_size_limit = 2.megabytes
+end

data/lib/webhukhs/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Webhukhs
+  VERSION = "0.5.0"
+end

data/lib/webhukhs.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require_relative "webhukhs/version"
+require_relative "webhukhs/engine"
+require_relative "webhukhs/jobs/processing_job"
+require "active_support/core_ext/class/attribute"
+
+module Webhukhs
+  def self.configuration
+    @configuration ||= Configuration.new
+  end
+
+  def self.configure
+    yield configuration
+  end
+end
+
+class Webhukhs::Configuration
+  class_attribute :processing_job_class, default: Webhukhs::ProcessingJob
+  class_attribute :active_handlers, default: {}
+  class_attribute :error_context, default: {}
+  class_attribute :request_body_size_limit, default: 512.kilobytes
+end

data/test/test-webhook-handlers/extract_id_handler.rb

@@ -0,0 +1,5 @@
+class ExtractIdHandler < WebhookTestHandler
+  def extract_event_id_from_request(action_dispatch_request)
+    JSON.parse(action_dispatch_request.body.read).fetch("event_id")
+  end
+end

data/test/test-webhook-handlers/failing_with_concealed_errors.rb

@@ -0,0 +1,7 @@
+class FailingWithConcealedErrors < Webhukhs::BaseHandler
+  def handle(_request)
+    raise "oops"
+  end
+
+  def expose_errors_to_sender? = false
+end

data/test/test-webhook-handlers/failing_with_exposed_errors.rb

@@ -0,0 +1,7 @@
+class FailingWithExposedErrors < Webhukhs::BaseHandler
+  def handle(_request)
+    raise "oops"
+  end
+
+  def expose_errors_to_sender? = true
+end

data/test/test-webhook-handlers/inactive_handler.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class InactiveHandler < WebhookTestHandler
+  def active? = false
+end

data/test/test-webhook-handlers/invalid_handler.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class InvalidHandler < WebhookTestHandler
+  def valid?(request) = false
+end

data/test/test-webhook-handlers/private_handler.rb

@@ -0,0 +1,3 @@
+class PrivateHandler < WebhookTestHandler
+  def expose_errors_to_sender? = false
+end

data/test/test-webhook-handlers/webhook_test_handler.rb

@@ -0,0 +1,13 @@
+class WebhookTestHandler < Webhukhs::BaseHandler
+  def valid?(request)
+    request.params.fetch(:isValid, false)
+  end
+
+  def process(webhook)
+    raise "Oops, failed" if webhook.request.params[:raiseDuringProcessing]
+    filename = webhook.request.params.fetch(:outputToFilename)
+    File.binwrite(filename, webhook.body)
+  end
+
+  def expose_errors_to_sender? = true
+end

data/test/test_app.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "active_record"
+require "action_pack"
+require "action_controller"
+require "rails"
+
+database = "development.sqlite3"
+ENV["DATABASE_URL"] = "sqlite3:#{database}"
+ActiveRecord::Base.establish_connection(adapter: "sqlite3", database: database)
+ActiveRecord::Base.logger = Logger.new(nil)
+ActiveRecord::Schema.define do
+  create_table "received_webhooks", force: :cascade do |t|
+    t.string "handler_event_id", null: false
+    t.string "handler_module_name", null: false
+    t.string "status", default: "received", null: false
+    t.binary "body", null: false
+    t.json "request_headers", null: true
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["handler_module_name", "handler_event_id"], name: "webhook_dedup_idx", unique: true
+    t.index ["status"], name: "index_received_webhooks_on_status"
+  end
+end
+
+require_relative "../lib/webhukhs"
+require_relative "test-webhook-handlers/webhook_test_handler"
+require_relative "test-webhook-handlers/inactive_handler"
+require_relative "test-webhook-handlers/invalid_handler"
+require_relative "test-webhook-handlers/private_handler"
+require_relative "test-webhook-handlers/failing_with_exposed_errors"
+require_relative "test-webhook-handlers/failing_with_concealed_errors"
+require_relative "test-webhook-handlers/extract_id_handler"
+
+Webhukhs.configure do |config|
+  config.active_handlers = {
+    test: "WebhookTestHandler",
+    inactive: "InactiveHandler",
+    invalid: "InvalidHandler",
+    private: "PrivateHandler",
+    "failing-with-exposed-errors": "FailingWithExposedErrors",
+    "failing-with-concealed-errors": "FailingWithConcealedErrors",
+    extract_id: "ExtractIdHandler"
+  }
+end
+
+class WebhukhsTestApp < Rails::Application
+  config.logger = Logger.new(nil)
+  config.autoload_paths << File.dirname(__FILE__) + "/test-webhook-handlers"
+  config.root = __dir__
+  config.eager_load = false
+  config.consider_all_requests_local = true
+  config.secret_key_base = "i_am_a_secret"
+  config.active_support.cache_format_version = 7.1
+  config.active_job.queue_adapter = :test
+  config.hosts << ->(host) { true } # Permit all hosts
+
+  routes.append do
+    mount Webhukhs::Engine, at: "/webhukhs"
+    post "/per-user-webhukhs/:user_id/:service_id" => "webhukhs/receive_webhooks#create"
+  end
+end
+
+WebhukhsTestApp.initialize!
+
+# run WebhukhsTestApp

data/test/test_helper.rb

@@ -0,0 +1,50 @@
+require_relative "test_app"
+require "rails/test_help"
+
+class ActiveSupport::TestCase
+  # Same as "assert_changes" in Rails but for countable entities.
+  # @return [*] return value of the block
+  # @example
+  #   assert_changes_by("Notification.count", exactly: 2) do
+  #     cause_two_notifications_to_get_delivered
+  #   end
+  def assert_changes_by(expression, message = nil, exactly: nil, at_least: nil, at_most: nil, &block)
+    # rubocop:disable Security/Eval
+    exp = expression.respond_to?(:call) ? expression : -> { eval(expression.to_s, block.binding) }
+    # rubocop:enable Security/Eval
+
+    raise "either exactly:, at_least: or at_most: must be specified" unless exactly || at_least || at_most
+    raise "exactly: is mutually exclusive with other options" if exactly && (at_least || at_most)
+    raise "at_most: must be larger than at_least:" if at_least && at_most && at_most < at_least
+
+    before = exp.call
+    retval = assert_nothing_raised(&block)
+
+    after = exp.call
+    delta = after - before
+
+    if exactly
+      at_most = exactly
+      at_least = exactly
+    end
+
+    # We do not make these an if/else since we allow both at_most and at_least
+    if at_most
+      error = "#{expression.inspect} changed by #{delta} which is more than #{at_most}"
+      error = "#{error}. It was #{before} and became #{after}"
+      error = "#{message.call}.\n" if message&.respond_to?(:call)
+      error = "#{message}.\n#{error}" if message && !message.respond_to?(:call)
+      assert delta <= at_most, error
+    end
+
+    if at_least
+      error = "#{expression.inspect} changed by #{delta} which is less than #{at_least}"
+      error = "#{error}. It was #{before} and became #{after}"
+      error = "#{message.call}.\n" if message&.respond_to?(:call)
+      error = "#{message}.\n#{error}" if message && !message.respond_to?(:call)
+      assert delta >= at_least, error
+    end
+
+    retval
+  end
+end