webhookdb 0.1.0

data/lib/webhookdb/webhook_subscription.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+require "webhookdb/jobs/webhook_subscription_delivery_attempt"
+
+# Webhook subscriptions have a few parts:
+#
+# - The WebhookSubscription itself (this model),
+#   which represents a user's desire to receive all webhooks at a URL.
+# - The individual Delivery, which is a single 'rowupsert' event being
+#   delivered to a subscription.
+#   That is, if multiple rowupserts are done, there will be multiple Deliveries
+#   to a single Subscription, one for each rowupsert.
+#   Likewise, if a single rowupsert is done, but there are multiple Subscriptions,
+#   there will be multiple Deliveries, one to each Subscription.
+# - Async job that listens for rowupsert events and enqueues new deliveries.
+# - When a delivery is 'enqueued', it is created in the database,
+#   and then a sidekiq job is put into Redis.
+#   This sidekiq job operates OUTSIDE of our normal job system
+#   since we do not want to bother with audit logging or routing
+#   (enough history is in the DB already, though we could add it if needed).
+# - We attempt the delivery until it succeeds, or we run out of attempts.
+#   See #attempt_delivery.
+#
+class Webhookdb::WebhookSubscription < Webhookdb::Postgres::Model(:webhook_subscriptions)
+  plugin :timestamps
+  plugin :column_encryption do |enc|
+    enc.column :webhook_secret
+  end
+
+  many_to_one :service_integration, class: Webhookdb::ServiceIntegration
+  many_to_one :organization, class: Webhookdb::Organization
+  many_to_one :created_by, class: Webhookdb::Customer
+
+  # Amount of time we wait for a response from the server.
+  TIMEOUT = 10.seconds
+  # An individual will be delivered this many times before giving up.
+  MAX_DELIVERY_ATTEMPTS = 25
+
+  dataset_module do
+    def active
+      return self.where(deactivated_at: nil)
+    end
+
+    def to_notify
+      return self.active
+    end
+  end
+
+  def active?
+    return !self.deactivated?
+  end
+
+  def deactivated?
+    return !!self.deactivated_at
+  end
+
+  def deactivate(at: Time.now)
+    self.deactivated_at = at
+    return self
+  end
+
+  def fetch_organization
+    return self.organization || self.service_integration.organization
+  end
+
+  def status
+    return self.deactivated? ? "deactivated" : "active"
+  end
+
+  # Deliver the webhook payload to the configured URL.
+  # This does NOT create or deal with WebhookSubscription::Delivery;
+  # it is for the actual delivering.
+  def deliver(service_name:, table_name:, row:, external_id:, external_id_column:, headers: {})
+    body = {
+      service_name:,
+      table_name:,
+      row:,
+      external_id:,
+      external_id_column:,
+    }
+    return Webhookdb::Http.post(
+      self.deliver_to_url,
+      body,
+      headers: {"Whdb-Webhook-Secret" => self.webhook_secret}.merge(headers),
+      timeout: TIMEOUT,
+      logger: self.logger,
+    )
+  end
+
+  def deliver_test_event(external_id: SecureRandom.hex(6))
+    return self.deliver(
+      service_name: "test service",
+      table_name: "test_table_name",
+      external_id:,
+      external_id_column: "external_id",
+      row: {data: ["alpha", "beta", "charlie", "delta"]},
+      headers: {"Whdb-Test-Event" => "1"},
+    )
+  end
+
+  def create_delivery(payload)
+    return Webhookdb::WebhookSubscription::Delivery.create(webhook_subscription: self, payload:)
+  end
+
+  # Create a new Delivery and enqueue it for async processing.
+  def enqueue_delivery(payload)
+    delivery = self.create_delivery(payload)
+    Webhookdb::Jobs::WebhookSubscriptionDeliveryEvent.perform_async(delivery.id)
+    return delivery
+  end
+
+  # Attempt to deliver the payload in +d+ to the configured URL (see #deliver).
+  # Noops if the subscription is deactivated.
+  #
+  # If the attempt succeeds, no attempts are enqueued.
+  #
+  # If the attempt fails, another async job to reattempt delivery
+  # will be enqueued for some time in the future based on the number of attempts.
+  # The timestamp and http status are stored on the delivery for future analysis.
+  #
+  # After too many failures, no more attempts will be enqueued.
+  # Instead, a developer alert is emitted.
+  #
+  # In the future, we will support manually re-attempting delivery (success of which should
+  # clear deactivated subscriptions), and automatic deactivation
+  # (after some criteria of abandonment has been met).
+  def attempt_delivery(d)
+    return if self.deactivated?
+    d.db.transaction do
+      d.lock!
+      attempt = d.attempt_count + 1
+      begin
+        r = self.deliver(**d.payload.symbolize_keys, headers: {"Whdb-Attempt" => attempt.to_s})
+        d.add_attempt(status: r.code)
+      rescue StandardError => e
+        self.logger.error(
+          "webhook_subscription_delivery_failure",
+          error: e,
+          webhook_subscription_id: self.id,
+          webhook_subscription_delivery_id: d.id,
+        )
+        d.add_attempt(status: e.is_a?(Webhookdb::Http::Error) ? e.status : 0)
+        if attempt < MAX_DELIVERY_ATTEMPTS
+          self._retry(d, attempt)
+        else
+          self._fatal(d, e)
+        end
+      ensure
+        d.save_changes
+      end
+    end
+  end
+
+  def _retry(delivery, attempt)
+    delay = self.class.backoff_for_attempt(attempt)
+    Webhookdb::Jobs::WebhookSubscriptionDeliveryEvent.perform_in(delay, delivery.id)
+  end
+
+  def self.backoff_for_attempt(attempt)
+    return 1 if attempt <= 1
+    return attempt * 2 if attempt <= 10
+    return attempt * 3 if attempt <= 20
+    return attempt * 4
+  end
+
+  def _fatal(d, e)
+    Webhookdb::DeveloperAlert.new(
+      subsystem: "Webhook Subscriptions",
+      emoji: ":hook:",
+      fallback: "Error delivering WebhookSubscription::Delivery[id: #{d.id}, subscription_id: #{self.id}]: #{e}",
+      fields: [
+        {title: "Org", value: self.fetch_organization.display_string, short: true},
+        {title: "Creator", value: self.created_by&.email, short: true},
+        {title: "Delivery", value: "#{d.id}, Subscription: #{self.id}, Attempts: #{d.attempt_count}"},
+        {title: "URL", value: self.deliver_to_url, short: false},
+        {title: "Exception", value: e.inspect, short: false},
+      ],
+    ).emit
+  end
+
+  def associated_type
+    return "organization" unless self.organization_id.nil?
+    return "service_integration" unless self.service_integration_id.nil?
+    return ""
+  end
+
+  def associated_id
+    return self.organization.key unless self.organization_id.nil?
+    return self.service_integration.opaque_id unless self.service_integration_id.nil?
+    return ""
+  end
+
+  #
+  # :Sequel Hooks:
+  #
+
+  def before_create
+    self[:opaque_id] ||= Webhookdb::Id.new_opaque_id("wsb")
+  end
+end
+
+# Table: webhook_subscriptions
+# ----------------------------------------------------------------------------------------------------------------------------------------------------------------
+# Columns:
+#  id                     | integer                  | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY
+#  deliver_to_url         | text                     | NOT NULL
+#  webhook_secret         | text                     | NOT NULL
+#  opaque_id              | text                     | NOT NULL
+#  service_integration_id | integer                  |
+#  organization_id        | integer                  |
+#  created_at             | timestamp with time zone | NOT NULL DEFAULT now()
+#  updated_at             | timestamp with time zone |
+#  created_by_id          | integer                  |
+#  deactivated_at         | timestamp with time zone |
+# Indexes:
+#  webhook_subscriptions_pkey          | PRIMARY KEY btree (id)
+#  webhook_subscriptions_opaque_id_key | UNIQUE btree (opaque_id)
+# Check constraints:
+#  service_integration_or_org | (service_integration_id IS NULL AND organization_id IS NOT NULL OR service_integration_id IS NOT NULL AND organization_id IS NULL)
+# Foreign key constraints:
+#  webhook_subscriptions_created_by_id_fkey          | (created_by_id) REFERENCES customers(id)
+#  webhook_subscriptions_organization_id_fkey        | (organization_id) REFERENCES organizations(id)
+#  webhook_subscriptions_service_integration_id_fkey | (service_integration_id) REFERENCES service_integrations(id)
+# Referenced By:
+#  webhook_subscription_deliveries | webhook_subscription_deliveries_webhook_subscription_id_fkey | (webhook_subscription_id) REFERENCES webhook_subscriptions(id)
+# ----------------------------------------------------------------------------------------------------------------------------------------------------------------

data/lib/webhookdb/windows_tz.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# Map Windows Timezone names to IANA names (and ActiveSupport timezones).
+# Outlook calendar uses these timezones.
+module Webhookdb::WindowsTZ
+  class << self
+    attr_accessor :_win_to_tz
+
+    # @return [Hash<String => ActiveSupport::TimeZone>]
+    def windows_name_to_tz
+      return self._win_to_tz if self._win_to_tz
+      win_to_tz = {}
+      self._win_to_tz = win_to_tz
+
+      all_win_names = Set.new
+      File.open(Webhookdb::DATA_DIR + "windows_tz.txt").each do |line|
+        line.strip!
+        next if line.blank? || line.start_with?("#")
+        iana, win = line.split(/\s/, 2)
+        next if win_to_tz.include?(win)
+        all_win_names.add(win)
+        tz = ActiveSupport::TimeZone[iana]
+        next if tz.nil?
+        win_to_tz[win] = tz
+      end
+      win_to_tz.each_key { |k| all_win_names.delete(k) }
+      raise Webhookdb::InvariantViolation, "unmapped windows timezones: #{all_win_names.join(', ')}" unless
+        all_win_names.empty?
+      return win_to_tz
+    end
+  end
+end

data/lib/webhookdb/xml.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+class Webhookdb::Xml
+  module Atom
+    def self.parse(thing)
+      Parser.new(thing).to_hash
+    end
+
+    def self.parse_entry(thing)
+      p = Parser.new(thing)
+      p.parse_entry(p.doc.root)
+    end
+
+    class Parser
+      attr_reader :doc
+
+      def initialize(thing)
+        @doc = Nokogiri::XML.parse(thing, &:noblanks)
+      end
+
+      def to_hash
+        entries = []
+        feed = {"entries" => entries}
+        @doc.root.children.each do |c|
+          if c.is_a?(Nokogiri::XML::Text)
+            next
+          elsif c.name == "entry"
+            entries << self.parse_entry(c)
+          elsif self.spec_attr?(c)
+            feed[self.fqn(c)] = self.parse_spec_attr(c)
+          elsif self.simple_text?(c)
+            feed[self.fqn(c)] = self.text(c)
+          else
+            feed[self.fqn(c)] = self.parse_to_hash(c)
+          end
+        end
+        return feed
+      end
+
+      protected def simple_text?(c) = c.children.size == 1 && c.children[0].is_a?(Nokogiri::XML::Text)
+      protected def spec_attr?(c) = ["link", "category"].include?(c.name)
+
+      protected def parse_spec_attr(c)
+        h = {}
+        c.attributes.each do |k, v|
+          h[k] = v.value
+        end
+        h["text"] = self.text(c) if simple_text?(c)
+        return h
+      end
+
+      def parse_entry(e)
+        h = {}
+        e.children.each do |c|
+          if c.name == "content"
+            content = {}
+            h["content"] = content
+            content["value"] = c.children.to_s if c.children.to_s.present?
+            c.attributes.each do |k, v|
+              content[k] = v.value
+            end
+          elsif self.spec_attr?(c)
+            h[self.fqn(c)] = self.parse_spec_attr(c)
+          else
+            h[self.fqn(c)] = c.text
+          end
+        end
+        return h
+      end
+
+      protected def fqn(c)
+        return c.name unless c.namespace&.prefix
+        return "#{c.namespace.prefix}:#{c.name}"
+      end
+
+      protected def parse_to_hash(c)
+        h = {}
+        c.children.each do |cc|
+          h[self.fqn(cc)] = self.text(cc)
+        end
+        return h
+      end
+
+      protected def text(c)
+        t = c.children.first
+        return "" if t.nil?
+        raise ArgumentError, "child is not text: #{c}" unless t.is_a?(Nokogiri::XML::Text)
+        return t.text
+      end
+    end
+  end
+end

data/lib/webhookdb.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+require "active_support"
+require "active_support/core_ext"
+require "appydays/configurable"
+require "appydays/loggable"
+require "money"
+require "pathname"
+require "phony"
+
+require "webhookdb/json"
+
+if (heroku_app = ENV.fetch("MERGE_HEROKU_ENV", nil))
+  text = `heroku config -j --app=#{heroku_app}`
+  json = Oj.load(text)
+  json.each do |k, v|
+    ENV[k] = v
+  end
+end
+
+Money.locale_backend = :i18n
+Money.default_currency = "USD"
+Money.rounding_mode = BigDecimal::ROUND_HALF_UP
+
+module Webhookdb
+  include Appydays::Loggable
+  include Appydays::Configurable
+
+  # Error raised when we cannot take an action
+  # because some condition has not been set up right.
+  class InvalidPrecondition < StandardError; end
+
+  # Error raised when, after we take an action,
+  # something we expect to have changed has not changed.
+  class InvalidPostcondition < StandardError; end
+
+  # Some invariant has been violated, which we never expect to see.
+  class InvariantViolation < StandardError; end
+
+  # Error raised when a customer gives us some invalid input.
+  # Allows the library to raise the error with the message,
+  # and is caught automatically by the service as a 400.
+  class InvalidInput < StandardError; end
+
+  # Raised when an organization's database cannot be modified.
+  class DatabaseLocked < StandardError; end
+
+  # Used in various places that need to short-circuit code in regression mode.
+  class RegressionModeSkip < StandardError; end
+
+  APPLICATION_NAME = "Webhookdb"
+  RACK_ENV = ENV.fetch("RACK_ENV", "development")
+  COMMIT = ENV.fetch("HEROKU_SLUG_COMMIT", "unknown-commit")
+  RELEASE = ENV.fetch("HEROKU_RELEASE_VERSION", "unknown-release")
+  RELEASE_CREATED_AT = ENV.fetch("HEROKU_RELEASE_CREATED_AT") { Time.at(0).utc.iso8601 }
+  INTEGRATION_TESTS_ENABLED = ENV.fetch("INTEGRATION_TESTS", false)
+
+  DATA_DIR = Pathname(__FILE__).dirname.parent + "data"
+
+  configurable(:webhookdb) do
+    setting :log_level_override,
+            nil,
+            key: "LOG_LEVEL",
+            side_effect: ->(v) { Appydays::Loggable.default_level = v if v }
+    setting :log_format, nil
+    setting :app_url, "http://localhost:18002"
+    setting :api_url, "http://localhost:#{ENV.fetch('PORT', 18_001)}"
+    setting :bust_idempotency, false
+    setting :http_user_agent, ""
+    setting :oss_repo_url, "https://github.com/webhookdb/webhookdb"
+    setting :support_email, "hello@webhookdb.com"
+    setting :use_globals_cache, false
+    setting :regression_mode, false
+  end
+
+  # Regression mode is true when we re replaying webhooks locally,
+  # or for some other reason, want to disable certain checks we use in production.
+  # For example, we may want to ignore certain errors (like if integrations are missing dependency rows),
+  # or disable certain validations (like always assume the webhook is valid).
+  def self.regression_mode?
+    return self.regression_mode
+  end
+
+  require "webhookdb/method_utilities"
+  extend Webhookdb::MethodUtilities
+
+  require "webhookdb/sentry"
+
+  def self.load_app
+    $stdout.sync = true
+    $stderr.sync = true
+
+    Appydays::Loggable.configure_12factor(format: self.log_format, application: APPLICATION_NAME)
+
+    require "webhookdb/postgres"
+    Webhookdb::Postgres.load_models
+  end
+
+  #
+  # :section: Globals cache
+  #
+
+  singleton_attr_reader :globals_cache
+  @globals_cache = {}
+
+  # If globals caching is enabled, see if there is a cached value under +key+
+  # and return it if so. If there is not, evaluate the given block and store that value.
+  # Generally used for looking up well-known database objects like certain roles.
+  def self.cached_get(key)
+    if self.use_globals_cache
+      result = self.globals_cache[key]
+      return result if result
+    end
+    result = yield()
+    self.globals_cache[key] = result
+    return result
+  end
+
+  #
+  # :section: Errors
+  #
+
+  class LockFailed < StandardError; end
+
+  ### Generate a key for the specified Sequel model +instance+ and
+  ### any additional +parts+ that can be used for idempotent requests.
+  def self.idempotency_key(instance, *parts)
+    key = "%s-%s" % [instance.class.implicit_table_name, instance.pk]
+
+    if instance.respond_to?(:updated_at) && instance.updated_at
+      parts << instance.updated_at
+    elsif instance.respond_to?(:created_at) && instance.created_at
+      parts << instance.created_at
+    end
+    parts << SecureRandom.hex(8) if self.bust_idempotency
+    key << "-" << parts.map(&:to_s).join("-") unless parts.empty?
+
+    return key
+  end
+
+  #
+  # :section: Unambiguous/promo code chars
+  #
+
+  # Remove ambiguous characters (L, I, 1 or 0, O) and vowels from possible codes
+  # to avoid creating ambiguous codes or real words.
+  UNAMBIGUOUS_CHARS = "CDFGHJKMNPQRTVWXYZ23469".chars.freeze
+
+  def self.take_unambiguous_chars(n)
+    return Array.new(n) { UNAMBIGUOUS_CHARS.sample }.join
+  end
+
+  # Convert a string into something we consistently use for slugs:
+  # a-z, 0-9, and underscores only. Leading numbers are converted to words.
+  #
+  # Acme + Corporation -> "acme_corporation"
+  # 1Byte -> "one_byte"
+  # 10Byte -> "one0_byte"
+  def self.to_slug(s)
+    raise ArgumentError, "s cannot be nil" if s.nil?
+    return "" if s.blank?
+    slug = s.downcase.strip.gsub(/[^a-z0-9]/, "_").squeeze("_")
+    slug = NUMBERS_TO_WORDS[slug.first] + slug[1..] if slug.first.match?(/[0-9]/)
+    return slug
+  end
+
+  NUMBERS_TO_WORDS = {
+    "0" => "zero",
+    "1" => "one",
+    "2" => "two",
+    "3" => "three",
+    "4" => "four",
+    "5" => "five",
+    "6" => "six",
+    "7" => "seven",
+    "8" => "eight",
+    "9" => "nine",
+  }.freeze
+
+  # Return the request user and admin stored in TLS. See service.rb for implementation.
+  #
+  # Note that the second return value (the admin) will be nil if not authed as an admin,
+  # and if an admin is impersonating, the impersonated customer is the first value.
+  #
+  # Both values will be nil if no user is authed or this is called outside of a request.
+  #
+  # Usually these fields should only be used where it would be sufficiently difficult
+  # to pass the current user through the stack.
+  # In the API, you should instead use the 'current customer' methods
+  # like current_customer, and admin_customer, NOT using TLS.
+  # Outside of the API, this should only be used for things like auditing;
+  # it should NOT, for example, ever be used to determine the 'customer owner' of objects
+  # being created. Nearly all code will be simpler if the current customer
+  # is passed around. But it would be too complex for some code (like auditing)
+  # so this system exists. Overuse of request_user_and_admin will inevitably lead to regret.
+  def self.request_user_and_admin
+    return Thread.current[:request_user], Thread.current[:request_admin]
+  end
+
+  # Return the request user stored in TLS. See service.rb for details.
+  def self.set_request_user_and_admin(user, admin, &block)
+    if !user.nil? && !admin.nil? && self.request_user_and_admin != [nil, nil]
+      raise Webhookdb::InvalidPrecondition, "request user is already set: #{user}, #{admin}"
+    end
+    Thread.current[:request_user] = user
+    Thread.current[:request_admin] = admin
+    return if block.nil?
+    begin
+      yield
+    ensure
+      Thread.current[:request_user] = nil
+      Thread.current[:request_admin] = nil
+    end
+  end
+end
+
+require "webhookdb/aggregate_result"
+require "webhookdb/dbutil"
+require "webhookdb/developer_alert"
+require "webhookdb/http"
+require "webhookdb/phone_number"
+require "webhookdb/replicator"
+require "webhookdb/typed_struct"
+require "webhookdb/webhook_response"

data/lib/webterm/apps.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "appydays/configurable"
+
+class Webhookdb::Webterm
+  include Appydays::Configurable
+
+  configurable(:webterm) do
+    setting :enforce_ssl, true
+  end
+
+  STATIC = File.expand_path("#{File.dirname(__FILE__)}/static")
+
+  Files = Rack::Files.new(STATIC)
+
+  class RedirectIndexHtmlToRoot
+    REDIRECTS = ["", "/index.html"].freeze
+
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      return [302, {"Location" => "/terminal/"}, []] if REDIRECTS.include?(env[Rack::PATH_INFO])
+      return @app.call(env)
+    end
+  end
+
+  class ServeIndexHtmlFromRoot
+    def initialize(app)
+      @app = app
+      @cached_html = nil
+    end
+
+    def call(env)
+      return @app.call(env) unless env[Rack::PATH_INFO] == "/"
+      if @cached_html.nil?
+        html = File.read(File.join(STATIC, "index.html"))
+        html.sub!("/* REPLACE_WHDB_ENV */", "window.whdbEnv = [['WEBHOOKDB_API_HOST', '#{Webhookdb.api_url}']]")
+        @cached_html = html.chars
+      end
+      return [200, {"Content-Type" => "text/html"}, @cached_html]
+    end
+  end
+end