webhookdb 0.1.0

data/lib/webhookdb/fixtures.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "faker"
+require "fluent_fixtures"
+
+require "webhookdb"
+
+module Webhookdb::Fixtures
+  extend FluentFixtures::Collection
+
+  # Set the path to use when finding fixtures for this collection
+  fixture_path_prefix "webhookdb/fixtures"
+
+  ::Faker::Config.locale = :en
+end

data/lib/webhookdb/formatting.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Webhookdb::Formatting
+  def self.blocks
+    return Blocks.new
+  end
+
+  class Blocks
+    def initialize
+      @arr = []
+    end
+
+    def blank
+      return self.line("")
+    end
+
+    def line(value)
+      @arr << Line.new(value)
+      return self
+    end
+
+    def table(headers, rows)
+      @arr << Table.new(headers, rows)
+      return self
+    end
+
+    def as_json(*a)
+      return @arr.as_json(*a)
+    end
+  end
+
+  class Line
+    attr_accessor :value
+
+    def initialize(value)
+      @value = value
+    end
+
+    def as_json(*)
+      return {type: "line", value: self.value}
+    end
+  end
+
+  class Table
+    attr_accessor :headers, :rows
+
+    def initialize(headers, rows)
+      @headers = headers
+      @rows = rows
+    end
+
+    def as_json(*o)
+      return {type: "table", value: {headers: self.headers, rows: self.rows.as_json(*o)}}
+    end
+  end
+end

data/lib/webhookdb/front.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "appydays/configurable"
+require "appydays/loggable"
+
+module Webhookdb::Front
+  include Appydays::Configurable
+
+  configurable(:front) do
+    # The api secret is used for webhook verification, the client id and secret are used for OAuth
+    setting :api_secret, "front_api_secret"
+    setting :client_id, "front_client_id"
+    setting :client_secret, "front_client_secret"
+    setting :http_timeout, 30
+  end
+
+  def self.oauth_callback_url = Webhookdb.api_url + "/v1/install/front/callback"
+
+  def self.verify_signature(request)
+    request.body.rewind
+    body = request.body.read
+    base_string = "#{request.env['HTTP_X_FRONT_REQUEST_TIMESTAMP']}:#{body}"
+    calculated_signature = OpenSSL::HMAC.base64digest(OpenSSL::Digest.new("sha256"), self.api_secret, base_string)
+    return calculated_signature == request.env["HTTP_X_FRONT_SIGNATURE"]
+  end
+
+  def self.webhook_response(request)
+    return Webhookdb::WebhookResponse.error("missing signature") unless request.env["HTTP_X_FRONT_SIGNATURE"]
+
+    from_front = Webhookdb::Front.verify_signature(request)
+    return Webhookdb::WebhookResponse.ok(status: 200) if from_front
+    return Webhookdb::WebhookResponse.error("invalid signature")
+  end
+
+  def self.initial_verification_request_response(request)
+    from_front = self.verify_signature(request)
+    if from_front
+      return Webhookdb::WebhookResponse.ok(
+        json: {challenge: request.env["HTTP_X_FRONT_CHALLENGE"]},
+        status: 200,
+      )
+    end
+    return Webhookdb::WebhookResponse.error("invalid credentials")
+  end
+
+  def self.auth_headers(token)
+    return {"Authorization" => "Bearer #{token}"}
+  end
+end

data/lib/webhookdb/github.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "active_support/security_utils"
+
+class Webhookdb::Github
+  include Appydays::Configurable
+
+  configurable(:github) do
+    setting :http_timeout, 30
+    setting :activity_cron_expression, "*/5 * * * *"
+  end
+
+  def self.parse_link_header(header)
+    return Webhookdb::Shopify.parse_link_header(header)
+  end
+
+  # see https://docs.github.com/en/webhooks/using-webhooks/validating-webhook-deliveries
+  def self.verify_webhook(body, hmac_header, webhook_secret)
+    calculated_hash = "sha256=" + OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new("sha256"), webhook_secret, body)
+    return ActiveSupport::SecurityUtils.secure_compare(calculated_hash, hmac_header)
+  end
+end

data/lib/webhookdb/google_calendar.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "appydays/configurable"
+
+module Webhookdb::GoogleCalendar
+  include Appydays::Configurable
+
+  configurable(:google_calendar) do
+    # How many calendars/events should we fetch in a single page?
+    # Higher uses slightly more memory but fewer API calls.
+    # Max of 2500.
+    setting :list_page_size, 2000
+    # How many rows should we upsert at a time?
+    # Higher is fewer upserts, but can create very large SQL strings,
+    # which can have negative performance.
+    setting :upsert_page_size, 500
+    # How long should watch channels live.
+    # Generally use Google's default (one week),
+    # but set shorter when testing.
+    setting :watch_ttl, 604_800
+    setting :http_timeout, 30
+  end
+
+  # Manual backfilling is not supported on Google Calendar integrations.
+  # If a manual backfill is attempted, direct customer to this url.
+  DOCUMENTATION_URL = "https://docs.webhookdb.com/guides/google-calendar/"
+
+  PUSH_NOT_SUPPORTED_SENTINEL_WATCH_ID = "ech-push-not-supported-for-requested-resource"
+end

data/lib/webhookdb/heroku.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "platform-api"
+
+require "webhookdb"
+
+class Webhookdb::Heroku
+  include Appydays::Configurable
+
+  configurable(:heroku) do
+    setting :oauth_id, "", key: "WEBHOOKDB_HEROKU_OAUTH_ID"
+    setting :oauth_token, "", key: "WEBHOOKDB_HEROKU_OAUTH_TOKEN"
+    setting :app_name, "", key: "HEROKU_APP_NAME"
+  end
+
+  def self.client
+    raise "No heroku:oauth_token configured" if self.oauth_token.blank?
+    @client ||= PlatformAPI.connect_oauth(self.oauth_token)
+    return @client
+  end
+end

data/lib/webhookdb/http.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require "httparty"
+
+require "appydays/loggable/httparty_formatter"
+
+module Webhookdb::Http
+  # Error raised when some API has rate limited us.
+  class BaseError < StandardError; end
+
+  class Error < BaseError
+    attr_reader :response, :body, :uri, :status, :http_method
+
+    def initialize(response, msg=nil)
+      @response = response
+      @body = response.body
+      @headers = response.headers.to_h
+      @status = response.code
+      @uri = response.request.last_uri.dup
+      if @uri.query.present?
+        cleaned_params = CGI.parse(@uri.query).map { |k, v| k.include?("secret") ? [k, ".snip."] : [k, v] }
+        @uri.query = HTTParty::Request::NON_RAILS_QUERY_STRING_NORMALIZER.call(cleaned_params)
+      end
+      @http_method = response.request.http_method::METHOD
+      super(msg || self.to_s)
+    end
+
+    def to_s
+      return "HttpError(status: #{self.status}, method: #{self.http_method}, uri: #{self.uri}, body: #{self.body})"
+    end
+
+    alias inspect to_s
+  end
+
+  def self.user_agent
+    return Webhookdb.http_user_agent unless Webhookdb.http_user_agent.blank?
+    return "WebhookDB/#{Webhookdb::RELEASE} https://webhookdb.com #{Webhookdb::RELEASE_CREATED_AT}"
+  end
+
+  def self.extract_url_auth(url)
+    parsed_uri = URI(url)
+    if parsed_uri.userinfo.present?
+      auth_params = {
+        username: URI.decode_www_form_component(parsed_uri.user || ""),
+        password: URI.decode_www_form_component(parsed_uri.password || ""),
+      }
+      parsed_uri.user = parsed_uri.password = nil
+      cleaned_url = parsed_uri.to_s
+      return cleaned_url, auth_params
+    end
+    return url, nil
+  end
+
+  def self.check!(response, **options)
+    # All oks are ok
+    return if response.code < 300
+    # We expect 300s if we aren't following redirects
+    return if response.code < 400 && !options[:follow_redirects]
+    # Raise for 400s, or 300s if we were meant to follow redirects
+    raise Error, response
+  end
+
+  def self.get(url, query={}, **options, &)
+    self._setup_required_args(options)
+    opts = {query:, headers: {}}.merge(**options)
+    opts[:headers]["User-Agent"] = self.user_agent
+    # See https://github.com/jnunemaker/httparty/issues/784#issuecomment-1585714745
+    # I *think* this should be safe to always use.
+    opts[:headers]["Connection"] ||= "keep-alive"
+    r = HTTParty.get(url, **opts, &)
+    self.check!(r, **opts)
+    return r
+  end
+
+  def self.post(url, body={}, headers: {}, method: nil, check: true, **options, &)
+    self._setup_required_args(options)
+    headers["Content-Type"] ||= "application/json"
+    headers["User-Agent"] = self.user_agent
+    body = body.to_json if !body.is_a?(String) && headers["Content-Type"].include?("json")
+    opts = {body:, headers:}.merge(**options)
+    r = HTTParty.send(method || :post, url, **opts, &)
+    self.check!(r, **options) if check
+    return r
+  end
+
+  def self._setup_required_args(options)
+    raise ArgumentError, "must pass :timeout keyword" unless options.key?(:timeout)
+
+    raise ArgumentError, "must pass :logger keyword" unless options.key?(:logger)
+    options[:log_format] = :appydays
+  end
+
+  # Convenience wrapper around Down that handles gzip.
+  # @return Array<Down::ChunkedIO, IO> Tuple
+  def self.chunked_download(request_url, rewindable: false, **down_kw)
+    io = Down::NetHttp.open(request_url, rewindable:, **down_kw)
+    if io.data[:headers].fetch("Content-Encoding", "").include?("gzip")
+      # If the response is gzipped, Down doesn't handle it properly.
+      # Wrap it with gzip reader, and force the encoding to binary
+      # the server may send back a header like Content-Type: text/plain; UTF-8,
+      # so each line Down yields via #gets will have force_encoding('utf-8').
+      # https://github.com/janko/down/issues/87
+      io.instance_variable_set(:@encoding, "binary")
+      io = Zlib::GzipReader.wrap(io)
+    end
+    return io
+  end
+
+  def self.gzipped?(string)
+    return false if string.length < 3
+    b = string[..2].bytes
+    return b[0] == 0x1f && b[1] == 0x8b
+  end
+end

data/lib/webhookdb/icalendar.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "appydays/configurable"
+
+module Webhookdb::Icalendar
+  # Manual backfilling is not supported on iCalendar integrations.
+  # If a manual backfill is attempted, direct customer to this url.
+  DOCUMENTATION_URL = "https://docs.webhookdb.com/guides/icalendar/"
+
+  include Appydays::Configurable
+
+  configurable(:icalendar) do
+    # Do not store events older then this when syncing recurring events.
+    # Many icalendar feeds are misconfigured and this prevents enumerating 2000+ years of recurrence.
+    setting :oldest_recurring_event, "1990-01-01", convert: ->(s) { Date.parse(s) }
+  end
+end

data/lib/webhookdb/id.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "webhookdb"
+
+module Webhookdb::Id
+  ID_BYTES = 16
+
+  def self.new_opaque_id(prefix)
+    b36 = self.rand_enc(ID_BYTES)
+    return "#{prefix}_#{b36}"
+  end
+
+  def self.rand_enc(blen)
+    b = SecureRandom.bytes(blen)
+    return Digest.hexencode(b).to_i(16).to_s(36)
+  end
+end

data/lib/webhookdb/idempotency.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require "webhookdb/postgres/model"
+
+# Support idempotent operations.
+# This is very useful when
+# 1) protecting the API against requests dispatched multiple times,
+# as browsers are liable to do, and
+# 2) designing parts of a system so they can be used idempotently, especially async jobs.
+# This ensures an event can be republished if a job fails, but jobs that worked won't be re-run.
+#
+# In general, you do not use Idempotency instances directly;
+# instead, you will use once_ever and every.
+# For example, to only send a welcome email once:
+#
+#   Webhookdb::Idempotency.once_ever.under_key("welcome-email-#{customer.id}") { send_welcome_email(customer) }
+#
+# Similarly, to prevent an action email from going out multiple times in a short period accidentally:
+#
+#   Webhookdb::Idempotency.every(1.hour).under_key("new-order-#{order.id}") { send_new_order_email(order) }
+#
+# Note that idempotency cannot be executed while already in a transaction.
+# If it were, the unique row would not be visible to other transactions.
+# So the new row must be committed, then the idempotency evaluated (and the callback potentially run).
+# To disable this check, set 'Postgres.unsafe_skip_transaction_check' to true,
+# usually using the :no_transaction_check spec metadata.
+#
+class Webhookdb::Idempotency < Webhookdb::Postgres::Model(:idempotencies)
+  extend Webhookdb::MethodUtilities
+
+  NOOP = :skipped
+
+  # Skip the transaction check. Useful in unit tests. See class docs for details.
+  singleton_predicate_accessor :skip_transaction_check
+
+  def self.once_ever
+    idem = self.new
+    idem.__once_ever = true
+    return idem
+  end
+
+  def self.every(interval)
+    idem = self.new
+    idem.__every = interval
+    return idem
+  end
+
+  attr_accessor :__every, :__once_ever
+
+  def under_key(key, &block)
+    self.key = key
+    return self.execute(&block) if block
+    return self
+  end
+
+  def execute
+    Webhookdb::Postgres.check_transaction(
+      self.db,
+      "Cannot use idempotency while already in a transaction, since side effects may not be idempotent",
+    )
+
+    self.class.dataset.insert_conflict.insert(key: self.key)
+    self.db.transaction do
+      idem = Webhookdb::Idempotency[key: self.key].lock!
+      if idem.last_run.nil?
+        result = yield()
+        idem.update(last_run: Time.now)
+        return result
+      end
+      return NOOP if self.__once_ever
+      return NOOP if Time.now < (idem.last_run + self.__every)
+      result = yield()
+      idem.update(last_run: Time.now)
+      return result
+    end
+  end
+end
+
+# Table: idempotencies
+# -------------------------------------------------------------------------------------
+# Columns:
+#  id         | integer                  | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY
+#  created_at | timestamp with time zone | NOT NULL DEFAULT now()
+#  updated_at | timestamp with time zone |
+#  last_run   | timestamp with time zone |
+#  key        | text                     |
+# Indexes:
+#  idempotencies_pkey    | PRIMARY KEY btree (id)
+#  idempotencies_key_key | UNIQUE btree (key)
+# -------------------------------------------------------------------------------------

data/lib/webhookdb/increase.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+class Webhookdb::Increase
+  extend Webhookdb::MethodUtilities
+  include Appydays::Configurable
+  include Appydays::Loggable
+
+  configurable(:increase) do
+    setting :http_timeout, 30
+  end
+
+  def self.webhook_response(request, webhook_secret)
+    http_signature = request.env["HTTP_X_BANK_WEBHOOK_SIGNATURE"]
+
+    return Webhookdb::WebhookResponse.error("missing hmac") if http_signature.nil?
+
+    request.body.rewind
+    request_data = request.body.read
+
+    computed_signature = OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new("sha256"), webhook_secret, request_data)
+
+    if http_signature != "sha256=" + computed_signature
+      # Invalid signature
+      self.logger.warn "increase signature verification error"
+      return Webhookdb::WebhookResponse.error("invalid hmac")
+    end
+
+    return Webhookdb::WebhookResponse.ok
+  end
+
+  # this helper function finds the relevant object data and helps us avoid repeated code
+  def self.find_desired_object_data(body)
+    return body.fetch("data", body)
+  end
+
+  # this function interprets webhook contents to assist with filtering webhooks by object type in our increase services
+  def self.contains_desired_object(webhook_body, desired_object_name)
+    object_of_interest = self.find_desired_object_data(webhook_body)
+    object_id = object_of_interest["id"]
+    return object_id.include?(desired_object_name)
+  end
+end

data/lib/webhookdb/intercom.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "appydays/configurable"
+
+module Webhookdb::Intercom
+  include Appydays::Configurable
+
+  configurable(:intercom) do
+    setting :client_id, "whdb_intercom_client_id", key: "INTERCOM_CLIENT_ID"
+    setting :client_secret, "whdb_intercom_client_secret", key: "INTERCOM_CLIENT_SECRET"
+    setting :http_timeout, 30
+    setting :page_size, 20
+  end
+
+  def self.verify_webhook(data, hmac_header)
+    calculated_hmac = "sha1=#{OpenSSL::HMAC.hexdigest('SHA1', self.client_secret, data)}"
+    return ActiveSupport::SecurityUtils.secure_compare(calculated_hmac, hmac_header)
+  end
+
+  def self.auth_headers(token)
+    return {"Intercom-Version" => "2.9", "Authorization" => "Bearer #{token}"}
+  end
+end

data/lib/webhookdb/jobs/amigo_test_jobs.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require "amigo/retry"
+require "amigo/queue_backoff_job"
+require "amigo/durable_job"
+require "sidekiq"
+
+# Use this to verify the behavior of durable jobs:
+#
+# - Ensure DISABLE_DURABLE_JOBS_POLL env var is set.
+# - `make run` and then `Webhookdb::Async.open_web` to go to Sidekiq web UI.
+# - `make run-workers`, and copy the PID.
+# - From pry: `Webhookdb::Jobs::DurableSleeper.setup_test_run(30)`
+# - Jobs are put into the queue and the workers will be 'running' (sleeping).
+# - Wait for some jobs to be done sleeping.
+# - `pkill -9 <pid>`, kills Sidekiq without cleanup.
+# - `Webhookdb::Jobs::DurableSleeper.print_status` will print
+#   the queue size. It would be '10' if you started with 30 jobs,
+#   10 processed, and the worker was killed while 10 more were processing
+#   (leaving 10 unprocessed jobs in the queue).
+#   It will also print dead jobs, which will be 0.
+#   It will also print processed jobs, will be be 10.
+# - Restart workers with `make run-workers`.
+# - The next 10 workers will run (queue). `print_status` returns 0 for queue and dead size,
+#   and 20 for jobs processed.
+# - Run `Amigo::DurableJob.poll_jobs`.
+# - Go to the web UI's Dead jobs. Observe 10 jobs are there. `print_status` also shows 10 jobs as dead.
+# - Retry those jobs.
+# - `print_status` shows 0 dead and 30 processed jobs.
+#
+class Webhookdb::Jobs::DurableSleeper
+  include Sidekiq::Job
+  include Amigo::DurableJob
+
+  MUX = Mutex.new
+  COUNTER_FILE = ".durable-sleeper-counter"
+
+  def self.heartbeat_extension
+    return 20.seconds
+  end
+
+  def perform(duration=5)
+    self.logger.info("sleeping")
+    sleep(duration)
+    MUX.synchronize do
+      done = File.read(COUNTER_FILE).to_i
+      done += 1
+      File.write(COUNTER_FILE, done.to_s)
+    end
+  end
+
+  def self.setup_test_run(count=30)
+    File.write(COUNTER_FILE, "0")
+    count.times { Webhookdb::Jobs::DurableSleeper.perform_async }
+  end
+
+  def self.print_status
+    done = File.read(COUNTER_FILE).to_i
+    puts "Queue Size: #{Sidekiq::Queue.new.size}"
+    puts "Processed:  #{done}"
+    puts "Dead Set:   #{Sidekiq::DeadSet.new.size}"
+  end
+end
+
+# Use this and BackoffShouldBeRun to test the behavior of BackoffJob.
+#
+# First, fill up the 'netout' queue with a ton of these slow jobs:
+# From pry: `Webhookdb::Async.require_jobs; 500.times { Webhookdb::Jobs::BackoffShouldBeRescheduled.perform_async }`
+#
+# Then, fill up the other queues with fast jobs:
+# `1000.times { Webhookdb::Jobs::BackoffShouldRun.perform_async }`
+#
+# Then go to http://localhost:18001/sidekiq (user/pass) to check the latency.
+# The netout queue should get slow,
+# but the other queues should not build up much of a backlog.
+class Webhookdb::Jobs::BackoffShouldBeRescheduled
+  include Sidekiq::Job
+  include Amigo::DurableJob # Uncomment to verify performance with durable jobs, which hit the DB.
+  include Amigo::QueueBackoffJob
+
+  sidekiq_options queue: "netout"
+
+  def perform(duration=3)
+    sleep(duration)
+  end
+end
+
+class Webhookdb::Jobs::BackoffShouldRun
+  include Sidekiq::Job
+
+  def perform(duration=0.1)
+    sleep(duration)
+  end
+end
+
+class Webhookdb::Jobs::RetryChecker
+  include Sidekiq::Job
+
+  def perform(action, interval, attempts)
+    case action
+      when "retry"
+        raise Amigo::Retry::Retry, interval
+      when "die"
+        raise Amigo::Retry::Die
+    else
+        raise Amigo::Retry::Die.new(attempts, interval)
+    end
+  end
+end
+
+class Webhookdb::Jobs::Erroring
+  include Sidekiq::Job
+
+  def perform(succeed: false)
+    return if succeed
+    raise "erroring as asked!"
+  end
+end

data/lib/webhookdb/jobs/backfill.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "amigo/queue_backoff_job"
+require "amigo/durable_job"
+require "webhookdb/async/job"
+require "webhookdb/jobs"
+
+class Webhookdb::Jobs::Backfill
+  extend Webhookdb::Async::Job
+  include Amigo::DurableJob
+  include Amigo::QueueBackoffJob
+
+  on "webhookdb.backfilljob.run"
+  sidekiq_options queue: "netout"
+
+  def dependent_queues
+    # This is really the lowest-priority job so always defer to other queues.
+    return super
+  end
+
+  def _perform(event)
+    bfjob = self.lookup_model(Webhookdb::BackfillJob, event.payload)
+    sint = bfjob.service_integration
+    self.with_log_tags(sint.log_tags.merge(backfill_job_id: bfjob.opaque_id)) do
+      if bfjob.finished?
+        self.logger.info "skipping_finished_backfill_job"
+      else
+        sint.replicator.backfill(bfjob)
+      end
+    end
+  end
+end

data/lib/webhookdb/jobs/create_mirror_table.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+
+class Webhookdb::Jobs::CreateMirrorTable
+  extend Webhookdb::Async::Job
+
+  on "webhookdb.serviceintegration.created"
+  sidekiq_options queue: "critical"
+
+  def _perform(event)
+    sint = self.lookup_model(Webhookdb::ServiceIntegration, event)
+    self.with_log_tags(sint.log_tags) do
+      svc = Webhookdb::Replicator.create(sint)
+      svc.create_table(if_not_exists: true)
+    end
+  end
+end