webhookdb 0.1.0

data/lib/webhookdb/async/job.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "amigo/job"
+
+require "webhookdb/async"
+
+module Webhookdb::Async::Job
+  def self.extended(cls)
+    cls.extend Amigo::Job
+    cls.include(InstanceMethods)
+  end
+
+  module InstanceMethods
+    def with_log_tags(tags, &)
+      Webhookdb::Async::JobLogger.with_log_tags(tags, &)
+    end
+  end
+end

data/lib/webhookdb/async/job_logger.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "appydays/loggable/sidekiq_job_logger"
+
+class Webhookdb::Async::JobLogger < Appydays::Loggable::SidekiqJobLogger
+  protected def slow_job_seconds
+    return Webhookdb::Async.slow_job_seconds
+  end
+
+  def self.durable_job_failure_notifier(job)
+    # See https://github.com/sidekiq/sidekiq/wiki/Job-Format#activejob-middleware-format
+    # for job format.
+    job = job.dup
+    # These fields always exist.
+    jargs = job.delete("args")
+    jcls = job.delete("class")
+    jid = job.delete("jid")
+    jq = job.delete("queue")
+    jcreated = job.delete("created_at")
+    safe_fields = [
+      {title: "Job ID", value: jid, short: true},
+      {title: "Job Class", value: "`#{jcls}`", short: true},
+      {title: "Args", value: "```#{jargs.to_json}```"},
+      {title: "Queue", value: jq, short: true},
+      {title: "Created At", value: self._ts("created_at", jcreated), short: true},
+    ]
+    # The remaining fields can be added dynamically.
+    other_fields = job.compact.
+      map { |k, v| {title: k.humanize, value: self._ts(k, v), short: true} }
+    Webhookdb::DeveloperAlert.new(
+      subsystem: "Job Died",
+      emoji: ":zombie:",
+      fallback: "Job #{jcls}[#{jid}][#{jargs}] moved to DeadSet",
+      fields: safe_fields + other_fields,
+    ).emit
+  end
+
+  def self._ts(k, v)
+    return nil if v.nil?
+    return v unless k.end_with?("_at")
+    return Time.at(v) if v.is_a?(Numeric)
+    return Time.parse(v) if v.is_a?(String)
+    return v
+  end
+end

data/lib/webhookdb/async/scheduled_job.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "amigo/scheduled_job"
+
+require "webhookdb/async"
+
+module Webhookdb::Async::ScheduledJob
+  def self.extended(cls)
+    cls.extend Amigo::ScheduledJob
+    cls.include(InstanceMethods)
+  end
+
+  module InstanceMethods
+    def with_log_tags(tags, &)
+      Webhookdb::Async::JobLogger.with_log_tags(tags, &)
+    end
+  end
+end

data/lib/webhookdb/async.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require "amigo/retry"
+require "amigo/durable_job"
+require "amigo/rate_limited_error_handler"
+require "appydays/configurable"
+require "appydays/loggable"
+require "sentry-sidekiq"
+require "sidekiq"
+require "sidekiq-cron"
+
+Sidekiq.strict_args!
+
+require "webhookdb"
+
+module Webhookdb::Async
+  include Appydays::Configurable
+  include Appydays::Loggable
+  extend Webhookdb::MethodUtilities
+
+  require "webhookdb/async/job_logger"
+  require "webhookdb/async/audit_logger"
+
+  configurable(:async) do
+    # The number of (Float) seconds that should be considered "slow" for a job.
+    # Jobs that take longer than this amount of time will be logged
+    # at `warn` level.
+    setting :slow_job_seconds, 1.0
+
+    setting :sidekiq_redis_url, "redis://localhost:6379/0", key: "REDIS_URL"
+    setting :sidekiq_redis_provider, ""
+    # For sidekiq web UI. Randomize a default so they will only be useful if set.
+    setting :web_username, SecureRandom.hex(8)
+    setting :web_password, SecureRandom.hex(8)
+
+    setting :error_reporting_sample_rate, 0.1
+    setting :error_reporting_ttl, 120
+
+    after_configured do
+      # Very hard to to test this, so it's not tested.
+      url = self.sidekiq_redis_provider.present? ? ENV.fetch(self.sidekiq_redis_provider, nil) : self.sidekiq_redis_url
+      redis_params = {url:}
+      if url.start_with?("rediss:") && ENV["HEROKU_APP_ID"]
+        # rediss: schema is Redis with SSL. They use self-signed certs, so we have to turn off SSL verification.
+        # There is not a clear KB on this, you have to piece it together from Heroku and Sidekiq docs.
+        redis_params[:ssl_params] = {verify_mode: OpenSSL::SSL::VERIFY_NONE}
+      end
+      Sidekiq.configure_server do |config|
+        config.redis = redis_params
+        config.options[:job_logger] = Webhookdb::Async::JobLogger
+        # We do NOT want the unstructured default error handler
+        config.error_handlers.replace([Webhookdb::Async::JobLogger.method(:error_handler)])
+        # We must then replace the otherwise-automatically-added sentry middleware
+        config.error_handlers << Amigo::RateLimitedErrorHandler.new(
+          Sentry::Sidekiq::ErrorHandler.new,
+          sample_rate: self.error_reporting_sample_rate,
+          ttl: self.error_reporting_ttl,
+        )
+        config.death_handlers << Webhookdb::Async::JobLogger.method(:death_handler)
+        config.server_middleware.add(Amigo::DurableJob::ServerMiddleware)
+        # We use the dead set to move jobs that we need to retry manually
+        config.options[:dead_max_jobs] = 999_999_999
+        config.server_middleware.add(Amigo::Retry::ServerMiddleware)
+      end
+
+      Amigo::DurableJob.failure_notifier = Webhookdb::Async::JobLogger.method(:durable_job_failure_notifier)
+
+      Sidekiq.configure_client do |config|
+        config.redis = redis_params
+        config.client_middleware.add(Amigo::DurableJob::ClientMiddleware)
+      end
+    end
+  end
+
+  def self.open_web
+    u = URI(Webhookdb.api_url)
+    u.user = self.web_username
+    u.password = self.web_password
+    u.path = "/sidekiq"
+    `open #{u}`
+  end
+
+  # Set up async for the web/client side of things.
+  # This performs common Amigo config,
+  # and sets up the routing/auditing jobs.
+  #
+  # Note that we must also require all async jobs,
+  # since in some cases we may have sidekiq middleware that needs
+  # access to the actual job class, so it must be available.
+  def self.setup_web
+    self._setup_common
+    Amigo.install_amigo_jobs
+    self._require_jobs
+    return true
+  end
+
+  # Set up the worker process.
+  # This peforms common Amigo config,
+  # sets up the routing/audit jobs (since jobs may publish to other jobs),
+  # requires the actual jobs,
+  # and starts the cron.
+  def self.setup_workers
+    self._setup_common
+    Amigo.install_amigo_jobs
+    self._require_jobs
+    Amigo.start_scheduler
+    return true
+  end
+
+  # Set up for tests.
+  # This performs common config and requires the jobs.
+  # It does not install the routing/auditing jobs,
+  # since those should only be installed at specific times.
+  def self.setup_tests
+    return if Amigo.structured_logging # assume we are set up
+    self._setup_common
+    self._require_jobs
+    return true
+  end
+
+  def self._require_jobs
+    Amigo::DurableJob.replace_database_settings(
+      loggers: [Webhookdb.logger],
+      **Webhookdb::Dbutil.configured_connection_options,
+    )
+    Gem.find_files(File.join("webhookdb/jobs/*.rb")).each do |path|
+      require path
+    end
+  end
+
+  def self._setup_common
+    raise "Async already setup, only call this once" if Amigo.structured_logging
+    Amigo.structured_logging = true
+    Amigo.log_callback = lambda { |j, lvl, msg, o|
+      lg = j ? Appydays::Loggable[j] : Webhookdb::Async::JobLogger.logger
+      lg.send(lvl, msg, o)
+    }
+  end
+end
+
+require "webhookdb/async/audit_logger"
+Amigo.audit_logger_class = Webhookdb::Async::AuditLogger

data/lib/webhookdb/aws.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require "aws-sdk-core"
+require "aws-sdk-sts"
+require "appydays/configurable"
+
+require "webhookdb" unless defined?(Webhookdb)
+
+module Webhookdb::AWS
+  extend Webhookdb::MethodUtilities
+  include Appydays::Configurable
+  include Appydays::Loggable
+
+  class ShimLogger
+    def initialize(logger, operation_prefix: "aws_", level: :debug, error_level: :warn)
+      @logger = logger
+      @param_formatter = ::Aws::Log::ParamFormatter.new({})
+      @param_filter = ::Aws::Log::ParamFilter.new({})
+      @operation_prefix = operation_prefix
+      @level = level
+      @error_level = error_level
+    end
+
+    # Logger method. Receives the unformatted response form the shim formatter,
+    # and logs a structured log.
+    # @param [Seahorse::Client::Response] response
+    def logshim(response)
+      level = @level
+      msg = @operation_prefix + response.context.operation_name.to_s
+      params = response.context.params
+      type = response.context.operation.input.shape.struct_class
+      ctx = {
+        aws_client: (response.context.client.class.name || "").delete_prefix("Aws::").delete_suffix("::Client"),
+        http_response_code: response.context.http_response.status_code,
+        elapsed: response.context[:logging_completed_at] - response.context[:logging_started_at],
+        request_params: @param_formatter.summarize(@param_filter.filter(params, type)),
+      }
+      ctx[:retries] = response.context.retries if response.context.retries.positive?
+      if response.error
+        level = @error_level
+        ctx[:error_class] = response.error.class.name
+        ctx[:error_message] = response.error.message
+      end
+      @logger.send(level, msg, ctx)
+    end
+  end
+
+  class ShimFormatter
+    # Normally this must return a string, but we need to have it return a raw response
+    # for use in the shim logger.
+    # @param [Seahorse::Client::Response] response
+    # @return [String]
+    def format(response)
+      # noinspection RubyMismatchedReturnType
+      return response
+    end
+  end
+
+  singleton_attr_reader :sts_client
+
+  configurable(:aws) do
+    # WebhookDB's AWS Account ID.
+    # Used for things like cross-account role assumption.
+    setting :external_account_id, "054088425385"
+
+    # Some stuff doesn't work right with explicit config, so force-set the ENV
+    setting :access_key_id,
+            "default-access",
+            key: "AWS_ACCESS_KEY_ID",
+            side_effect: ->(v) { ENV["AWS_ACCESS_KEY_ID"] = v }
+    setting :secret_access_key,
+            "default-secret",
+            key: "AWS_SECRET_ACCESS_KEY",
+            side_effect: ->(v) { ENV["AWS_SECRET_ACCESS_KEY"] = v }
+    setting :region,
+            "us-west-2",
+            key: "AWS_REGION",
+            side_effect: ->(v) { ENV["AWS_REGION"] = v }
+    setting :http_timeout, 30
+
+    after_configured do
+      ::Aws.config.update(
+        logger: ShimLogger.new(self.logger),
+        log_formatter: ShimFormatter.new,
+        log_level: :logshim,
+      )
+
+      @sts_client = Aws::STS::Client.new
+    end
+  end
+
+  REGIONS_TO_LOCATIONS = Aws.partitions.each_with_object({}) do |partition, h|
+    partition.regions.each do |region|
+      h[region.name] = region.description
+    end
+  end.freeze
+  LOCATIONS_TO_REGIONS = REGIONS_TO_LOCATIONS.to_a.to_h(&:reverse!).freeze
+end

data/lib/webhookdb/backfill_job.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+# Represents the boundaries around a single execution of backfilling an integration.
+# Each instance points to a single run of a single integration.
+# There may be child jobs pointing to dependent integrations,
+# or a parent job for a dependency backfill.
+#
+# When creating jobs, you can create a single job (a 'shallow' backfill)
+# with +create+, or use +create_recursive+ to create jobs for all dependencies
+# (a 'job group').
+#
+# Each job tracks when the backfill starts and ends.
+# Iterating the full job graph can determine if a group is fully finished,
+# or still in-progress.
+#
+class Webhookdb::BackfillJob < Webhookdb::Postgres::Model(:backfill_jobs)
+  plugin :timestamps
+
+  many_to_one :service_integration, class: "Webhookdb::ServiceIntegration"
+  many_to_one :parent_job, class: "Webhookdb::BackfillJob"
+  one_to_many :child_jobs, class: "Webhookdb::BackfillJob", key: :parent_job_id
+  many_to_one :created_by, class: "Webhookdb::Customer"
+
+  attr_accessor :_fixture_cascade
+
+  # @return [Webhookdb::BackfillJob]
+  def self.create_recursive(service_integration:, incremental:, created_by: nil, parent_job: nil, criteria: nil)
+    self.db.transaction do
+      root = self.create(service_integration:, parent_job:, incremental:, created_by:, criteria: criteria || {})
+      root.setup_recursive
+      root
+    end
+  end
+
+  # You should use ::create_recursive instead.
+  # This is mostly here for use in tests/fixtures.
+  def setup_recursive
+    raise Webhookdb::InvalidPrecondition, "already has children" if self.child_jobs.present?
+    self.service_integration.dependents.map do |dep|
+      self.class.create_recursive(service_integration: dep, parent_job: self, incremental:, criteria:)
+    end
+  end
+
+  def incremental? = self.incremental
+
+  def started? = !!self.started_at
+  def finished? = !!self.finished_at
+
+  def fully_finished_at
+    parent_finished = self.finished_at
+    return nil if parent_finished.nil?
+    children_finished = self.child_jobs.map(&:fully_finished_at)
+    return nil if children_finished.any?(&:nil?)
+    children_finished << parent_finished
+    return children_finished.max
+  end
+
+  def fully_finished? = !!self.fully_finished_at
+
+  def status
+    return "enqueued" unless self.started?
+    return "finished" if self.fully_finished?
+    return "inprogress"
+  end
+
+  def enqueue
+    self.publish_deferred("run", self.id)
+  end
+
+  def enqueue_children
+    self.child_jobs.each(&:enqueue)
+  end
+
+  #
+  # :section: Sequel Hooks
+  #
+
+  def before_create
+    self[:opaque_id] ||= Webhookdb::Id.new_opaque_id("bfj")
+  end
+end
+
+# Table: backfill_jobs
+# ---------------------------------------------------------------------------------------------------------------------------
+# Columns:
+#  id                     | integer                  | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY
+#  created_at             | timestamp with time zone | NOT NULL DEFAULT now()
+#  updated_at             | timestamp with time zone |
+#  started_at             | timestamp with time zone |
+#  finished_at            | timestamp with time zone |
+#  opaque_id              | text                     | NOT NULL
+#  service_integration_id | integer                  | NOT NULL
+#  parent_job_id          | integer                  |
+#  created_by_id          | integer                  |
+#  incremental            | boolean                  | NOT NULL
+# Indexes:
+#  backfill_jobs_pkey                         | PRIMARY KEY btree (id)
+#  backfill_jobs_opaque_id_key                | UNIQUE btree (opaque_id)
+#  backfill_jobs_parent_job_id_index          | btree (parent_job_id)
+#  backfill_jobs_service_integration_id_index | btree (service_integration_id)
+# Foreign key constraints:
+#  backfill_jobs_created_by_id_fkey          | (created_by_id) REFERENCES customers(id) ON DELETE SET NULL
+#  backfill_jobs_parent_job_id_fkey          | (parent_job_id) REFERENCES backfill_jobs(id)
+#  backfill_jobs_service_integration_id_fkey | (service_integration_id) REFERENCES service_integrations(id) ON DELETE CASCADE
+# Referenced By:
+#  backfill_jobs | backfill_jobs_parent_job_id_fkey | (parent_job_id) REFERENCES backfill_jobs(id)
+# ---------------------------------------------------------------------------------------------------------------------------

data/lib/webhookdb/backfiller.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+# Implementation of a generic backfill pattern.
+class Webhookdb::Backfiller
+  # Called for each item.
+  def handle_item(item)
+    raise NotImplementedError
+  end
+
+  def fetch_backfill_page(pagination_token, last_backfilled:)
+    raise NotImplementedError
+  end
+
+  # Use nil last_backfilled for a full sync, pass it for an incremental.
+  # Should be service integration last_backfilled_at, the timestamp of
+  # the latest resource, etc.
+  def backfill(last_backfilled)
+    pagination_token = nil
+    loop do
+      page, next_pagination_token = self._fetch_backfill_page_with_retry(
+        pagination_token, last_backfilled:,
+      )
+      if page.nil?
+        msg = "Fetching a page should return an empty array, not nil. The service response probably is missing a key?"
+        raise TypeError, msg
+      end
+      pagination_token = next_pagination_token
+      page.each do |item|
+        self.handle_item(item)
+      end
+      Amigo::DurableJob.heartbeat
+      break if pagination_token.blank?
+      if Webhookdb.regression_mode?
+        Webhookdb.logger.warn("regression_mode_backfill_termination", backfiller: self.to_s, pagination_token:)
+        break
+      end
+    end
+    self.flush_pending_inserts if self.respond_to?(:flush_pending_inserts)
+  end
+
+  def max_backfill_retry_attempts
+    return 3
+  end
+
+  def wait_for_retry_attempt(attempt:)
+    Webhookdb::Backfiller.do_retry_wait(attempt)
+  end
+
+  # Make this easy to mock
+  def self.do_retry_wait(seconds)
+    Kernel.sleep(seconds)
+  end
+
+  def _fetch_backfill_page_with_retry(pagination_token, last_backfilled: nil, attempt: 1)
+    return self.fetch_backfill_page(pagination_token, last_backfilled:)
+  rescue Webhookdb::Http::BaseError => e
+    raise e if attempt >= self.max_backfill_retry_attempts
+    # Assume we'll never succeed on a 401, so don't bother retrying.
+    raise e if e.is_a?(Webhookdb::Http::Error) && e.status == 401
+    self.wait_for_retry_attempt(attempt:)
+    return self._fetch_backfill_page_with_retry(pagination_token, last_backfilled:, attempt: attempt + 1)
+  end
+
+  module Bulk
+    def upsert_page_size = raise NotImplementedError("how many items should be upserted at a time")
+    def prepare_body(_body) = raise NotImplementedError("add/remove keys from body before upsert")
+    def upserting_replicator = raise NotImplementedError("the replicator being upserted")
+    def remote_key_column_name = @remote_key_column_name ||= self.upserting_replicator._remote_key_column.name
+
+    def pending_inserts = @pending_inserts ||= {}
+    # Should `_update_where_expr` be used or not?
+    # Default false, since most bulk upserting is backfill,
+    # which should only involve upserting new rows anyway.
+    def conditional_upsert? = false
+
+    def dry_run? = false
+
+    # Add the item to pending upserts, and run the page upsert if needed.
+    # Return the key, and the item being upserted.
+    # @return [Array(String, Hash),Array(nil)]
+    def handle_item(body)
+      self.prepare_body(body)
+      inserting = self.upserting_replicator.upsert_webhook_body(body, upsert: false)
+      return nil, nil if inserting.nil?
+      k = inserting.fetch(self.remote_key_column_name)
+      self.pending_inserts[k] = inserting
+      self.flush_pending_inserts if self.pending_inserts.size >= self.upsert_page_size
+      return k, inserting
+    end
+
+    def flush_pending_inserts
+      return if self.dry_run?
+      return if self.pending_inserts.empty?
+      rows_to_insert = self.pending_inserts.values
+      update_where = self.conditional_upsert? ? self.upserting_replicator._update_where_expr : nil
+      self.upserting_replicator.admin_dataset(timeout: :fast) do |ds|
+        insert_ds = ds.insert_conflict(
+          target: self.upserting_replicator._remote_key_column.name,
+          update: self.upserting_replicator._upsert_update_expr(rows_to_insert.first),
+          update_where:,
+        )
+        insert_ds.multi_insert(rows_to_insert)
+      end
+      self.pending_inserts.clear
+    end
+  end
+end

data/lib/webhookdb/cloudflare.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "appydays/configurable"
+require "appydays/loggable"
+require "oj"
+
+class Webhookdb::Cloudflare
+  include Appydays::Configurable
+  include Appydays::Loggable
+
+  configurable(:cloudflare) do
+    setting :api_token, "set-me-to-token"
+    setting :host, "https://api.cloudflare.com"
+  end
+
+  def self.headers
+    return {
+      "Authorization" => "Bearer #{self.api_token}",
+    }
+  end
+
+  # https://api.cloudflare.com/#dns-records-for-a-zone-create-dns-record
+  def self.create_zone_dns_record(name:, content:, zone_id:, type: "CNAME", ttl: 1)
+    body = {
+      type:,
+      name:,
+      content:,
+      ttl:,
+    }
+    response = Webhookdb::Http.post(
+      self.host + "/client/v4/zones/#{zone_id}/dns_records",
+      body,
+      headers: self.headers,
+      logger: self.logger,
+      timeout: nil,
+    )
+    return Oj.load(response.body)
+  end
+end