webhookdb 1.4.0 → 1.5.0

data/lib/webhookdb/concurrent.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Webhookdb::Concurrent
+  class Timeout < Timeout::Error; end
+
+  # Baseclass for pools for doing work across threads.
+  # Note that these concurrent pools are not for repeated use,
+  # like a normal threadpool. They are for 'fanning out' a single operation
+  # across multiple threads.
+  #
+  # Tasks should not error; if they error, the pool will becomes inoperable:
+  # +post+ and +join+ will re-raise the first task error.
+  class Pool
+    # Add work to the pool.
+    # Will block if no workers are free.
+    # Re-raises the pool's error if the pool has an error.
+    # This is important as we don't want the caller to keep adding work,
+    # if the pool is inoperable.
+    def post(&) = raise NotImplementedError
+
+    # Wait for all work to finish.
+    # Re-raise the first exception for any pool error.
+    def join = raise NotImplementedError
+  end
+
+  # Fake +Pool+ that does work in the calling thread,
+  # but behaves like a threaded pool (ie, tasks do not raise).
+  class SerialPool < Pool
+    def post
+      raise @exception if @exception
+      begin
+        yield
+      rescue StandardError => e
+        @exception = e
+      end
+    end
+
+    def join
+      raise @exception if @exception
+    end
+  end
+
+  # Pool that does work across a given number of threads.
+  # +queue_size+ is how many items can be in the queue before +post+ blocks.
+  # +threads+ defaults to +queue_size+, allowing at most +queue_size+ concurrent work,
+  # which fits the idea of a parallelized pool well.
+  #
+  # If you want the calling thread to queue up a bunch of work ahead of time,
+  # you can use a +Concurrent::ThreadPoolExecutor+. This pool will not allow the enqueing of more work
+  # while the queue is full.
+  class ParallelizedPool < Pool
+    def initialize(queue_size, timeout: nil, threads: nil)
+      super()
+      threads ||= queue_size
+      @timeout = timeout
+      @threads = (1..threads).map do
+        Thread.new do
+          loop { break unless self.do_work }
+        end
+      end
+      @queue = Thread::SizedQueue.new(queue_size)
+      @exception = nil
+    end
+
+    protected def do_work
+      task = @queue.pop
+      return false if task.nil?
+      if task == STOP
+        @queue.close
+        return false
+      end
+      begin
+        task.call
+      rescue StandardError => e
+        @exception ||= e
+        return false
+      end
+      return true
+    end
+
+    def post(&task)
+      raise @exception if @exception
+      added = @queue.push(task, timeout: @timeout)
+      raise Timeout, "waited #{@timeout} to add to the queue" if added.nil?
+      return true
+    end
+
+    def join
+      @queue.push(STOP)
+      @threads.each(&:join)
+      raise @exception if @exception
+    end
+
+    STOP = :stop
+  end
+end

data/lib/webhookdb/connection_cache.rb

@@ -140,20 +140,41 @@ class Webhookdb::ConnectionCache
       conn ||= take_conn(url, single_threaded: true, extensions: [:pg_json, :pg_streaming])
       db_loans[:loaned][t] = conn
     end
-    conn << "SET statement_timeout TO #{timeout * 1000}" if timeout.present?
-    conn << "BEGIN;" if transaction
+    trash_conn = false
     begin
-      result = yield conn
-      conn << "COMMIT;" if transaction
-    rescue Sequel::DatabaseError
-      conn << "ROLLBACK;" if transaction
+      # All database operations need global handling to ensure property pool management.
+      conn << "SET statement_timeout TO #{timeout * 1000}" if timeout.present?
+      conn << "BEGIN;" if transaction
+      begin
+        result = yield conn
+        conn << "COMMIT;" if transaction
+      rescue Sequel::DatabaseError => e
+        # Roll back on any database error; but if we're disconnected, don't bother
+        # since we know the rollback won't reach the database.
+        conn << "ROLLBACK;" if transaction && !e.is_a?(Sequel::DatabaseDisconnectError)
+        raise
+      end
+    rescue Sequel::DatabaseDisconnectError
+      # If we're disconnected, trash this connection rather than re-adding it back to the pool.
+      trash_conn = true
       raise
     ensure
-      conn << "SET statement_timeout TO 0" if timeout.present?
+      reraise = nil
+      if timeout.present?
+        begin
+          # If the timeout fails for whatever reason, assume the connection is toast
+          # and don't return it to the pool.
+          conn << "SET statement_timeout TO 0"
+        rescue Sequel::DatabaseError => e
+          reraise = e
+          trash_conn = true
+        end
+      end
       @mutex.synchronize do
         @dbs_for_urls[url][:loaned].delete(t)
-        @dbs_for_urls[url][:available] << Available.new(conn, Time.now)
+        @dbs_for_urls[url][:available] << Available.new(conn, Time.now) unless trash_conn
       end
+      raise reraise if reraise
     end
     self.prune(url) if now > self.next_prune_at
     return result

data/lib/webhookdb/customer.rb

@@ -16,9 +16,9 @@ class Webhookdb::Customer < Webhookdb::Postgres::Model(:customers)
   class SignupDisabled < Webhookdb::WebhookdbError; end
 
   configurable(:customer) do
-    setting :signup_email_allowlist, ["*"], convert: ->(s) { s.split }
+    setting :signup_email_allowlist, ["*"], convert: lambda(&:split)
     setting :skip_authentication, false
-    setting :skip_authentication_allowlist, [], convert: ->(s) { s.split }
+    setting :skip_authentication_allowlist, [], convert: lambda(&:split)
   end
 
   # The bcrypt hash cost. Changing this would invalidate all passwords!

data/lib/webhookdb/database_document.rb

@@ -10,7 +10,7 @@ class Webhookdb::DatabaseDocument < Webhookdb::Postgres::Model(:database_documen
   include Appydays::Configurable
   configurable(:database_document) do
     setting :skip_authentication, false
-    setting :skip_authentication_allowlist, [], convert: ->(s) { s.split }
+    setting :skip_authentication_allowlist, [], convert: lambda(&:split)
   end
 
   plugin :column_encryption do |enc|

data/lib/webhookdb/db_adapter/default_sql.rb

@@ -8,20 +8,7 @@ module Webhookdb::DBAdapter::DefaultSql
     return s
   end
 
-  def create_table_sql(table, columns, if_not_exists: false)
-    createtable = +"CREATE TABLE "
-    createtable << "IF NOT EXISTS " if if_not_exists
-    createtable << self.qualify_table(table)
-    lines = ["#{createtable} ("]
-    columns[0...-1]&.each { |c| lines << "  #{self.column_create_sql(c)}," }
-    lines << "  #{self.column_create_sql(columns.last)}"
-    lines << ")"
-    return lines.join("\n")
-  end
-
-  def identifier_quote_char
-    raise NotImplementedError
-  end
+  def identifier_quote_char = raise NotImplementedError
 
   # We write our own escaper because we want to only escape what's needed;
   # otherwise we want to avoid quoting identifiers.

data/lib/webhookdb/db_adapter/partition.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class Webhookdb::DBAdapter::Partition < Webhookdb::TypedStruct
+  attr_reader :parent_table, :partition_name, :suffix
+
+  def initialize(**kwargs)
+    super
+    self.typecheck!(:parent_table, Webhookdb::DBAdapter::Table)
+    self.typecheck!(:partition_name, Symbol)
+    self.typecheck!(:suffix, Symbol)
+  end
+
+  def partition_table = Webhookdb::DBAdapter::Table.new(name: self.partition_name, schema: self.parent_table.schema)
+end

data/lib/webhookdb/db_adapter/partitioning.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class Webhookdb::DBAdapter::Partitioning < Webhookdb::TypedStruct
+  HASH = :hash
+  RANGE = :range
+
+  attr_reader :by, :column
+end

data/lib/webhookdb/db_adapter/pg.rb

@@ -11,9 +11,7 @@ class Webhookdb::DBAdapter::PG < Webhookdb::DBAdapter
   VERIFY_TIMEOUT = 2
   VERIFY_STATEMENT = "SELECT 1"
 
-  def identifier_quote_char
-    return '"'
-  end
+  def identifier_quote_char = '"'
 
   def create_index_sql(index, concurrently:)
     tgts = index.targets.map { |c| self.escape_identifier(c.name) }.join(", ")
@@ -26,7 +24,72 @@ class Webhookdb::DBAdapter::PG < Webhookdb::DBAdapter
     return "CREATE#{uniq} INDEX#{concurrent} IF NOT EXISTS #{idxname} ON #{tblname} (#{tgts})#{where}"
   end
 
-  def column_create_sql(column)
+  def create_index_sqls(index, concurrently:, partitions: [])
+    return super if partitions.empty?
+    result = []
+    result << self.create_index_sql(index, concurrently: false).gsub(" ON ", " ON ONLY ")
+    partitions.each do |partition|
+      partition_idx = index.change(table: partition.partition_table, name: "#{index.name}#{partition.suffix}")
+      result << self.create_index_sql(partition_idx, concurrently:)
+      result << "ALTER INDEX #{index.name} ATTACH PARTITION #{partition_idx.name}"
+    end
+    return result
+  end
+
+  def create_table_sql(table, columns, if_not_exists: false, partition: nil)
+    columns = columns.to_a
+    createtable = +"CREATE TABLE "
+    createtable << "IF NOT EXISTS " if if_not_exists
+    createtable << self.qualify_table(table)
+
+    partitioned_pks = []
+    partitioned_uniques = []
+    if partition
+      # We cannot use PRIMARY KEY or UNIQUE when partitioning,
+      # so set those columns as if they're not
+      columns.each_with_index do |c, i|
+        if c.pk?
+          # Set the type to the serial type as if it's a normal PK
+          type = case c.type
+            when BIGINT
+              :bigserial
+            when INTEGER
+              :serial
+            else
+              c.type
+          end
+          columns[i] = c.change(pk: false, type:)
+          partitioned_pks << c
+        elsif c.unique?
+          columns[i] = c.change(unique: false)
+          partitioned_uniques << c
+        end
+      end
+    end
+    tbl_lines = columns.map { |c| self.create_column_sql(c) }
+    tbl_lines.concat(partitioned_pks.map do |c|
+      pkcols = [partition.column, c.name].uniq.join(", ")
+      "PRIMARY KEY (#{pkcols})"
+    end)
+    tbl_lines.concat(partitioned_uniques.map { |c| "UNIQUE (#{partition.column}, #{c.name})" })
+    lines = ["#{createtable} ("]
+    lines << ("  " + tbl_lines.join(",\n  "))
+    lines << ")"
+    if partition
+      m = case partition.by
+        when Webhookdb::DBAdapter::Partitioning::HASH
+          "HASH"
+        when Webhookdb::DBAdapter::Partitioning::RANGE
+          "RANGE"
+        else
+          raise ArgumentError, "unknown partition method: #{partition.by}"
+      end
+      lines << "PARTITION BY #{m} (#{partition.column})"
+    end
+    return lines.join("\n")
+  end
+
+  def create_column_sql(column)
     modifiers = +""
     coltype = COLTYPE_MAP.fetch(column.type)
     if column.pk?
@@ -42,8 +105,15 @@ class Webhookdb::DBAdapter::PG < Webhookdb::DBAdapter
     return "#{colname} #{coltype}#{modifiers}"
   end
 
+  def create_hash_partition_sql(table, partition_count, remainder)
+    tbl = self.qualify_table(table)
+    s = "CREATE TABLE #{tbl}_#{remainder} PARTITION OF #{tbl} " \
+        "FOR VALUES WITH (MODULUS #{partition_count}, REMAINDER #{remainder})"
+    return s
+  end
+
   def add_column_sql(table, column, if_not_exists: false)
-    c = self.column_create_sql(column)
+    c = self.create_column_sql(column)
     ifne = if_not_exists ? " IF NOT EXISTS" : ""
     return "ALTER TABLE #{self.qualify_table(table)} ADD COLUMN#{ifne} #{c}"
   end
@@ -92,5 +162,7 @@ class Webhookdb::DBAdapter::PG < Webhookdb::DBAdapter
     TEXT_ARRAY => "text[]",
     TIMESTAMP => "timestamptz",
     UUID => "uuid",
+    :serial => "serial",
+    :bigserial => "bigserial",
   }.freeze
 end

data/lib/webhookdb/db_adapter/snowflake.rb

@@ -28,10 +28,21 @@ class Webhookdb::DBAdapter::Snowflake < Webhookdb::DBAdapter
   end
 
   def create_index_sql(*)
-    raise NotImplementedError, "Snowflake does not support indices"
+    raise Webhookdb::InvalidPrecondition, "Snowflake does not support indices"
   end
 
-  def column_create_sql(column)
+  def create_table_sql(table, columns, if_not_exists: false, **)
+    createtable = +"CREATE TABLE "
+    createtable << "IF NOT EXISTS " if if_not_exists
+    createtable << self.qualify_table(table)
+    lines = ["#{createtable} ("]
+    columns[0...-1]&.each { |c| lines << "  #{self.create_column_sql(c)}," }
+    lines << "  #{self.create_column_sql(columns.last)}"
+    lines << ")"
+    return lines.join("\n")
+  end
+
+  def create_column_sql(column)
     modifiers = +""
     if column.unique?
       modifiers << " UNIQUE NOT NULL"
@@ -44,7 +55,7 @@ class Webhookdb::DBAdapter::Snowflake < Webhookdb::DBAdapter
   end
 
   def add_column_sql(table, column, if_not_exists: false)
-    c = self.column_create_sql(column)
+    c = self.create_column_sql(column)
     # Snowflake has no 'ADD COLUMN IF NOT EXISTS' so we need to query the long way around
     add_sql = "ALTER TABLE #{self.qualify_table(table)} ADD COLUMN #{c}"
     return add_sql unless if_not_exists
@@ -118,9 +129,7 @@ class Webhookdb::DBAdapter::Snowflake < Webhookdb::DBAdapter
     conn.execute(statement)
   end
 
-  def identifier_quote_char
-    return ""
-  end
+  def identifier_quote_char = ""
 
   COLTYPE_MAP = {
     BIGINT => "bigint",

data/lib/webhookdb/db_adapter.rb

@@ -2,6 +2,8 @@
 
 class Webhookdb::DBAdapter
   require "webhookdb/db_adapter/column_types"
+  require "webhookdb/db_adapter/partition"
+  require "webhookdb/db_adapter/partitioning"
 
   class UnsupportedAdapter < Webhookdb::ProgrammingError; end
 
@@ -149,21 +151,41 @@ class Webhookdb::DBAdapter
     raise NotImplementedError
   end
 
+  # Return the CREATE TABLE sql to create table with columns.
   # @param [Table] table
   # @param [Array<Column>] columns
   # @param [Schema] schema
-  # @param [Boolean] if_not_exists
+  # @param [TrueClass,FalseClass] if_not_exists If true, use CREATE TABLE IF NOT EXISTS.
+  # @param partition [Webhookdb::DBAdapter::Partitioning,nil] If provided,
+  #   adds a "PARTITION BY HASH (partition_column_name)" to the returned SQL.
   # @return [String]
-  def create_table_sql(table, columns, schema: nil, if_not_exists: false)
+  def create_table_sql(table, columns, schema: nil, if_not_exists: false, partition: nil)
     raise NotImplementedError
   end
 
+  # We write our own escaper because we want to only escape what's needed;
+  # otherwise we want to avoid quoting identifiers.
+  def escape_identifier(s) = raise NotImplementedError
+
   # @param [Index] index
   # @return [String]
   def create_index_sql(index, concurrently:)
     raise NotImplementedError
   end
 
+  # Create indices, including for partitions.
+  # By default, just call create_index_sql and return it in a single-item array.
+  # Override if creating indices while using partitions requires extra logic.
+  # @param partitions [Array<Webhookdb::DBAdapter::Partition>]
+  # @return [Array<String>]
+  def create_index_sqls(index, concurrently:, partitions: [])
+    _ = partitions
+    return [self.create_index_sql(index, concurrently:)]
+  end
+
+  # @param column [Column] The column to create SQL for.
+  def create_column_sql(column) = raise NotImplementedError
+
   # @param [Table] table
   # @param [Column] column
   # @param [Boolean] if_not_exists

data/lib/webhookdb/fixtures/logged_webhooks.rb

@@ -35,6 +35,10 @@ module Webhookdb::Fixtures::LoggedWebhooks
     self.response_status = rand(400..599)
   end
 
+  decorator :truncated do |t=Time.now|
+    self.truncated_at = t
+  end
+
   decorator :with_organization do |org={}|
     org = Webhookdb::Fixtures.organization.create(org) unless org.is_a?(Webhookdb::Organization)
     self.organization = org

data/lib/webhookdb/fixtures/organization_error_handlers.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "faker"
+
+require "webhookdb/fixtures"
+
+module Webhookdb::Fixtures::OrganizationErrorHandlers
+  extend Webhookdb::Fixtures
+
+  fixtured_class Webhookdb::Organization::ErrorHandler
+
+  base :organization_error_handler do
+    self.url ||= Faker::Internet.url
+  end
+
+  before_saving do |instance|
+    instance.organization ||= Webhookdb::Fixtures.organization.create
+    instance
+  end
+end

data/lib/webhookdb/http.rb

@@ -2,6 +2,7 @@
 
 require "appydays/configurable"
 require "appydays/loggable/httparty_formatter"
+require "down/httpx"
 require "httparty"
 
 module Webhookdb::Http
@@ -96,25 +97,38 @@ module Webhookdb::Http
     options[:log_level] = self.log_level
   end
 
-  # Convenience wrapper around Down that handles gzip.
+  # Convenience wrapper around Down so we can use our preferred implementation.
+  # See commit history for more info.
   # @return Array<Down::ChunkedIO, IO> Tuple
   def self.chunked_download(request_url, rewindable: false, **down_kw)
-    io = Down::NetHttp.open(request_url, rewindable:, **down_kw)
-    if io.data[:headers].fetch("Content-Encoding", "").include?("gzip")
-      # If the response is gzipped, Down doesn't handle it properly.
-      # Wrap it with gzip reader, and force the encoding to binary
-      # the server may send back a header like Content-Type: text/plain; UTF-8,
-      # so each line Down yields via #gets will have force_encoding('utf-8').
-      # https://github.com/janko/down/issues/87
-      io.instance_variable_set(:@encoding, "binary")
-      io = Zlib::GzipReader.wrap(io)
-    end
+    uri = URI(request_url)
+    raise URI::InvalidURIError, "#{request_url} must be an http/s url" unless ["http", "https"].include?(uri.scheme)
+    down_kw[:headers] ||= {}
+    down_kw[:headers]["User-Agent"] ||= self.user_agent
+    io = Down::Httpx.open(uri, rewindable:, **down_kw)
     return io
   end
+end
+
+class Down::Httpx
+  alias _original_response_error! response_error!
+  def response_error!(response)
+    # For some reason, Down's httpx backend uses TooManyRedirects for every status code...
+    raise Down::NotModified if response.status == 304
+    return self._original_response_error!(response)
+  end
+end
 
-  def self.gzipped?(string)
-    return false if string.length < 3
-    b = string[..2].bytes
-    return b[0] == 0x1f && b[1] == 0x8b
+class HTTPX::Response::Body
+  alias _original_initialize initialize
+  def initialize(*)
+    _original_initialize(*)
+    # If the encoding is an invalid one like 'utf8' vs 'utf-8', modify what's was in the charset.
+    # See https://github.com/HoneyryderChuck/httpx/issues/66
+    return unless @encoding.is_a?(String) && (md = @encoding.match(/^(utf)(\d+)$/))
+    @encoding = "#{md[1]}-#{md[2]}"
   end
 end
+
+# Not sure why, but Down uses this, loads the plugin, but the constant isn't defined.
+require "httpx/plugins/follow_redirects"

data/lib/webhookdb/icalendar.rb

@@ -7,20 +7,41 @@ module Webhookdb::Icalendar
   # If a manual backfill is attempted, direct customer to this url.
   DOCUMENTATION_URL = "https://docs.webhookdb.com/guides/icalendar/"
 
+  EVENT_REPLICATORS = ["icalendar_event_v1", "icalendar_event_v1_partitioned"].freeze
+
   include Appydays::Configurable
 
   configurable(:icalendar) do
-    # Do not store events older then this when syncing recurring events.
+    # Do not store events older than this when syncing recurring events.
     # Many icalendar feeds are misconfigured and this prevents enumerating 2000+ years of recurrence.
-    setting :oldest_recurring_event, "1990-01-01", convert: ->(s) { Date.parse(s) }
-    # Sync icalendar calendars only this often.
-    # Most services only update every day or so. Assume it takes 5s to sync each feed (request, parse, upsert).
-    # If you have 10,000 feeds, that is 50,000 seconds, or almost 14 hours of processing time,
-    # or two threads for 7 hours. The resyncs are spread out across the sync period
-    # (ie, no thundering herd every 8 hours), but it is still a good idea to sync as infrequently as possible.
+    setting :oldest_recurring_event, "2000-01-01", convert: ->(s) { Date.parse(s) }
+    # Calendars feeds are considered 'fresh' if they have been synced this long ago or less.
+    # Most services only update every day or so.
+    # Assume it takes 5s to sync each feed (request, parse, upsert).
+    # If you have 10,000 feeds, that is 50,000 seconds,
+    # or almost 14 hours of processing time, or two threads for 7 hours.
     setting :sync_period_hours, 6
+    # When stale feeds are scheduled for a resync,
+    # 'smear' them along this duration. Using 0 would immediately enqueue syncs of all stale feeds,
+    # which could saturate the job server. The number here means that feeds will be refreshed between every
+    # +sync_period_hours+ and +sync_period_hours+ + +sync_period_splay_hours+.
+    setting :sync_period_splay_hours, 1
+    # Number of threads for the 'precheck' threadpool, used when enqueing icalendar sync jobs.
+    # Since the precheck process uses many threads, but each check is resource-light and not latency-sensitive,
+    # we use a shared threadpool for it.
+    setting :precheck_feed_change_pool_size, 12
+
+    # Cancelled events that were last updated this long ago are deleted from the database.
+    setting :stale_cancelled_event_threshold_days, 20
+    # The stale row deleter job will look for rows this far before the threshold.
+    setting :stale_cancelled_event_lookback_days, 3
 
-    # Cancelled events that were cancelled this long ago are deleted from the database.
-    setting :stale_cancelled_event_threshold_days, 35
+    # The URL of the icalproxy server, if using one.
+    # See https://github.com/webhookdb/icalproxy for more info.
+    # Used to get property HTTP semantics for any icalendar feed, like Etag and HEAD requests.
+    setting :proxy_url, ""
+    # Api key of the icalproxy server, if using one.
+    # See https://github.com/webhookdb/icalproxy
+    setting :proxy_api_key, ""
   end
 end

data/lib/webhookdb/jobs/amigo_test_jobs.rb

@@ -103,7 +103,7 @@ class Webhookdb::Jobs::RetryChecker
       when "die"
         raise Amigo::Retry::Die
     else
-        raise Amigo::Retry::Die.new(attempts, interval)
+        raise Amigo::Retry::OrDie.new(attempts, interval)
     end
   end
 end

data/lib/webhookdb/jobs/backfill.rb

@@ -13,40 +13,36 @@ class Webhookdb::Jobs::Backfill
   on "webhookdb.backfilljob.run"
   sidekiq_options queue: "netout"
 
-  def dependent_queues
-    # This is really the lowest-priority job so always defer to other queues.
-    return super
-  end
+  # This is really the lowest-priority job so always defer to other queues.
 
   def _perform(event)
     begin
       bfjob = self.lookup_model(Webhookdb::BackfillJob, event.payload)
     rescue RuntimeError => e
-      self.logger.info "skipping_missing_backfill_job", error: e
+      self.set_job_tags(result: "skipped_missing_backfill_job", exception: e)
       return
     end
     sint = bfjob.service_integration
     bflock = bfjob.ensure_service_integration_lock
-    self.with_log_tags(sint.log_tags.merge(backfill_job_id: bfjob.opaque_id)) do
-      sint.db.transaction do
-        unless bflock.lock?
-          self.logger.info "skipping_locked_backfill_job"
-          bfjob.update(finished_at: Time.now)
-          break
-        end
-        bfjob.refresh
-        if bfjob.finished?
-          self.logger.info "skipping_finished_backfill_job"
-          break
-        end
-        begin
-          sint.replicator.backfill(bfjob)
-        rescue Webhookdb::Replicator::CredentialsMissing
-          # The credentials could have been cleared out, so just finish this job.
-          self.logger.info "skipping_backfill_job_without_credentials"
-          bfjob.update(finished_at: Time.now)
-          break
-        end
+    self.set_job_tags(sint.log_tags.merge(backfill_job_id: bfjob.opaque_id))
+    sint.db.transaction do
+      unless bflock.lock?
+        self.set_job_tags(result: "skipped_locked_backfill_job")
+        bfjob.update(finished_at: Time.now)
+        break
+      end
+      bfjob.refresh
+      if bfjob.finished?
+        self.set_job_tags(result: "skipped_finished_backfill_job")
+        break
+      end
+      begin
+        sint.replicator.backfill(bfjob)
+      rescue Webhookdb::Replicator::CredentialsMissing
+        # The credentials could have been cleared out, so just finish this job.
+        self.set_job_tags(result: "skipped_backfill_job_without_credentials")
+        bfjob.update(finished_at: Time.now)
+        break
       end
     end
   end

data/lib/webhookdb/jobs/create_mirror_table.rb

@@ -10,9 +10,8 @@ class Webhookdb::Jobs::CreateMirrorTable
 
   def _perform(event)
     sint = self.lookup_model(Webhookdb::ServiceIntegration, event)
-    self.with_log_tags(sint.log_tags) do
-      svc = Webhookdb::Replicator.create(sint)
-      svc.create_table(if_not_exists: true)
-    end
+    self.set_job_tags(sint.log_tags)
+    svc = Webhookdb::Replicator.create(sint)
+    svc.create_table(if_not_exists: true)
   end
 end