webhookdb 1.4.0 → 1.5.0

data/lib/webhookdb/replicator/base.rb

@@ -62,7 +62,7 @@ class Webhookdb::Replicator::Base
   # and the arguments used to upsert it (arguments to upsert_webhook),
   # and should return the body string to respond back with.
   #
-  # @param [Hash] upserted
+  # @param [Hash,Array] upserted
   # @param [Webhookdb::Replicator::WebhookRequest] request
   # @return [String]
   def synchronous_processing_response_body(upserted:, request:)
@@ -318,10 +318,12 @@ for information on how to refresh data.)
   # Find a dependent service integration with the given service name.
   # If none are found, return nil. If multiple are found, raise,
   # as this should only be used for automatically managed integrations.
+  # @param service_name [String,Array<String>]
   # @return [Webhookdb::ServiceIntegration,nil]
   def find_dependent(service_name)
-    sints = self.service_integration.dependents.filter { |si| si.service_name == service_name }
-    raise Webhookdb::InvalidPrecondition, "there are multiple #{service_name} integrations in dependents" if
+    names = service_name.respond_to?(:to_ary) ? service_name : [service_name]
+    sints = self.service_integration.dependents.filter { |si| names.include?(si.service_name) }
+    raise Webhookdb::InvalidPrecondition, "there are multiple #{names.join('/')} integrations in dependents" if
       sints.length > 1
     return sints.first
   end
@@ -356,7 +358,9 @@ for information on how to refresh data.)
     columns << self.data_column
     adapter = Webhookdb::DBAdapter::PG.new
     result = Webhookdb::Replicator::SchemaModification.new
-    result.transaction_statements << adapter.create_table_sql(table, columns, if_not_exists:)
+    create_table = adapter.create_table_sql(table, columns, if_not_exists:, partition: self.partitioning)
+    result.transaction_statements << create_table
+    result.transaction_statements.concat(self.create_table_partitions(adapter))
     self.indices(table).each do |dbindex|
       result.transaction_statements << adapter.create_index_sql(dbindex, concurrently: false)
     end
@@ -364,33 +368,69 @@ for information on how to refresh data.)
     return result
   end
 
+  # True if the replicator uses partitioning.
+  def partition? = false
+  # Non-nil only if +partition?+ is true.
+  # @return [Webhookdb::DBAdapter::Partitioning,nil]
+  def partitioning = nil
+
+  # Return the partitions belonging to the table.
+  # Return an empty array if this replicator is not partitioned.
+  # @return [Array<Webhookdb::DBAdapter::Partition>]
+  def existing_partitions(_db)
+    raise NotImplementedError if self.partition?
+    return []
+  end
+
+  def create_table_partitions(adapter)
+    return [] unless self.partition?
+    # We only need create_table partitions when we create the table.
+    # Range partitions would be created on demand, when inserting rows and the partition doesn't exist.
+    return [] unless self.partitioning.by == Webhookdb::DBAdapter::Partitioning::HASH
+
+    max_partition = self.service_integration.partition_value
+    raise Webhookdb::InvalidPrecondition, "partition value must be positive" unless max_partition.positive?
+    stmts = (0...max_partition).map do |i|
+      adapter.create_hash_partition_sql(self.dbadapter_table, max_partition, i)
+    end
+    return stmts
+  end
+
   # We need to give indices a persistent name, unique across the schema,
   # since multiple indices within a schema cannot share a name.
   #
   # Note that in certain RDBMS (Postgres) index names cannot exceed a certian length;
   # Postgres will silently truncate them. This can result in an index not being created
-  # if it shares the same name as another index and we use 'CREATE INDEX IF NOT EXISTS.'
+  # if it shares the same name as another index, and we use 'CREATE INDEX IF NOT EXISTS.'
   #
   # To avoid this, if the generated name exceeds a certain size, an md5 hash of the column names is used.
   #
   # @param columns [Array<Webhookdb::DBAdapter::Column, Webhookdb::Replicator::Column>] Must respond to :name.
+  # @param identifier [String,nil] Use this instead of a combination of column names.
+  #   Only use this where multiple indices are needed for the same columns, but something like the 'where'
+  #   condition is different.
   # @return [String]
-  protected def index_name(columns)
+  protected def index_name(columns, identifier: nil)
     raise Webhookdb::InvalidPrecondition, "sint needs an opaque id" if self.service_integration.opaque_id.blank?
     colnames = columns.map(&:name).join("_")
     opaque_id = self.service_integration.opaque_id
     # Handle old IDs without the leading 'svi_'.
     opaque_id = "idx#{opaque_id}" if /\d/.match?(opaque_id[0])
-    name = "#{opaque_id}_#{colnames}_idx"
-    if name.size > MAX_INDEX_NAME_LENGTH
-      # We don't have the 32 extra chars for a full md5 hash.
-      # We can't convert to Base64 or whatever, since we don't want to depend on case sensitivity.
-      # So just lop off a few characters (normally 2) from the end of the md5.
-      # The collision space is so small (some combination of column names would need to have the
-      # same md5, which is unfathomable), we're not really worried about it.
-      colnames_md5 = Digest::MD5.hexdigest(colnames)
-      available_chars = MAX_INDEX_NAME_LENGTH - "#{opaque_id}__idx".size
-      name = "#{opaque_id}_#{colnames_md5[...available_chars]}_idx"
+
+    if identifier
+      name = "#{opaque_id}_#{identifier}_idx"
+    else
+      name = "#{opaque_id}_#{colnames}_idx"
+      if name.size > MAX_INDEX_NAME_LENGTH
+        # We don't have the 32 extra chars for a full md5 hash.
+        # We can't convert to Base64 or whatever, since we don't want to depend on case sensitivity.
+        # So just lop off a few characters (normally 2) from the end of the md5.
+        # The collision space is so small (some combination of column names would need to have the
+        # same md5, which is unfathomable), we're not really worried about it.
+        colnames_md5 = Digest::MD5.hexdigest(colnames)
+        available_chars = MAX_INDEX_NAME_LENGTH - "#{opaque_id}__idx".size
+        name = "#{opaque_id}_#{colnames_md5[...available_chars]}_idx"
+      end
     end
     raise Webhookdb::InvariantViolation, "index names cannot exceed 63 chars, got #{name.size} in '#{name}'" if
       name.size > 63
@@ -406,7 +446,12 @@ for information on how to refresh data.)
 
   # @return [Webhookdb::DBAdapter::Column]
   def remote_key_column
-    return self._remote_key_column.to_dbadapter(unique: true, nullable: false)
+    c = self._remote_key_column
+    if c.index?
+      msg = "_remote_key_column index:true should not be set, since it automatically gets a unique index"
+      Kernel.warn msg
+    end
+    return c.to_dbadapter(unique: true, nullable: false, index: false)
   end
 
   # @return [Webhookdb::DBAdapter::Column]
@@ -465,6 +510,9 @@ for information on how to refresh data.)
   # Each integration needs a single remote key, like the Shopify order id for shopify orders,
   # or sid for Twilio resources. This column must be unique for the table, like a primary key.
   #
+  # NOTE: Do not set index:true. The remote key column always must be unique,
+  # so it gets a unique index automatically.
+  #
   # @abstract
   # @return [Webhookdb::Replicator::Column]
   def _remote_key_column
@@ -495,9 +543,16 @@ for information on how to refresh data.)
     end
     self._extra_index_specs.each do |spec|
       targets = spec.columns.map { |n| dba_cols_by_name.fetch(n) }
-      idx_name = self.index_name(targets)
+      idx_name = self.index_name(targets, identifier: spec.identifier)
       result << Webhookdb::DBAdapter::Index.new(name: idx_name.to_sym, table:, targets:, where: spec.where)
     end
+    index_names = result.map(&:name)
+    if (dupes = index_names.find_all.with_index { |n, idx| idx != index_names.rindex(n) }).any?
+      msg = "Duplicate index names detected. Use the 'name' attribute to differentiate: " +
+        dupes.map(&:to_s).join(", ")
+      raise Webhookdb::Replicator::BrokenSpecification, msg
+    end
+
     return result
   end
 
@@ -520,7 +575,7 @@ for information on how to refresh data.)
 
   # @return [Webhookdb::Replicator::SchemaModification]
   def ensure_all_columns_modification
-    existing_cols, existing_indices = nil
+    existing_cols, existing_indices, existing_partitions = nil
     max_pk = 0
     sint = self.service_integration
     self.admin_dataset do |ds|
@@ -531,6 +586,7 @@ for information on how to refresh data.)
         tablename: sint.table_name,
       ).select_map(:indexname).to_set
       max_pk = ds.max(:pk) || 0
+      existing_partitions = self.existing_partitions(ds.db)
     end
     adapter = Webhookdb::DBAdapter::PG.new
     table = self.dbadapter_table
@@ -577,7 +633,9 @@ for information on how to refresh data.)
     # Add missing indices. This should happen AFTER the UPDATE calls so the UPDATEs don't have to update indices.
     self.indices(table).map do |index|
       next if existing_indices.include?(index.name.to_s)
-      result.nontransaction_statements << adapter.create_index_sql(index, concurrently: true)
+      result.nontransaction_statements.concat(
+        adapter.create_index_sqls(index, concurrently: true, partitions: existing_partitions),
+      )
     end
 
     result.application_database_statements << sint.ensure_sequence_sql if self.requires_sequence?
@@ -641,6 +699,7 @@ for information on how to refresh data.)
   # like when we have to take different action based on a request method.
   #
   # @param body [Hash]
+  # @return [Array,Hash] Inserted rows, or array of inserted rows if many.
   def upsert_webhook_body(body, **kw)
     return self.upsert_webhook(Webhookdb::Replicator::WebhookRequest.new(body:), **kw)
   end
@@ -649,13 +708,14 @@ for information on how to refresh data.)
   # NOT a Rack::Request.
   #
   # @param [Webhookdb::Replicator::WebhookRequest] request
+  # @return [Array,Hash] Inserted rows, or array of inserted rows if many.
   def upsert_webhook(request, **kw)
     return self._upsert_webhook(request, **kw)
   rescue Amigo::Retry::Error
     # Do not log this since it's expected/handled by Amigo
     raise
   rescue StandardError => e
-    self.logger.error("upsert_webhook_error", request: request.as_json, error: e)
+    self.logger.error("upsert_webhook_error", {request: request.as_json}, e)
     raise
   end
 
@@ -664,9 +724,23 @@ for information on how to refresh data.)
   #
   # @param request [Webhookdb::Replicator::WebhookRequest]
   # @param upsert [Boolean] If false, just return what would be upserted.
+  # @return [Array,Hash] Inserted rows, or array of inserted rows if many.
   def _upsert_webhook(request, upsert: true)
-    resource, event = self._resource_and_event(request)
-    return nil if resource.nil?
+    resource_or_list, event = self._resource_and_event(request)
+    return nil if resource_or_list.nil?
+    if resource_or_list.is_a?(Array)
+      unless event.nil?
+        msg = "resource_and_event cannot return an array of resources with a non-nil event"
+        raise Webhookdb::InvalidPostcondition, msg
+      end
+      return resource_or_list.map do |resource|
+        self._upsert_webhook_single_resource(request, resource:, event:, upsert:)
+      end
+    end
+    return self._upsert_webhook_single_resource(request, resource: resource_or_list, event:, upsert:)
+  end
+
+  def _upsert_webhook_single_resource(request, resource:, event:, upsert:)
     enrichment = self._fetch_enrichment(resource, event, request)
     prepared = self._prepare_for_insert(resource, event, request, enrichment)
     raise Webhookdb::InvalidPostcondition if prepared.key?(:data)
@@ -676,12 +750,11 @@ for information on how to refresh data.)
     inserting[:enrichment] = self._to_json(enrichment) if self._store_enrichment_body?
     inserting.merge!(prepared)
     return inserting unless upsert
-    remote_key_col = self._remote_key_column
     updating = self._upsert_update_expr(inserting, enrichment:)
     update_where = self._update_where_expr
     upserted_rows = self.admin_dataset(timeout: :fast) do |ds|
       ds.insert_conflict(
-        target: remote_key_col.name,
+        target: self._upsert_conflict_target,
         update: updating,
         update_where:,
       ).insert(inserting)
@@ -692,6 +765,12 @@ for information on how to refresh data.)
     return inserting
   end
 
+  # The target for ON CONFLICT. Usually the remote key column name,
+  # except if the remote id is a compound unique index, like for partitioned tables.
+  # Can be a symbol, array of symbols representing the column names, a +Sequel.lit+, etc.
+  # See +Sequel::Dataset.insert_conflict+ :target option for details.
+  def _upsert_conflict_target = self._remote_key_column.name
+
   # The NULL ASCII character (\u0000), when present in a string ("\u0000"),
   # and then encoded into JSON ("\\u0000") is invalid in PG JSONB- its strings cannot contain NULLs
   # (note that JSONB does not store the encoded string verbatim, it parses it into PG types, and a PG string
@@ -796,7 +875,7 @@ for information on how to refresh data.)
   #
   # @abstract
   # @param [Webhookdb::Replicator::WebhookRequest] request
-  # @return [Array<Hash>,nil]
+  # @return [Array<Hash,Array>,nil]
   def _resource_and_event(request)
     raise NotImplementedError
   end
@@ -906,10 +985,10 @@ for information on how to refresh data.)
   # - The table OID for this replicator
   # - The given key
   #
-  # Note this this establishes a new DB connection for the advisory lock;
+  # Note this establishes a new DB connection for the advisory lock;
   # we have had issues with advisory locks on reused connections,
   # and this is safer than having a lock that is never released.
-  protected def with_advisory_lock(key, &)
+  def with_advisory_lock(key, &)
     url = self.service_integration.organization.admin_connection_url_raw
     got = nil
     Webhookdb::Dbutil.borrow_conn(url) do |conn|
@@ -969,7 +1048,7 @@ for information on how to refresh data.)
     rescue TypeError, NoMethodError => e
       # if we don't incur an HTTP error, but do incur an Error due to differences in the shapes of anticipated
       # response data in the `fetch_backfill_page` function, we can assume that the credentials are okay
-      self.logger.info "verify_backfill_credentials_expected_failure", error: e
+      self.logger.info "verify_backfill_credentials_expected_failure", e
       return CredentialVerificationResult.new(verified: true, message: "")
     end
     return CredentialVerificationResult.new(verified: true, message: "")
@@ -1190,6 +1269,34 @@ or leave blank to choose the first option.
     return self._webhook_endpoint
   end
 
+  # Avoid writes under the following conditions:
+  #
+  # - A table lock is taken on the table
+  # - A vacuum is in progress on the table
+  #
+  # Of course, in most situations we want to write anyway,
+  # but there are some cases (lower-priority replicators for example)
+  # where we can reschedule the job to happen in the future instead.
+  def avoid_writes?
+    # We will need to handle this differently when not under Postgres, but for now,
+    # just assume Postgres.
+    # Find the admin URL for the organization's server (NOT the organization admin url, it can't see system processes).
+    # Then check for 1) vacuums in progress, 2) locks.
+    self.service_integration.organization.readonly_connection do |db|
+      count = db[:pg_locks].
+        join(:pg_class, {oid: :relation}).
+        join(:pg_namespace, {oid: :relnamespace}).
+        where(
+          locktype: "relation",
+          nspname: self.service_integration.organization.replication_schema,
+          relname: self.service_integration.table_name,
+          mode: ["ShareUpdateExclusiveLock", "ExclusiveLock", "AccessExclusiveLock"],
+        ).limit(1).count
+      return true if count&.positive?
+    end
+    return false
+  end
+
   protected def _webhook_endpoint
     return self.service_integration.unauthed_webhook_endpoint
   end

data/lib/webhookdb/replicator/base_stale_row_deleter.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+# Delete stale rows (like cancelled calendar events) not updated (row_updated_at or whatever column)
+# in the window between +stale_at+ back to +lookback_window+.
+# This avoids endlessly adding to a table where we expect rows to become stale over time.
+class Webhookdb::Replicator::BaseStaleRowDeleter
+  # @return [Webhookdb::Replicator::Base]
+  attr_reader :replicator
+
+  def initialize(replicator)
+    @replicator = replicator
+  end
+
+  # When a row is considered 'stale'.
+  # For example, a value of +35.days+ would treat any row older than 35 days as stale.
+  # @return [ActiveSupport::Duration]
+  def stale_at
+    raise NotImplementedError
+  end
+
+  # How far from +stale_at+ to "look back" for stale rows.
+  # We cannot just use "row_updated_at < stale_at" since this would scan ALL the rows
+  # every time we delete rows. Instead, we only want to scale rows where
+  # "row_updated_at < stale_at AND row_updated_at > (stale_at - lookback_window)".
+  # For example, a +stale_at+ of 20 days and a +lookback_window+ of 7 days
+  # would look to delete rows 20 to 27 days old.
+  #
+  # If the stale row deleter is run daily, a good lookback window would be 2-3 days,
+  # since as long as the job is running we shouldn't find rows that aren't cleaned up.
+  #
+  # Use +run_initial+ to do a full table scan,
+  # which may be necessary when running this feature for a table for the first time.
+  # @return [ActiveSupport::Duration]
+  def lookback_window
+    raise NotImplementedError
+  end
+
+  # Name of the column, like +:row_updated_at+.
+  # @return [Symbol]
+  def updated_at_column
+    raise NotImplementedError
+  end
+
+  # Other additional 'stale' conditions, like {status: 'cancelled'}
+  # @return [Hash]
+  def stale_condition
+    raise NotImplementedError
+  end
+
+  # The row delete is done in chunks to avoid long locks.
+  # The default seems safe, but it's exposed if you need to play around with it,
+  # and can be done via configuration if needed at some point.
+  # @return [Integer]
+  def chunk_size = 10_000
+
+  # How small should the incremental lookback window be? See +run+ for details.
+  # A size of 1 hour, and a lookback window of 2 days, would yield at least 48 delete queries.
+  def incremental_lookback_size = 1.hour
+
+  # Run the deleter.
+  # @param lookback_window [nil,ActiveSupport::Duration] The lookback window
+  #   (how many days before +stale_cutoff+ to look for rows). Use +nil+ to look for all rows.
+  def run(lookback_window: self.lookback_window)
+    # The algorithm to delete stale rows is complex for a couple of reasons.
+    # The native solution is "delete rows where updated_at > (stale_at - lookback_window) AND updated_at < stale_at"
+    # However, this would cause a single massive query over the entire candidate row space,
+    # which has problems:
+    # - The query can be very slow
+    # - Deadlocks can happen due to the slow query.
+    # - If the query is interrupted (due to a worker restart), all progress is lost.
+    # - Scanning the large 'updated at timestamp' index can cause the database to do a sequential scan.
+    #
+    # Instead, we need to do issue a series of fast queries over small 'updated at' windows:
+    #
+    # - Break the lookback period into hour-long windows.
+    #   If the lookback_window is 2 days, this would issue 48 queries.
+    #   But each one would be very fast, since the column is indexed.
+    # - For each small window, delete in chunks, like:
+    #      DELETE from "public"."icalendar_event_v1_aaaa"
+    #      WHERE pk IN (
+    #        SELECT pk FROM "public"."icalendar_event_v1_aaaa"
+    #        WHERE row_updated_at >= (hour start)
+    #        AND row_updated_at < (hour end)
+    #        LIMIT (chunk size)
+    #      )
+    # - Issue each DELETE within a transaction with seqscan disabled.
+    #   This is crude, but we know for our usage case that we never want a seqscan.
+    # - Using the chunked delete with the hour-long (small-sized) windows
+    #   is important. Because each chunk requires scanning potentially the entire indexed row space,
+    #   it would take longer and longer to find 10k rows to fill the chunk.
+    #   This is, for example, the same performance problem that OFFSET/LIMIT pagination
+    #   has at later pages (but not earlier pages).
+    self.replicator.admin_dataset do |ds|
+      stale_window_late = Time.now - self.stale_at
+      stale_window_early = lookback_window.nil? ? ds.min(self.updated_at_column) : stale_window_late - lookback_window
+      # If we are querying the whole table (no lookback window), and have no rows,
+      # there's nothing to clean up.
+      break if stale_window_early.nil?
+
+      # We must disable vacuuming for this sort of cleanup.
+      # Otherwise, it will take a LONG time since we use a series of short deletes.
+      self.set_autovacuum(ds.db, false)
+      if self.replicator.partition?
+        # If the replicator is partitioned, we need to delete stale rows on partition separately.
+        # We DELETE with a LIMIT in chunks, but when we run this on the main table, it'll run the query
+        # on every partition BEFORE applying the limit. You'll see this manifest with speed,
+        # but also the planner using a sequential scan for the delete, rather than hitting an index.
+        # Instead, DELETE from each partition in chunks, which will use the indices, and apply the limit properly.
+        self.replicator.existing_partitions(ds.db).each do |p|
+          pdb = ds.db[self.replicator.qualified_table_sequel_identifier(table: p.partition_name)]
+          self._run_delete(pdb, stale_window_early:, stale_window_late:)
+        end
+      else
+        self._run_delete(ds, stale_window_early:, stale_window_late:)
+      end
+    end
+  ensure
+    # Open a new connection in case the previous one is trashed for whatever reason.
+    self.replicator.admin_dataset do |ds|
+      self.set_autovacuum(ds.db, true)
+    end
+  end
+
+  def _run_delete(ds, stale_window_early:, stale_window_late:)
+    base_ds = ds.where(self.stale_condition).limit(self.chunk_size).select(:pk)
+    window_start = stale_window_early
+    until window_start >= stale_window_late
+      window_end = window_start + self.incremental_lookback_size
+      inner_ds = base_ds.where(self.updated_at_column => window_start..window_end)
+      loop do
+        # Due to conflicts where a feed is being inserted while the delete is happening,
+        # this may raise an error like:
+        #   deadlock detected
+        #   DETAIL:  Process 18352 waits for ShareLock on transaction 435085606; blocked by process 24191.
+        #   Process 24191 waits for ShareLock on transaction 435085589; blocked by process 18352.
+        #   HINT:  See server log for query details.
+        #   CONTEXT:  while deleting tuple (2119119,3) in relation "icalendar_event_v1_aaaa"
+        # So we don't explicitly handle deadlocks, but could if it becomes an issue.
+        delete_ds = ds.where(pk: inner_ds)
+        # Disable seqscan for the delete. We can end up with seqscans if the planner decides
+        # it's a better choice given the 'updated at' index, but for our purposes we know
+        # we never want to use it (the impact is negligible on small tables,
+        # and catastrophic on large tables).
+        sql_lines = [
+          "BEGIN",
+          "SET LOCAL enable_seqscan='off'",
+          delete_ds.delete_sql,
+          "COMMIT",
+        ]
+        deleted = ds.db << sql_lines.join(";\n")
+        break if deleted != self.chunk_size
+      end
+      window_start = window_end
+    end
+  end
+
+  def set_autovacuum(db, on)
+    return if self.replicator.partition?
+    arg = on ? "on" : "off"
+    db << "ALTER TABLE #{self.replicator.schema_and_table_symbols.join('.')} SET (autovacuum_enabled='#{arg}')"
+  end
+
+  # Run with +lookback_window+ as +nil+, which does a full table scan.
+  def run_initial = self.run(lookback_window: nil)
+end

data/lib/webhookdb/replicator/email_octopus_contact_v1.rb

@@ -29,7 +29,6 @@ class Webhookdb::Replicator::EmailOctopusContactV1 < Webhookdb::Replicator::Base
       :compound_identity,
       TEXT,
       data_key: "<compound key, see converter>",
-      index: true,
       optional: true,
       converter: CONV_REMOTE_KEY,
     )