webhookdb 1.3.1 → 1.5.0

data/lib/webhookdb/sync_target.rb

@@ -3,6 +3,9 @@
 require "sequel/advisory_lock"
 require "sequel/database"
 
+require "webhookdb/concurrent"
+require "webhookdb/jobs/sync_target_run_sync"
+
 # Support exporting WebhookDB data into external services,
 # such as another Postgres instance or data warehouse (Snowflake, etc).
 #
@@ -21,9 +24,9 @@ class Webhookdb::SyncTarget < Webhookdb::Postgres::Model(:sync_targets)
   include Appydays::Configurable
   include Webhookdb::Dbutil
 
-  class Deleted < StandardError; end
-  class InvalidConnection < StandardError; end
-  class SyncInProgress < StandardError; end
+  class Deleted < Webhookdb::WebhookdbError; end
+  class InvalidConnection < Webhookdb::WebhookdbError; end
+  class SyncInProgress < Webhookdb::WebhookdbError; end
 
   # Advisory locks for sync targets use this as the first int, and the id as the second.
   ADVISORY_LOCK_KEYSPACE = 2_000_000_000
@@ -32,6 +35,7 @@ class Webhookdb::SyncTarget < Webhookdb::Postgres::Model(:sync_targets)
   DB_VERIFY_TIMEOUT = 2000
   DB_VERIFY_STATEMENT = "SELECT 1"
   RAND = Random.new
+  MAX_STATS = 200
 
   configurable(:sync_target) do
     # Allow installs to set this much lower if they want a faster sync.
@@ -52,6 +56,12 @@ class Webhookdb::SyncTarget < Webhookdb::Postgres::Model(:sync_targets)
     # we must allow sync targets to use http urls. This should only
     # be used internally, and never in production.
     setting :allow_http, false
+    # Syncing may require serverside cursors, which open a transaction.
+    # To avoid long-lived transactions, any sync which has a transaction,
+    # and goes on longer than +max_transaction_seconds+,
+    # will 'soft abort' the sync and reschedule itself to continue
+    # using a new transaction.
+    setting :max_transaction_seconds, 10.minutes.to_i
 
     after_configured do
       if Webhookdb::RACK_ENV == "test"
@@ -87,8 +97,10 @@ class Webhookdb::SyncTarget < Webhookdb::Postgres::Model(:sync_targets)
   dataset_module do
     def due_for_sync(as_of:)
       never_synced = Sequel[last_synced_at: nil]
-      next_due_at = Sequel[:last_synced_at] + (Sequel.lit("INTERVAL '1 second'") * Sequel[:period_seconds])
-      due_before_now = next_due_at <= as_of
+      # Use 'last_synced_at <= (now - internal)' rather than 'last_synced_at + interval <= now'
+      # so we can use the last_synced_at index.
+      cutoff = (Sequel[as_of].cast("TIMESTAMPTZ") - (Sequel.lit("INTERVAL '1 second'") * Sequel[:period_seconds]))
+      due_before_now = Sequel[:last_synced_at] <= cutoff
       return self.where(never_synced | due_before_now)
     end
   end
@@ -168,13 +180,29 @@ class Webhookdb::SyncTarget < Webhookdb::Postgres::Model(:sync_targets)
         timeout: HTTP_VERIFY_TIMEOUT,
         follow_redirects: true,
       )
-    rescue Timeout::Error => e
-      raise InvalidConnection, "POST to #{cleanurl} timed out: #{e.message}"
-    rescue Webhookdb::Http::Error => e
-      raise InvalidConnection, "POST to #{cleanurl} failed: #{e.message}"
+    rescue StandardError => e
+      raise InvalidConnection, "POST to #{cleanurl} failed: #{e.message}" if
+        e.is_a?(Webhookdb::Http::Error) || self.transport_error?(e)
+      raise
     end
   end
 
+  # Return true if the given error is considered a 'transport' error,
+  # like a timeout, socket error, dns error, etc.
+  # This isn't a consistent class type.
+  def self.transport_error?(e)
+    return true if e.is_a?(Timeout::Error)
+    return true if e.is_a?(SocketError)
+    return true if e.is_a?(OpenSSL::SSL::SSLError)
+    # SystemCallError are Errno errors, we can get them when the url no longer resolves.
+    return true if e.is_a?(SystemCallError)
+    # Socket::ResolutionError is an error but I guess it's defined in C and we can't raise it in tests.
+    # Anything with an error_code assume is some transport-level issue and treat it as a connection issue,
+    # not a coding issue.
+    return true if e.respond_to?(:error_code)
+    return false
+  end
+
   def next_scheduled_sync(now:)
     return self.next_sync(self.period_seconds, now)
   end
@@ -200,6 +228,13 @@ class Webhookdb::SyncTarget < Webhookdb::Postgres::Model(:sync_targets)
     return RAND.rand(1..max_jitter)
   end
 
+  # @return [ActiveSupport::Duration,Integer]
+  def latency(now: Time.now)
+    return 0 if self.last_synced_at.nil?
+    return 0 if self.last_synced_at > now
+    return now - self.last_synced_at
+  end
+
   # Running a sync involves some work we always do (export, transform),
   # and then work that varies per-adapter (load).
   #
@@ -239,6 +274,7 @@ class Webhookdb::SyncTarget < Webhookdb::Postgres::Model(:sync_targets)
     # since the session will be ended.
     Webhookdb::Dbutil.borrow_conn(Webhookdb::Postgres::Model.uri) do |db|
       self.advisory_lock(db).with_lock? do
+        self.logger.info "starting_sync"
         routine = if self.connection_url.start_with?("https://", "http://")
                     # Note that http links are not secure and should only be used for development purposes
                     HttpRoutine.new(now, self)
@@ -261,6 +297,16 @@ class Webhookdb::SyncTarget < Webhookdb::Postgres::Model(:sync_targets)
     return displaysafe_url(self.connection_url)
   end
 
+  def log_tags
+    return {
+      sync_target_id: self.id,
+      sync_target_connection_url: self.displaysafe_connection_url,
+      service_integration_id: self.service_integration_id,
+      service_integration_service: self.service_integration.service_name,
+      service_integration_table: self.service_integration.table_name,
+    }
+  end
+
   # @return [String]
   def associated_type
     # Eventually we need to support orgs
@@ -284,6 +330,39 @@ class Webhookdb::SyncTarget < Webhookdb::Postgres::Model(:sync_targets)
     return "#{schema_name}.#{table_name}"
   end
 
+  # :section: Stats
+
+  def add_sync_stat(start, exception: nil, response_status: nil)
+    stat = {"t" => s2ms(start), "d" => s2ms(Time.now - start)}
+    stat["e"] = exception.class.name if exception
+    stat["rs"] = response_status unless response_status.nil?
+    stats = self.sync_stats
+    stats.prepend(stat)
+    stats.pop if stats.size > MAX_STATS
+    self.will_change_column(:sync_stats)
+  end
+
+  protected def s2ms(t) = (t.to_f * 1000).to_i
+  protected def ms2s(ms) = ms / 1000.0
+
+  def sync_stat_summary
+    return {} if self.sync_stats.empty?
+    earliest = self.sync_stats.last
+    latest = self.sync_stats.first
+    average_latency = (self.sync_stats.sum { |st| ms2s(st["d"]) }) / self.sync_stats.size
+    errors = self.sync_stats.count { |st| st["e"] || st["rs"] }
+    calls_per_minute = 60 / average_latency
+    rpm = self.page_size * calls_per_minute
+    rpm *= self.parallelism if self.parallelism.positive?
+    return {
+      latest: Time.at(ms2s(latest["t"]).to_i),
+      earliest: Time.at(ms2s(earliest["t"]).to_i),
+      average_latency: average_latency.round(2),
+      average_rows_minute: rpm.to_i,
+      errors:,
+    }
+  end
+
   # @return [Webhookdb::Organization]
   def organization
     return self.service_integration.organization
@@ -341,40 +420,112 @@ class Webhookdb::SyncTarget < Webhookdb::Postgres::Model(:sync_targets)
       end
     end
 
-    def record(last_synced_at)
-      self.sync_target.update(last_synced_at:)
+    def perform_db_op(&)
+      yield
     rescue Sequel::NoExistingObject => e
       raise Webhookdb::SyncTarget::Deleted, e
     end
+
+    def record(last_synced_at)
+      self.perform_db_op do
+        self.sync_target.update(last_synced_at:)
+      end
+    end
+
+    def with_stat(&)
+      start = Time.now
+      begin
+        yield
+        self.sync_target.add_sync_stat(start)
+      rescue Webhookdb::Http::Error => e
+        self.sync_target.add_sync_stat(start, response_status: e.status)
+        raise
+      rescue StandardError => e
+        self.sync_target.add_sync_stat(start, exception: e)
+        raise
+      end
+    end
+
+    def to_ms(t)
+      return (t.to_f * 1000).to_i
+    end
   end
 
   class HttpRoutine < Routine
+    def initialize(*)
+      super
+      @inflight_timestamps = []
+      @cleanurl, @authparams = Webhookdb::Http.extract_url_auth(self.sync_target.connection_url)
+      @threadpool = if self.sync_target.parallelism.zero?
+                      Webhookdb::Concurrent::SerialPool.new
+        else
+          Webhookdb::Concurrent::ParallelizedPool.new(self.sync_target.parallelism)
+      end
+      @mutex = Thread::Mutex.new
+    end
+
     def run
+      timeout_at = Time.now + Webhookdb::SyncTarget.max_transaction_seconds
       page_size = self.sync_target.page_size
+      sync_result = :complete
       self.dataset_to_sync do |ds|
         chunk = []
-        ds.paged_each(rows_per_fetch: page_size) do |row|
+        ds.paged_each(rows_per_fetch: page_size, cursor_name: "synctarget_#{self.sync_target.id}_cursor") do |row|
           chunk << row
-          self._flush_http_chunk(chunk) if chunk.size >= page_size
+          if chunk.size >= page_size
+            # Do not share chunks across threads
+            self._flush_http_chunk(chunk.dup)
+            chunk.clear
+            if Time.now >= timeout_at && Thread.current[:sidekiq_context]
+              # If we've hit the timeout, stop any further syncing
+              sync_result = :timeout
+              break
+            end
+          end
         end
         self._flush_http_chunk(chunk) unless chunk.empty?
-        # We should save 'now' as the timestamp, rather than the last updated row.
-        # This is important because other we'd keep trying to sync the last row synced.
-        self.record(self.now)
+        @threadpool.join
+        case sync_result
+          when :timeout
+            # If the sync timed out, use the last recorded sync timestamp,
+            # and re-enqueue the job, so the sync will pick up where it left off.
+            self.sync_target.logger.info("sync_target_transaction_timeout", self.sync_target.log_tags)
+            Webhookdb::Jobs::SyncTargetRunSync.perform_async(self.sync_target.id)
+          else
+            # The sync completed normally.
+            # Save 'now' as the timestamp, rather than the last updated row.
+            # This is important because other we'd keep trying to sync the last row synced.
+            self.record(self.now)
+        end
+      end
+    rescue Webhookdb::Concurrent::Timeout => e
+      # This should never really happen, but it does, so record it while we debug it.
+      self.perform_db_op do
+        self.sync_target.save_changes
       end
-    rescue Webhookdb::Http::Error, Errno::ECONNRESET, Net::ReadTimeout, Net::OpenTimeout, OpenSSL::SSL::SSLError => e
-      # This is handled well so no need to re-raise.
+      self.sync_target.logger.error("sync_target_pool_timeout_error", self.sync_target.log_tags, e)
+    rescue StandardError => e
+      # Errors talking to the http server are handled well so no need to re-raise.
       # We already committed the last page that was successful,
       # so we can just stop syncing at this point to try again later.
-
+      raise e unless e.is_a?(Webhookdb::Http::Error) || Webhookdb::SyncTarget.transport_error?(e)
+      self.perform_db_op do
+        # Save any outstanding stats.
+        self.sync_target.save_changes
+      end
       # Don't spam our logs with downstream errors
       idem_key = "sync_target_http_error-#{self.sync_target.id}-#{e.class.name}"
       Webhookdb::Idempotency.every(1.hour).in_memory.under_key(idem_key) do
-        self.sync_target.logger.warn("sync_target_http_error", error: e)
+        self.sync_target.logger.warn("sync_target_http_error", self.sync_target.log_tags, e)
       end
     end
 
     def _flush_http_chunk(chunk)
+      chunk_ts = chunk.last.fetch(self.replicator.timestamp_column.name)
+      @mutex.synchronize do
+        @inflight_timestamps << chunk_ts
+        @inflight_timestamps.sort!
+      end
       sint = self.sync_target.service_integration
       body = {
         rows: chunk,
@@ -383,19 +534,34 @@ class Webhookdb::SyncTarget < Webhookdb::Postgres::Model(:sync_targets)
         table: sint.table_name,
         sync_timestamp: self.now,
       }
-      cleanurl, authparams = Webhookdb::Http.extract_url_auth(self.sync_target.connection_url)
-      Webhookdb::Http.post(
-        cleanurl,
-        body,
-        timeout: sint.organization.sync_target_timeout,
-        logger: self.sync_target.logger,
-        basic_auth: authparams,
-      )
-      latest_ts = chunk.last.fetch(self.replicator.timestamp_column.name)
-      # The client committed the sync page we sent. Record it in case of a future error,
-      # so we don't re-send the same page.
-      self.record(latest_ts)
-      chunk.clear
+      @threadpool.post do
+        self.with_stat do
+          Webhookdb::Http.post(
+            @cleanurl,
+            body,
+            timeout: sint.organization.sync_target_timeout,
+            logger: self.sync_target.logger,
+            basic_auth: @authparams,
+          )
+        end
+        # On success, we want to commit the latest timestamp we sent to the client,
+        # so it can be recorded. Then in the case of an error on later rows,
+        # we won't re-sync rows we've already processed (with earlier updated timestamps).
+        @mutex.synchronize do
+          this_ts_idx = @inflight_timestamps.index { |t| t == chunk_ts }
+          raise Webhookdb::InvariantViolation, "timestamp no longer found!?" if this_ts_idx.nil?
+          # However, we only want to record the timestamp if this request is the earliest inflight request;
+          # ie, if a later request finishes before an earlier one, we don't want to record the timestamp
+          # of the later request as 'finished' since the earlier one didn't finish.
+          # This does mean though that, if the earliest request errors, we'll throw away the work
+          # done by the later request.
+          # Note that each row can only appear in a sync once, even if it is modified after the sync starts;
+          # thus, parallel httpsync should be fine for most clients to handle,
+          # since race conditions *on the same row* cannot happen even with parallel httpsync.
+          self.record(chunk_ts) if this_ts_idx.zero?
+          @inflight_timestamps.delete_at(this_ts_idx)
+        end
+      end
     end
   end
 
@@ -445,7 +611,9 @@ class Webhookdb::SyncTarget < Webhookdb::Postgres::Model(:sync_targets)
       schema_expr = schema_lines.join(";\n") + ";"
       if schema_expr != self.sync_target.last_applied_schema
         adapter_conn.execute(schema_expr)
-        self.sync_target.update(last_applied_schema: schema_expr)
+        self.perform_db_op do
+          self.sync_target.update(last_applied_schema: schema_expr)
+        end
       end
       tempfile = Tempfile.new("whdbsyncout-#{self.sync_target.id}")
       begin

data/lib/webhookdb/system_log_event.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require "webhookdb/postgres/model"
+
+class Webhookdb::SystemLogEvent < Webhookdb::Postgres::Model(:system_log_events)
+  plugin :text_searchable, terms: [:title, :body]
+
+  many_to_one :actor, class: "Webhookdb::Customer"
+end

data/lib/webhookdb/tasks/admin.rb

@@ -7,7 +7,7 @@ require "webhookdb"
 module Webhookdb::Tasks
   class Admin < Rake::TaskLib
     def initialize
-      super()
+      super
       namespace :admin do
         desc "Add roles to the named org"
         task :role, [:org_key, :role] do |_, args|

data/lib/webhookdb/tasks/annotate.rb

@@ -9,7 +9,7 @@ require "webhookdb/postgres"
 module Webhookdb::Tasks
   class Annotate < Rake::TaskLib
     def initialize
-      super()
+      super
       desc "Update model annotations"
       task :annotate do
         unless `git diff`.blank?

data/lib/webhookdb/tasks/db.rb

@@ -9,7 +9,7 @@ require "webhookdb/postgres"
 module Webhookdb::Tasks
   class DB < Rake::TaskLib
     def initialize
-      super()
+      super
       namespace :db do
         desc "Drop all tables in the public schema."
         task :drop_tables do
@@ -41,6 +41,18 @@ module Webhookdb::Tasks
         desc "Re-create the database tables. Drop tables and migrate."
         task reset: ["db:drop_tables", "db:migrate"]
 
+        desc "Set all model tables to UNLOGGED. Do this after test migrating. NEVER PROD."
+        task :unlogged do
+          raise "Unly run under RACK_ENV=test" unless Webhookdb::RACK_ENV == "test"
+          require "webhookdb/postgres"
+          Webhookdb::Postgres.load_superclasses
+          Webhookdb::Postgres.each_model_superclass do |sc|
+            sc.tsort.reverse_each do |m|
+              self.exec(sc.db, "ALTER TABLE #{m.tablename} SET UNLOGGED")
+            end
+          end
+        end
+
         task :drop_replication_databases do
           require "webhookdb/postgres"
           Webhookdb::Postgres.load_superclasses

data/lib/webhookdb/tasks/docs.rb

@@ -9,7 +9,7 @@ require "webhookdb/postgres"
 module Webhookdb::Tasks
   class Docs < Rake::TaskLib
     def initialize
-      super()
+      super
       namespace :docs do
         desc "Write out auto-generated docs for integrations."
         task :replicators, [:out, :name] do |_, args|

data/lib/webhookdb/tasks/fixture.rb

@@ -8,7 +8,7 @@ require "webhookdb/postgres"
 module Webhookdb::Tasks
   class Fixture < Rake::TaskLib
     def initialize
-      super()
+      super
       namespace :fixture do
         desc "Create a bunch of fake integrations and fill them with data."
         task :full do

data/lib/webhookdb/tasks/message.rb

@@ -5,7 +5,7 @@ require "rake/tasklib"
 module Webhookdb::Tasks
   class Message < Rake::TaskLib
     def initialize
-      super()
+      super
       namespace :message do
         desc "Render the specified message"
         task :render, [:template_class, :out] do |_t, args|

data/lib/webhookdb/tasks/regress.rb

@@ -9,7 +9,7 @@ require "webhookdb/postgres"
 module Webhookdb::Tasks
   class Regress < Rake::TaskLib
     def initialize
-      super()
+      super
       namespace :regress do
         desc "Creates databases for all orgs that do not have them."
         task :prepare do

data/lib/webhookdb/tasks/release.rb

@@ -7,7 +7,7 @@ require "webhookdb"
 module Webhookdb::Tasks
   class Release < Rake::TaskLib
     def initialize
-      super()
+      super
       desc "Migrate replication tables for each integration, ensure all columns and backfill new columns."
       task :migrate_replication_tables do
         Webhookdb.load_app

data/lib/webhookdb/tasks/sidekiq.rb

@@ -7,7 +7,7 @@ require "webhookdb"
 module Webhookdb::Tasks
   class Sidekiq < Rake::TaskLib
     def initialize
-      super()
+      super
       namespace :sidekiq do
         desc "Clear the Sidekiq redis DB (flushdb). " \
              "Only use on local, and only for legit reasons, " \

data/lib/webhookdb/tasks/specs.rb

@@ -8,7 +8,7 @@ require "webhookdb"
 module Webhookdb::Tasks
   class Specs < Rake::TaskLib
     def initialize
-      super()
+      super
       namespace :specs do
         desc "Run API integration tests in the 'integration' folder of this gem. " \
              "To run your own tests, create a task similar to this one, " \

data/lib/webhookdb/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Webhookdb
-  VERSION = "1.3.1"
+  VERSION = "1.5.0"
 end

data/lib/webhookdb/webhook_subscription.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "webhookdb/jobs/webhook_subscription_delivery_attempt"
+require "webhookdb/jobs/webhook_subscription_delivery_event"
 
 # Webhook subscriptions have a few parts:
 #
@@ -136,9 +136,8 @@ class Webhookdb::WebhookSubscription < Webhookdb::Postgres::Model(:webhook_subsc
       rescue StandardError => e
         self.logger.error(
           "webhook_subscription_delivery_failure",
-          error: e,
-          webhook_subscription_id: self.id,
-          webhook_subscription_delivery_id: d.id,
+          {webhook_subscription_id: self.id, webhook_subscription_delivery_id: d.id},
+          e,
         )
         d.add_attempt(status: e.is_a?(Webhookdb::Http::Error) ? e.status : 0)
         if attempt < MAX_DELIVERY_ATTEMPTS

data/lib/webhookdb.rb

@@ -19,6 +19,8 @@ Money.locale_backend = :i18n
 Money.default_currency = "USD"
 Money.rounding_mode = BigDecimal::ROUND_HALF_UP
 
+ActiveSupport.to_time_preserves_timezone = true
+
 module Appydays::Configurable
   def self.fetch_env(keys, default=:__keyerror, env: ENV)
     keys = [keys] unless keys.respond_to?(:to_ary)
@@ -35,27 +37,33 @@ module Webhookdb
   include Appydays::Configurable
   extend Webhookdb::MethodUtilities
 
+  # Base class for all WebhookDB errors.
+  class WebhookdbError < StandardError; end
+
+  # Class for errors that usually should not be rescued from.
+  class ProgrammingError < WebhookdbError; end
+
   # Error raised when we cannot take an action
   # because some condition has not been set up right.
-  class InvalidPrecondition < StandardError; end
+  class InvalidPrecondition < ProgrammingError; end
 
   # Error raised when, after we take an action,
   # something we expect to have changed has not changed.
-  class InvalidPostcondition < StandardError; end
+  class InvalidPostcondition < ProgrammingError; end
 
   # Some invariant has been violated, which we never expect to see.
-  class InvariantViolation < StandardError; end
+  class InvariantViolation < ProgrammingError; end
 
   # Error raised when a customer gives us some invalid input.
   # Allows the library to raise the error with the message,
   # and is caught automatically by the service as a 400.
-  class InvalidInput < StandardError; end
+  class InvalidInput < WebhookdbError; end
 
   # Raised when an organization's database cannot be modified.
-  class DatabaseLocked < StandardError; end
+  class DatabaseLocked < WebhookdbError; end
 
   # Used in various places that need to short-circuit code in regression mode.
-  class RegressionModeSkip < StandardError; end
+  class RegressionModeSkip < WebhookdbError; end
 
   APPLICATION_NAME = "Webhookdb"
   RACK_ENV = Appydays::Configurable.fetch_env(["RACK_ENV", "RUBY_ENV"], "development")
@@ -78,7 +86,7 @@ module Webhookdb
             nil,
             key: "LOG_LEVEL",
             side_effect: ->(v) { Appydays::Loggable.default_level = v if v }
-    setting :log_format, nil
+    setting :log_format, :json_trunc
     setting :app_url, "http://localhost:18002"
     setting :api_url, "http://localhost:#{ENV.fetch('PORT', 18_001)}"
     setting :bust_idempotency, false
@@ -93,6 +101,8 @@ module Webhookdb
     end
   end
 
+  def self.admin_url = self.api_url
+
   # Regression mode is true when we re replaying webhooks locally,
   # or for some other reason, want to disable certain checks we use in production.
   # For example, we may want to ignore certain errors (like if integrations are missing dependency rows),
@@ -134,7 +144,23 @@ module Webhookdb
   # :section: Errors
   #
 
-  class LockFailed < StandardError; end
+  class LockFailed < WebhookdbError; end
+
+  # This exception is rescued in the API and returned to the caller via +merror!+.
+  # This is mostly used in synchronously-processed replicators that need to return
+  # very specific types of errors during processing.
+  # This is very rare.
+  class ExceptionCarrier < WebhookdbError
+    attr_reader :status, :code, :more, :headers
+
+    def initialize(status, message, code: nil, more: {}, headers: {})
+      super(message)
+      @status = status
+      @code = code
+      @more = more
+      @headers = headers
+    end
+  end
 
   ### Generate a key for the specified Sequel model +instance+ and
   ### any additional +parts+ that can be used for idempotent requests.