webhookdb 1.2.2 → 1.3.0

data/lib/webhookdb/replicator/url_recorder_v1.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+class Webhookdb::Replicator::UrlRecorderV1 < Webhookdb::Replicator::Base
+  include Appydays::Loggable
+
+  # @return [Webhookdb::Replicator::Descriptor]
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "url_recorder_v1",
+      ctor: ->(sint) { self.new(sint) },
+      feature_roles: [],
+      resource_name_singular: "URL Recorder",
+      supports_webhooks: true,
+      supports_backfill: false,
+      description: "Record any visit to the webhook URL for later inspection. " \
+                   "Useful for recording scans, like visiting QR Code. After the webhook, " \
+                   "visitors can be redirected, or shown a Markdown page.",
+    )
+  end
+
+  def _remote_key_column = Webhookdb::Replicator::Column.new(:unique_id, BIGINT)
+
+  def requires_sequence? = true
+
+  def _denormalized_columns
+    col = Webhookdb::Replicator::Column
+    return [
+      col.new(:inserted_at, TIMESTAMP, index: true),
+      col.new(:request_method, TEXT),
+      col.new(:path, TEXT),
+      col.new(:full_url, TEXT),
+      col.new(:user_agent, TEXT),
+      col.new(:ip, TEXT),
+      col.new(:content_type, TEXT),
+      col.new(:parsed_query, OBJECT),
+      col.new(:parsed_body, OBJECT),
+      col.new(:raw_body, TEXT),
+    ]
+  end
+
+  def _timestamp_column_name = :inserted_at
+  def _resource_and_event(_request) = [{}, nil]
+
+  def _update_where_expr = self.qualified_table_sequel_identifier[:inserted_at] < Sequel[:excluded][:inserted_at]
+
+  def _prepare_for_insert(_resource, event, request, enrichment)
+    # rr = Rack::Request.new
+    rr = request.rack_request
+    raise Webhookdb::InvalidPrecondition, "#{request} must have rack_request set" if rr.nil?
+    r = {
+      "unique_id" => self.service_integration.sequence_nextval,
+      "inserted_at" => Time.now,
+      "request_method" => rr.request_method,
+      "path" => rr.path,
+      "parsed_query" => rr.GET,
+      "raw_query" => rr.query_string,
+      "full_url" => rr.url,
+      "user_agent" => rr.user_agent,
+      "ip" => rr.ip,
+      "content_type" => rr.content_type,
+      "raw_body" => nil,
+      "parsed_body" => nil,
+    }
+    if !request.body.is_a?(String)
+      # If we were able to parse the request body (usually means it's JSON), store it.
+      r["parsed_body"] = request.body
+    elsif rr.POST.present?
+      # If Rack was able to parse the request body (usually means it's form encoded), store it.
+      r["parsed_body"] = rr.POST
+    else
+      # Store the raw body if nothing can parse it.
+      r["raw_body"] = request.body
+    end
+    return super(r, event, request, enrichment)
+  end
+
+  def _resource_to_data(*) = {}
+
+  def _webhook_response(_request) = self.redirect? ? self._redirect_response : self._page_response
+
+  def process_webhooks_synchronously? = true
+
+  def synchronous_processing_response_body(*)
+    resp = self.redirect? ? self._redirect_response : self._page_response
+    return resp.body
+  end
+
+  def redirect? = self.service_integration.api_url =~ %r{^https?://}
+
+  def _redirect_response
+    headers = {"Location" => self.service_integration.api_url, "Content-Type" => "text/plain"}
+    return Webhookdb::WebhookResponse.new(status: 302, headers:, body: "")
+  end
+
+  def _page_response
+    headers = {"Content-Type" => "text/html; charset=UTF-8"}
+    content = self.service_integration.api_url
+    content_is_doc = content.start_with?("<!DOCTYPE") || content.starts_with?("<html")
+    body = if content_is_doc
+             content
+    else
+      tmpl_file = File.open(Webhookdb::DATA_DIR + "messages/replicators/url-recorder.liquid")
+      liquid_tmpl = Liquid::Template.parse(tmpl_file.read)
+      liquid_tmpl.render!({"content" => content})
+    end
+    return Webhookdb::WebhookResponse.new(status: 200, headers:, body:)
+  end
+
+  def calculate_webhook_state_machine
+    step = Webhookdb::Replicator::StateMachineStep.new
+    if self.service_integration.api_url.blank?
+      step.output = %(After users visit the WebhookDB endpoint,
+they can either be redirected to a location of your own,
+or we'll render an HTML page to show them.
+
+To use a redirect, input a URL starting with 'https://'.
+
+To render a page, paste in the HTML:
+
+- If the text starts with an `html` tag, it will be used as-is
+  for the page's HTML, so you can use your own styles.
+- Otherwise we assume the content is a relatively simple message,
+  and it's rendered with basic WebhookDB styles.)
+      return step.prompting("URL, HTML, or text").api_url(self.service_integration)
+    end
+    step.output = %(
+All set! Every visit to
+  #{self.webhook_endpoint}
+will be recorded.
+
+If you want to modify what users see after the visit is recorded,
+run `webhookdb integration reset #{self.descriptor.name}.
+
+#{self._query_help_output})
+    return step.completed
+  end
+end

data/lib/webhookdb/replicator/webhook_request.rb

@@ -2,4 +2,8 @@
 
 class Webhookdb::Replicator::WebhookRequest < Webhookdb::TypedStruct
   attr_accessor :body, :headers, :path, :method
+  # @!attribute rack_request
+  # When a webhook is processed synchronously, this will be set to the Rack::Request.
+  # Normal (async) webhook processing does not have this available.
+  attr_accessor :rack_request
 end

data/lib/webhookdb/replicator.rb

@@ -70,6 +70,12 @@ class Webhookdb::Replicator
     # for example in case 'backfill' is called but not supported.
     attr_reader :documentation_url
 
+    # If this integration uses /v1/install to set up,
+    # or some other link like a marketplace URL,
+    # provide it here. Note that you can add a redirect to Webhookdb::Apps::REDIRECTS
+    # to provide a pretty, arbitrary URL.
+    attr_reader :install_url
+
     # Markdown description of this replicator.
     attr_reader :description
 
@@ -94,6 +100,7 @@ class Webhookdb::Replicator
       description: nil,
       enterprise: false,
       documentation_url: nil,
+      install_url: nil,
       documentable: nil
     )
       raise ArgumentError, "must support one or both of webhooks and backfill" unless
@@ -107,6 +114,7 @@ class Webhookdb::Replicator
         dependency_descriptor:,
         documentation_url:,
         api_docs_url:,
+        install_url:,
         enterprise:
       )
       @ctor = ctor.is_a?(Class) ? ctor.method(:new) : ctor

data/lib/webhookdb/role.rb

@@ -3,6 +3,8 @@
 require "webhookdb/postgres/model"
 
 class Webhookdb::Role < Webhookdb::Postgres::Model(:roles)
+  plugin :text_searchable, terms: [:name]
+
   # n.b. Because of the uniqueness constraint on "name", there is only one "admin" role. Its meaning
   # depends on the context: if the customer has this role, they are an admin; if the org membership has
   # this role, the customer is an org admin.
@@ -30,8 +32,9 @@ end
 # Table: roles
 # ---------------------------------------------------------------------------------------------------------------------------
 # Columns:
-#  id   | integer | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY
-#  name | text    | NOT NULL
+#  id          | integer  | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY
+#  name        | text     | NOT NULL
+#  text_search | tsvector |
 # Indexes:
 #  roles_pkey     | PRIMARY KEY btree (id)
 #  roles_name_key | UNIQUE btree (name)

data/lib/webhookdb/saved_query.rb

@@ -2,6 +2,7 @@
 
 class Webhookdb::SavedQuery < Webhookdb::Postgres::Model(:saved_queries)
   plugin :timestamps
+  plugin :text_searchable, terms: [:organization, :created_by]
 
   CLI_EDITABLE_FIELDS = ["description", "sql", "public"].freeze
   INFO_FIELDS = {
@@ -26,3 +27,25 @@ class Webhookdb::SavedQuery < Webhookdb::Postgres::Model(:saved_queries)
 
   def run_url = "#{Webhookdb.api_url}/v1/saved_queries/#{self.opaque_id}/run"
 end
+
+# Table: saved_queries
+# ------------------------------------------------------------------------------------------------------
+# Columns:
+#  id              | integer                  | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY
+#  created_at      | timestamp with time zone | NOT NULL DEFAULT now()
+#  updated_at      | timestamp with time zone |
+#  organization_id | integer                  | NOT NULL
+#  created_by_id   | integer                  |
+#  opaque_id       | text                     | NOT NULL
+#  description     | text                     | NOT NULL
+#  sql             | text                     | NOT NULL
+#  public          | boolean                  | NOT NULL DEFAULT false
+#  text_search     | tsvector                 |
+# Indexes:
+#  saved_queries_pkey                  | PRIMARY KEY btree (id)
+#  saved_queries_opaque_id_key         | UNIQUE btree (opaque_id)
+#  saved_queries_organization_id_index | btree (organization_id)
+# Foreign key constraints:
+#  saved_queries_created_by_id_fkey   | (created_by_id) REFERENCES customers(id) ON DELETE SET NULL
+#  saved_queries_organization_id_fkey | (organization_id) REFERENCES organizations(id) ON DELETE CASCADE
+# ------------------------------------------------------------------------------------------------------

data/lib/webhookdb/saved_view.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+class Webhookdb::SavedView < Webhookdb::Postgres::Model(:saved_views)
+  plugin :timestamps
+  plugin :text_searchable, terms: [:organization, :created_by]
+
+  DOCS_URL = "https://docs.webhookdb.com/docs/integrating/saved-views.html"
+
+  class InvalidQuery < Webhookdb::InvalidInput; end
+
+  many_to_one :organization, class: "Webhookdb::Organization"
+  many_to_one :created_by, class: "Webhookdb::Customer"
+
+  def self.feature_role
+    return Webhookdb.cached_get("saved-view-feature-role") do
+      Webhookdb::Role.find_or_create_or_find(name: "saved_views")
+    end
+  end
+
+  def self.create_or_replace(organization:, sql:, name:, **kw)
+    Webhookdb::DBAdapter.validate_identifier!(name, type: "view")
+    self.db.transaction do
+      sv = self.find_or_create_or_find(organization:, name:) do |new|
+        new.sql = sql
+      end
+      sv.update(sql:, **kw)
+
+      # Verify that the underlying query is readonly, by running it as a readonly user.
+      if (_, errmsg = organization.execute_readonly_query_with_help(sql)) && errmsg.present?
+        raise InvalidQuery, errmsg
+      end
+
+      # Create the view now that we've asserted it's readonly
+      qname = Webhookdb::DBAdapter::PG.new.escape_identifier(name)
+      organization.admin_connection do |conn|
+        conn << "CREATE OR REPLACE VIEW #{qname} AS (#{sql})"
+      end
+      return sv
+    end
+  end
+
+  def before_destroy
+    raise Webhookdb::InvariantViolation, "#{self.inspect} name became invalid somehow" unless
+      Webhookdb::DBAdapter.valid_identifier?(self.name)
+    if self.organization.admin_connection_url_raw.present?
+      qname = Webhookdb::DBAdapter::PG.new.escape_identifier(self.name)
+      self.organization.admin_connection do |conn|
+        conn << "DROP VIEW IF EXISTS #{qname}"
+      end
+    end
+    super
+  end
+end
+
+# Table: saved_views
+# ----------------------------------------------------------------------------------------------------
+# Columns:
+#  id              | integer                  | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY
+#  created_at      | timestamp with time zone | NOT NULL DEFAULT now()
+#  updated_at      | timestamp with time zone |
+#  organization_id | integer                  | NOT NULL
+#  name            | text                     | NOT NULL
+#  sql             | text                     | NOT NULL
+#  created_by_id   | integer                  |
+#  text_search     | tsvector                 |
+# Indexes:
+#  saved_views_pkey                     | PRIMARY KEY btree (id)
+#  saved_views_organization_id_name_key | UNIQUE btree (organization_id, name)
+#  saved_views_organization_id_index    | btree (organization_id)
+# Foreign key constraints:
+#  saved_views_created_by_id_fkey   | (created_by_id) REFERENCES customers(id) ON DELETE SET NULL
+#  saved_views_organization_id_fkey | (organization_id) REFERENCES organizations(id) ON DELETE CASCADE
+# ----------------------------------------------------------------------------------------------------

data/lib/webhookdb/sentry.rb

@@ -12,6 +12,7 @@ module Webhookdb::Sentry
 
   configurable(:sentry) do
     setting :dsn, ""
+    setting :log_level, :warn
 
     # Apply the current configuration to Sentry.
     # See https://docs.sentry.io/clients/ruby/config/ for more info.
@@ -22,6 +23,7 @@ module Webhookdb::Sentry
         Sentry.init do |config|
           config.dsn = dsn
           config.logger = self.logger
+          config.logger.level = self.log_level
         end
       else
         Sentry.instance_variable_set(:@main_hub, nil)

data/lib/webhookdb/service/entities.rb

@@ -54,10 +54,6 @@ module Webhookdb::Service::Entities
         end
       end
     end
-
-    expose :message do |_instance, options|
-      options[:message] || ""
-    end
   end
 
   class Image < Base

data/lib/webhookdb/service/helpers.rb

@@ -142,6 +142,11 @@ module Webhookdb::Service::Helpers
     merror!(403, message, code: "permission_check")
   end
 
+  def check_feature_access!(org, role)
+    return if org.feature_roles.include?(role)
+    permission_error!("This feature is not enabled for your organization.")
+  end
+
   # Raise a 400 error for unstructured validation.
   # @param errors [Array<String>,String] Error messages, like 'password is invalid'.
   # @param message [String] If not given, build it from the errors list.

data/lib/webhookdb/service/middleware.rb

@@ -38,6 +38,14 @@ module Webhookdb::Service::Middleware
                  credentials: false,
                  expose: "*"
       end
+      allow do
+        origins("*")
+        resource "/v1/db/run_sql",
+                 headers: :any,
+                 methods: [:get, :post],
+                 credentials: false,
+                 expose: "*"
+      end
     end
   end
 
@@ -45,6 +53,7 @@ module Webhookdb::Service::Middleware
     builder.use(Rack::ContentLength)
     builder.use(Rack::Chunked)
     builder.use(Sentry::Rack::CaptureExceptions)
+    builder.use(Rack::Deflater)
   end
 
   def self.add_dev_middleware(builder)

data/lib/webhookdb/service/types.rb

@@ -7,18 +7,20 @@ module Webhookdb::Service::Types
     ctx.const_set(:NormalizedEmail, NormalizedEmail)
     ctx.const_set(:NormalizedPhone, NormalizedPhone)
     ctx.const_set(:CommaSepArray, CommaSepArray)
+    ctx.const_set(:TrimmedString, TrimmedString)
   end
 
-  class NormalizedEmail
-    def self.parse(value)
-      return value.downcase.strip
-    end
+  class NormalizedEmail < String
+    def self.parse(value) = self.new(value.downcase.strip)
   end
 
-  class NormalizedPhone
-    def self.parse(value)
-      return Webhookdb::PhoneNumber::US.normalize(value)
-    end
+  class NormalizedPhone < String
+    def self.parse(value) = self.new(Webhookdb::PhoneNumber::US.normalize(value))
+  end
+
+  class TrimmedString < String
+    def self.parse(value) = self.new(value.strip)
+    def self.map(arr) = arr.map { |a| self.new(a) }
   end
 
   class CommaSepArray

data/lib/webhookdb/service/validators.rb

@@ -20,8 +20,7 @@ module Webhookdb::Service::Validators
     def validate_param!(attr_name, params)
       val = params[attr_name]
       return if val.blank? && @allow_blank
-      re = Webhookdb::DBAdapter::VALID_IDENTIFIER
-      return if re.match?(val)
+      return if Webhookdb::DBAdapter.valid_identifier?(val)
       raise Grape::Exceptions::Validation.new(
         params: [@scope.full_name(attr_name)],
         message: "is not a valid database identifier for WebhookDB. " +

data/lib/webhookdb/service/view_api.rb

@@ -23,7 +23,7 @@ module Webhookdb::Service::ViewApi
       |
         tmpl_file = File.open(Webhookdb::DATA_DIR + data_rel_path)
         liquid_tmpl = Liquid::Template.parse(tmpl_file.read)
-        rendered = liquid_tmpl.render!(vars.stringify_keys, registers: {})
+        rendered = liquid_tmpl.render!(vars.deep_stringify_keys, registers: {})
         _endpoint.content_type content_type
         if serialize_view_params
           _endpoint.cookies[:whdbviewparams] = {path: data_rel_path, vars:, content_type:}.to_json

data/lib/webhookdb/service_integration.rb

@@ -20,6 +20,7 @@ class Webhookdb::ServiceIntegration < Webhookdb::Postgres::Model(:service_integr
   }.freeze
 
   plugin :timestamps
+  plugin :text_searchable, terms: [:service_name, :table_name, :organization]
   plugin :column_encryption do |enc|
     enc.column :data_encryption_secret
     enc.column :webhook_secret
@@ -145,10 +146,12 @@ class Webhookdb::ServiceIntegration < Webhookdb::Postgres::Model(:service_integr
   def destroy_self_and_all_dependents
     self.dependents.each(&:destroy_self_and_all_dependents)
 
-    begin
-      self.replicator.admin_dataset(timeout: :fast) { |ds| ds.db << "DROP TABLE #{self.table_name}" }
-    rescue Sequel::DatabaseError => e
-      raise unless e.wrapped_exception.is_a?(PG::UndefinedTable)
+    if self.organization.admin_connection_url.present?
+      begin
+        self.replicator.admin_dataset(timeout: :fast) { |ds| ds.db << "DROP TABLE #{self.table_name}" }
+      rescue Sequel::DatabaseError => e
+        raise e unless e.wrapped_exception.is_a?(PG::UndefinedTable)
+      end
     end
     self.destroy
   end
@@ -222,11 +225,7 @@ class Webhookdb::ServiceIntegration < Webhookdb::Postgres::Model(:service_integr
 
   def rename_table(to:)
     Webhookdb::Organization::DatabaseMigration.guard_ongoing!(self.organization)
-    unless Webhookdb::DBAdapter::VALID_IDENTIFIER.match?(to)
-      msg = "Sorry, this is not a valid table name. " + Webhookdb::DBAdapter::INVALID_IDENTIFIER_MESSAGE
-      msg += " And we see you what you did there ;)" if to.include?(";") && to.downcase.include?("drop")
-      raise TableRenameError, msg
-    end
+    Webhookdb::DBAdapter.validate_identifier!(to, type: "table")
     self.db.transaction do
       begin
         self.organization.admin_connection { |db| db << "ALTER TABLE #{self.table_name} RENAME TO #{to}" }
@@ -333,7 +332,7 @@ class Webhookdb::ServiceIntegration < Webhookdb::Postgres::Model(:service_integr
 end
 
 # Table: service_integrations
-# -----------------------------------------------------------------------------------------------------------------------------------------------------------
+# ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 # Columns:
 #  id                        | integer                  | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY
 #  created_at                | timestamp with time zone | NOT NULL DEFAULT now()
@@ -350,6 +349,8 @@ end
 #  depends_on_id             | integer                  |
 #  data_encryption_secret    | text                     |
 #  skip_webhook_verification | boolean                  | NOT NULL DEFAULT false
+#  webhookdb_api_key         | text                     |
+#  text_search               | tsvector                 |
 # Indexes:
 #  service_integrations_pkey          | PRIMARY KEY btree (id)
 #  service_integrations_opaque_id_key | UNIQUE btree (opaque_id)
@@ -358,8 +359,9 @@ end
 #  service_integrations_depends_on_id_fkey   | (depends_on_id) REFERENCES service_integrations(id) ON DELETE RESTRICT
 #  service_integrations_organization_id_fkey | (organization_id) REFERENCES organizations(id)
 # Referenced By:
-#  backfill_jobs         | backfill_jobs_service_integration_id_fkey         | (service_integration_id) REFERENCES service_integrations(id) ON DELETE CASCADE
-#  service_integrations  | service_integrations_depends_on_id_fkey           | (depends_on_id) REFERENCES service_integrations(id) ON DELETE RESTRICT
-#  sync_targets          | sync_targets_service_integration_id_fkey          | (service_integration_id) REFERENCES service_integrations(id) ON DELETE CASCADE
-#  webhook_subscriptions | webhook_subscriptions_service_integration_id_fkey | (service_integration_id) REFERENCES service_integrations(id)
-# -----------------------------------------------------------------------------------------------------------------------------------------------------------
+#  backfill_jobs                          | backfill_jobs_service_integration_id_fkey                       | (service_integration_id) REFERENCES service_integrations(id) ON DELETE CASCADE
+#  backfill_job_service_integration_locks | backfill_job_service_integration_lo_service_integration_id_fkey | (service_integration_id) REFERENCES service_integrations(id) ON DELETE CASCADE
+#  service_integrations                   | service_integrations_depends_on_id_fkey                         | (depends_on_id) REFERENCES service_integrations(id) ON DELETE RESTRICT
+#  sync_targets                           | sync_targets_service_integration_id_fkey                        | (service_integration_id) REFERENCES service_integrations(id) ON DELETE CASCADE
+#  webhook_subscriptions                  | webhook_subscriptions_service_integration_id_fkey               | (service_integration_id) REFERENCES service_integrations(id)
+# ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

data/lib/webhookdb/spec_helpers/shared_examples_for_replicators.rb

@@ -45,7 +45,7 @@ RSpec.shared_examples "a replicator" do |name|
       if expected_row
         expect(ds.first).to match(expected_row)
       else
-        expect(ds.first[:data]).to eq(expected_data)
+        expect(ds.first[:data].to_h).to eq(expected_data)
       end
     end
   end
@@ -59,7 +59,7 @@ RSpec.shared_examples "a replicator" do |name|
       if expected_row
         expect(ds.first).to match(expected_row)
       else
-        expect(ds.first[:data]).to eq(expected_data)
+        expect(ds.first[:data].to_h).to eq(expected_data)
       end
       # this is how a fully qualified table is represented (schema->table, table->column)
       expect(ds.opts[:from].first).to have_attributes(table: "xyz", column: svc.service_integration.table_name.to_sym)
@@ -277,7 +277,7 @@ RSpec.shared_examples "a replicator that prevents overwriting new data with old"
       if expected_old_row
         expect(ds.first).to match(expected_old_row)
       else
-        expect(ds.first[:data]).to eq(expected_old_data)
+        expect(ds.first[:data].to_h).to eq(expected_old_data)
       end
 
       upsert_webhook(svc, body: new_body)
@@ -285,7 +285,7 @@ RSpec.shared_examples "a replicator that prevents overwriting new data with old"
       if expected_new_row
         expect(ds.first).to match(expected_new_row)
       else
-        expect(ds.first[:data]).to eq(expected_new_data)
+        expect(ds.first[:data].to_h).to eq(expected_new_data)
       end
     end
   end
@@ -299,7 +299,7 @@ RSpec.shared_examples "a replicator that prevents overwriting new data with old"
       if expected_new_row
         expect(ds.first).to match(expected_new_row)
       else
-        expect(ds.first[:data]).to eq(expected_new_data)
+        expect(ds.first[:data].to_h).to eq(expected_new_data)
       end
 
       upsert_webhook(svc, body: old_body)
@@ -307,7 +307,7 @@ RSpec.shared_examples "a replicator that prevents overwriting new data with old"
       if expected_new_row
         expect(ds.first).to match(expected_new_row)
       else
-        expect(ds.first[:data]).to eq(expected_new_data)
+        expect(ds.first[:data].to_h).to eq(expected_new_data)
       end
     end
   end
@@ -475,7 +475,7 @@ RSpec.shared_examples "a replicator that deals with resources and wrapped events
     upsert_webhook(svc, body: resource_json, headers: resource_headers)
     svc.readonly_dataset do |ds|
       expect(ds.all).to have_length(1)
-      expect(ds.first[:data]).to eq(resource_json)
+      expect(ds.first[:data].to_h).to eq(resource_json)
     end
   end
 
@@ -484,7 +484,7 @@ RSpec.shared_examples "a replicator that deals with resources and wrapped events
     upsert_webhook(svc, body: resource_in_envelope_json, headers: resource_in_envelope_headers)
     svc.readonly_dataset do |ds|
       expect(ds.all).to have_length(1)
-      expect(ds.first[:data]).to eq(resource_json)
+      expect(ds.first[:data].to_h).to eq(resource_json)
     end
   end
 end

data/lib/webhookdb/spec_helpers/whdb.rb

@@ -104,13 +104,14 @@ module Webhookdb::SpecHelpers::Whdb
     this.let(:request_method) { nil }
     this.let(:request_body) { nil }
     this.let(:request_headers) { nil }
+    this.let(:rack_request) { nil }
     this.let(:webhook_request) do
       Webhookdb::Replicator::WebhookRequest.new(
-        body: request_body, method: request_method, path: request_path, headers: request_headers,
+        body: request_body, method: request_method, path: request_path, headers: request_headers, rack_request:,
       )
     end
     this.define_method(:upsert_webhook) do |svc, **kw|
-      params = {body: request_body, headers: request_headers, method: request_method, path: request_path}
+      params = {body: request_body, headers: request_headers, method: request_method, path: request_path, rack_request:}
       params.merge!(**kw)
       svc.upsert_webhook(Webhookdb::Replicator::WebhookRequest.new(**params))
     end

data/lib/webhookdb/subscription.rb

@@ -23,6 +23,7 @@ class Webhookdb::Subscription < Webhookdb::Postgres::Model(:subscriptions)
 
   plugin :timestamps
   plugin :soft_deletes
+  plugin :text_searchable, terms: [:organization]
 
   configurable(:subscription) do
     setting :billing_enabled, false
@@ -196,6 +197,7 @@ end
 #  stripe_id          | text                     | NOT NULL
 #  stripe_customer_id | text                     | NOT NULL DEFAULT ''::text
 #  stripe_json        | jsonb                    | DEFAULT '{}'::jsonb
+#  text_search        | tsvector                 |
 # Indexes:
 #  subscriptions_pkey                     | PRIMARY KEY btree (id)
 #  subscriptions_stripe_id_key            | UNIQUE btree (stripe_id)

data/lib/webhookdb/sync_target.rb

@@ -74,6 +74,7 @@ class Webhookdb::SyncTarget < Webhookdb::Postgres::Model(:sync_targets)
   end
 
   plugin :timestamps
+  plugin :text_searchable, terms: [:service_integration, :created_by]
   plugin :column_encryption do |enc|
     enc.column :connection_url
   end
@@ -327,7 +328,8 @@ class Webhookdb::SyncTarget < Webhookdb::Postgres::Model(:sync_targets)
     # it wasn't in the last sync; however that is likely not a big problem
     # since clients need to handle updates in any case.
     def dataset_to_sync
-      @replicator.readonly_dataset do |ds|
+      # Use admin dataset, since the client could be using all their readonly conns.
+      @replicator.admin_dataset do |ds|
         # Find rows updated before we started
         tscond = (@timestamp_expr <= @now)
         # Find rows updated after the last sync was run
@@ -364,7 +366,12 @@ class Webhookdb::SyncTarget < Webhookdb::Postgres::Model(:sync_targets)
       # This is handled well so no need to re-raise.
       # We already committed the last page that was successful,
       # so we can just stop syncing at this point to try again later.
-      self.sync_target.logger.warn("sync_target_http_error", error: e)
+
+      # Don't spam our logs with downstream errors
+      idem_key = "sync_target_http_error-#{self.sync_target.id}-#{e.class.name}"
+      Webhookdb::Idempotency.every(1.hour).in_memory.under_key(idem_key) do
+        self.sync_target.logger.warn("sync_target_http_error", error: e)
+      end
     end
 
     def _flush_http_chunk(chunk)
@@ -480,6 +487,7 @@ end
 #  last_synced_at         | timestamp with time zone |
 #  last_applied_schema    | text                     | NOT NULL DEFAULT ''::text
 #  page_size              | integer                  | NOT NULL
+#  text_search            | tsvector                 |
 # Indexes:
 #  sync_targets_pkey                         | PRIMARY KEY btree (id)
 #  sync_targets_opaque_id_key                | UNIQUE btree (opaque_id)

data/lib/webhookdb/tasks/message.rb

@@ -28,7 +28,9 @@ module Webhookdb::Tasks
           SemanticLogger.add_appender(io: feedback_io)
 
           commit = Webhookdb::RACK_ENV != "test"
-          delivery = Webhookdb::Message::Delivery.preview(template_class_name.classify, commit:)
+          clsname = template_class_name.classify
+          (clsname += "s") if template_class_name.end_with?("s") && !clsname.end_with?("s")
+          delivery = Webhookdb::Message::Delivery.preview(clsname, commit:)
           feedback_io << "*** Created MessageDelivery: #{delivery.values}\n\n"
           feedback_io << delivery.body_with_mediatype!("text/plain")&.content
           feedback_io << "\n\n"

data/lib/webhookdb/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Webhookdb
-  VERSION = "1.2.2"
+  VERSION = "1.3.0"
 end