webhookdb 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d22b47783f1aaabc29dcc0fa023c0fee5bc9088d303103d1cf0cf859f807df0f
-  data.tar.gz: 41f7fcafef5d6724060e99fd40c5cfd19e9cd8d58c5736ccdd1bbaef9ecca826
+  metadata.gz: d2c4da6abd3c5def6f27e180aaf435a4e9293eae4a1f926a065b87c1979372b3
+  data.tar.gz: 72de7406adc5049255878c5bf16fe05e92716d18c2f2492d996dbddb6b0aa307
 SHA512:
-  metadata.gz: 5d33bedf7341af3216f623a9cc4328615d7a383251fc6d6b7b21000ce0366c99ea53d31f15f92d4a0bc754084faa063b1feb81e0174a966b27c7fda538f65a9a
-  data.tar.gz: 57d0fe4657c4c54a2b36fc3f71f7ca14017473b06be658f87f813670d79d424d51692717e6c1e9558ce0c1314f068b1b78c967bbf49c5d3d8c43bb08c3af874f
+  metadata.gz: 725e0d6183cd7bfd259a4202c888e419658f77288de59adf2df1a6e48c36a06dd4f91857d43ac07f54e81aa9ee882a8e65813d3cb92d6cee0576d74a745ed28c
+  data.tar.gz: 6811f58d123c6693758c75c3e5ba4de991c7a5e3d621de581ac32fe76efb63ca8e910757cc3e42d18029fd10389c11d74b8255601781ed5c9267e042e0f7ca96

data/db/migrations/039_saved_query.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    create_table(:saved_queries) do
+      primary_key :id
+      timestamptz :created_at, null: false, default: Sequel.function(:now)
+      timestamptz :updated_at
+      foreign_key :organization_id, :organizations, null: false, unique: true, on_delete: :cascade
+      foreign_key :created_by_id, :customers, on_delete: :set_null
+      text :opaque_id, null: false, unique: true
+      text :description, null: false
+      text :sql, null: false
+      boolean :public, null: false, default: false
+    end
+  end
+end

data/lib/webhookdb/api/db.rb

@@ -124,25 +124,8 @@ class Webhookdb::API::Db < Webhookdb::API::V1
       end
       post :sql do
         org = lookup_org!(allow_connstr_auth: true)
-        begin
-          r = org.execute_readonly_query(params[:query])
-        rescue Sequel::DatabaseError => e
-          self.logger.error("db_query_database_error", error: e)
-          # We want to handle InsufficientPrivileges and UndefinedTable explicitly
-          # since we can hint the user at what to do.
-          # Otherwise, we should just return the Postgres exception.
-          case e.wrapped_exception
-            when PG::UndefinedTable
-              missing_table = e.wrapped_exception.message.match(/relation (.+) does not/)&.captures&.first
-              msg = "The table #{missing_table} does not exist. Run `webhookdb db tables` to see available tables." if
-                missing_table
-            when PG::InsufficientPrivilege
-              msg = "You do not have permission to perform this query. Queries must be read-only."
-          else
-              msg = e.wrapped_exception.message
-          end
-          merror!(403, msg, code: "invalid_query")
-        end
+        r, msg = execute_readonly_query(org, params[:query])
+        merror!(403, msg, code: "invalid_query") if r.nil?
         status 200
         present({rows: r.rows, headers: r.columns, max_rows_reached: r.max_rows_reached})
       end

data/lib/webhookdb/api/helpers.rb

@@ -250,4 +250,31 @@ module Webhookdb::API::Helpers
       service_integration_opaque_id: opaque_id,
     )
   end
+
+  # Run the given SQL inside the org, and use special error handling if it fails.
+  # @return [Array<Webhookdb::Organization::QueryResult,String,nil>] Tuple of query result, and optional message.
+  #   On query success, return <QueryResult, nil>.
+  #   On DatabaseError, return <nil, message>.
+  #   On other types of errors, raise.
+  def execute_readonly_query(org, sql)
+    result = org.execute_readonly_query(sql)
+    return result, nil
+  rescue Sequel::DatabaseError => e
+    self.logger.error("db_query_database_error", error: e)
+    # We want to handle InsufficientPrivileges and UndefinedTable explicitly
+    # since we can hint the user at what to do.
+    # Otherwise, we should just return the Postgres exception.
+    msg = ""
+    case e.wrapped_exception
+      when PG::UndefinedTable
+        missing_table = e.wrapped_exception.message.match(/relation (.+) does not/)&.captures&.first
+        msg = "The table #{missing_table} does not exist. Run `webhookdb db tables` to see available tables." if
+          missing_table
+      when PG::InsufficientPrivilege
+        msg = "You do not have permission to perform this query. Queries must be read-only."
+      else
+        msg = e.wrapped_exception.message
+    end
+    return [nil, msg]
+  end
 end

data/lib/webhookdb/api/saved_queries.rb

@@ -0,0 +1,219 @@
+# frozen_string_literal: true
+
+require "webhookdb/api"
+require "webhookdb/saved_query"
+
+class Webhookdb::API::SavedQueries < Webhookdb::API::V1
+  resource :organizations do
+    route_param :org_identifier do
+      resource :saved_queries do
+        helpers do
+          def lookup!
+            org = lookup_org!
+            # We can add other identifiers in the future
+            cq = org.saved_queries_dataset[opaque_id: params[:query_identifier]]
+            merror!(403, "There is no saved query with that identifier.") if cq.nil?
+            return cq
+          end
+
+          def guard_editable!(customer, cq)
+            return if customer === cq.created_by
+            return if has_admin?(cq.organization, customer:)
+            permission_error!("You must be the query's creator or an org admin.")
+          end
+
+          def execute_readonly_query_with_suggestion(org, sql)
+            r, message = execute_readonly_query(org, sql)
+            return r, nil unless r.nil?
+            msg = "Something went wrong running your query. Perhaps a table it depends on was deleted. " \
+                  "Check out #{Webhookdb::SavedQuery::DOCS_URL} for troubleshooting tips. " \
+                  "Here's what went wrong: #{message}"
+            return r, msg
+          end
+        end
+
+        desc "Returns a list of all custom queries associated with the org."
+        get do
+          queries = lookup_org!.saved_queries
+          message = ""
+          if queries.empty?
+            message = "This organization doesn't have any saved queries yet.\n" \
+                      "Use `webhookdb saved-query create` to set one up."
+          end
+          present_collection queries, with: SavedQueryEntity, message:
+        end
+
+        desc "Creates a custom query."
+        params do
+          optional :description, type: String, prompt: "What is the query used for? "
+          optional :sql, type: String, prompt: "Enter the SQL you would like to run: "
+          optional :public, type: Boolean
+        end
+        post :create do
+          cust = current_customer
+          org = lookup_org!
+          _, errmsg = execute_readonly_query_with_suggestion(org, params[:sql])
+          if errmsg
+            Webhookdb::API::Helpers.prompt_for_required_param!(
+              request,
+              :sql,
+              "Enter a new query:",
+              output: "That query was invalid. #{errmsg}\n" \
+                      "You can iterate on your query by connecting to your database from any SQL editor.\n" \
+                      "Use `webhookdb db connection` to get your query string.",
+            )
+          end
+          cq = Webhookdb::SavedQuery.create(
+            description: params[:description],
+            sql: params[:sql],
+            organization: org,
+            created_by: cust,
+            public: params[:public] || false,
+          )
+          message = "You have created a new saved query with an id of '#{cq.opaque_id}'. " \
+                    "You can run it through the CLI, or through the API with or without authentication. " \
+                    "See #{Webhookdb::SavedQuery::DOCS_URL} for more information."
+          status 200
+          present cq, with: SavedQueryEntity, message:
+        end
+
+        route_param :query_identifier, type: String do
+          desc "Returns the query with the given identifier."
+          get do
+            cq = lookup!
+            status 200
+            message = "See #{Webhookdb::SavedQuery::DOCS_URL} to see how to run or modify your query."
+            present cq, with: SavedQueryEntity, message:
+          end
+
+          desc "Runs the query with the given identifier."
+          get :run do
+            _customer = current_customer
+            org = lookup_org!
+            cq = lookup!
+            r, msg = execute_readonly_query_with_suggestion(org, cq.sql)
+            merror!(400, msg) if r.nil?
+            status 200
+            present({rows: r.rows, headers: r.columns, max_rows_reached: r.max_rows_reached})
+          end
+
+          desc "Updates the field on a custom query."
+          params do
+            optional :field, type: String, prompt: "What field would you like to update (one of: " \
+                                                   "#{Webhookdb::SavedQuery::CLI_EDITABLE_FIELDS.join(', ')}): "
+            optional :value, type: String, prompt: "What is the new value? "
+          end
+          post :update do
+            customer = current_customer
+            cq = lookup!
+            guard_editable!(customer, cq)
+            # Instead of specifying which values are valid for the optional `field` param in the param declaration,
+            # we do the validation here so that we can provide a more helpful error message
+            unless Webhookdb::SavedQuery::CLI_EDITABLE_FIELDS.include?(params[:field])
+              merror!(400, "That field is not editable.")
+            end
+            value = params[:value]
+            case params[:field]
+              when "public"
+                begin
+                  value = Webhookdb.parse_bool(value)
+                rescue ArgumentError => e
+                  Webhookdb::API::Helpers.prompt_for_required_param!(
+                    request,
+                    :value,
+                    e.message + "\nAny boolean-like string (true, false, yes, no, etc) will work:",
+                  )
+                end
+                cq.public = value
+              when "sql"
+                r, msg = execute_readonly_query_with_suggestion(cq.organization, value)
+                if r.nil?
+                  Webhookdb::API::Helpers.prompt_for_required_param!(
+                    request,
+                    :value,
+                    "Enter your query:",
+                    output: msg,
+                  )
+                end
+                cq.sql = value
+              else
+                cq.send(:"#{params[:field]}=", value)
+            end
+            cq.save_changes
+            status 200
+            # Do not render the value here, it can be very long.
+            message = "You have updated '#{params[:field]}' on saved query '#{cq.opaque_id}'."
+            present cq, with: SavedQueryEntity, message:
+          end
+
+          params do
+            optional :field, type: String, values: Webhookdb::SavedQuery::INFO_FIELDS.keys + [""]
+          end
+          post :info do
+            cq = lookup!
+            data = Webhookdb::SavedQuery::INFO_FIELDS.
+              to_h { |k, v| [k.to_sym, cq.send(v)] }
+
+            field_name = params[:field]
+            blocks = Webhookdb::Formatting.blocks
+            if field_name.present?
+              blocks.line(data.fetch(field_name.to_sym))
+            else
+              rows = data.map do |k, v|
+                [k.to_s.humanize, v.to_s]
+              end
+              blocks.table(["Field", "Value"], rows)
+            end
+            r = {blocks: blocks.as_json}
+            status 200
+            present r
+          end
+
+          post :delete do
+            customer = current_customer
+            cq = lookup!
+            guard_editable!(customer, cq)
+            cq.destroy
+            status 200
+            present cq, with: SavedQueryEntity,
+                        message: "You have successfully deleted the saved query '#{cq.description}'."
+          end
+        end
+      end
+    end
+  end
+
+  resource :saved_queries do
+    route_param :query_identifier, type: String do
+      get :run do
+        # This endpoint can be used publicly, so should expose as little information as possible.
+        # Do not expose permissions or query details.
+        cq = Webhookdb::SavedQuery[opaque_id: params[:query_identifier]]
+        forbidden! if cq.nil?
+        if cq.private?
+          authed = Webhookdb::API::ConnstrAuth.find_authed([cq.organization], request)
+          if !authed && (cust = current_customer?)
+            authed = !cust.verified_memberships_dataset.where(organization: cq.organization).empty?
+            end
+          forbidden! unless authed
+        end
+        r, _ = execute_readonly_query(cq.organization, cq.sql)
+        merror!(400, "Something went wrong running the query.") if r.nil?
+        status 200
+        present({rows: r.rows, headers: r.columns, max_rows_reached: r.max_rows_reached})
+      end
+    end
+  end
+
+  class SavedQueryEntity < Webhookdb::API::BaseEntity
+    expose :opaque_id, as: :id
+    expose :description
+    expose :sql
+    expose :public
+    expose :run_url
+
+    def self.display_headers
+      return [[:id, "Id"], [:description, "Description"], [:public, "Public"], [:run_url, "Run URL"]]
+    end
+  end
+end

data/lib/webhookdb/api/sync_targets.rb

@@ -112,18 +112,14 @@ class Webhookdb::API::SyncTargets < Webhookdb::API::V1
             params do
               use :connection_url
               use :sync_target_params
-              optional :service_integration_opaque_id,
-                       type: String, allow_blank: false,
-                       desc: "This is a deprecated parameter. In the future, please use `service_integration_identifier`."
-              optional :service_integration_identifier, type: String, allow_blank: false
-              at_least_one_of :service_integration_opaque_id, :service_integration_identifier
+              requires :service_integration_identifier, type: String, allow_blank: false
             end
             route_setting :target_type, target_type_resource
             post :create do
               customer = current_customer
               org = lookup_org!(customer:)
               ensure_admin!(org, customer:)
-              identifier = params[:service_integration_identifier] || params[:service_integration_opaque_id]
+              identifier = params[:service_integration_identifier]
               sint = lookup_service_integration!(org, identifier)
 
               validate_period!(sint.organization, params[:period_seconds])

data/lib/webhookdb/api/webhook_subscriptions.rb

@@ -6,46 +6,40 @@ class Webhookdb::API::WebhookSubscriptions < Webhookdb::API::V1
   resource :organizations do
     route_param :org_identifier, type: String do
       resource :webhook_subscriptions do
-        desc "Return all webhook subscriptions for the given org, and all integrations."
+        desc "Return all notifications for the given org and its integrations."
         get do
           org = lookup_org!
           subs = org.all_webhook_subscriptions
           message = ""
           if subs.empty?
             message = "Organization #{org.name} has no webhook subscriptions set up.\n" \
-                      "Use `webhookdb webhooks create` to set one up."
+                      "Use `webhookdb notifications create` to set one up."
           end
           status 200
           present_collection subs, with: Webhookdb::API::WebhookSubscriptionEntity, message:
         end
 
         params do
-          optional :url, prompt: "Enter the URL that WebhookDB should POST webhooks to:"
-          optional :webhook_secret, prompt: "Enter a random secret used to sign and verify webhooks to the given url:"
           optional :service_integration_identifier,
                    type: String,
-                   desc: "If provided, attach the webhook subscription to this integration rather than the org."
-          optional :service_integration_opaque_id,
-                   type: String,
-                   desc: "This is a deprecated parameter. In the future, please use `service_integration_identifier`."
+                   desc: "If provided, attach the webhook subscription to this integration rather than the org.",
+                   prompt: "Which integration is this for? Use the service name, table name, or opaque id.\n" \
+                           "See your integrations with `webhookdb integrations list`:"
+          optional :url, prompt: "Enter the URL that WebhookDB should POST notifications to:"
+          optional :webhook_secret,
+                   prompt: "Enter a random secret used to sign and verify notifications to the given url:"
         end
         post :create do
           org = lookup_org!
-          sint = nil
-          identifier = params[:service_integration_identifier] || params[:service_integration_opaque_id]
-          sint = lookup_service_integration!(org, identifier) if identifier.present?
+          sint = lookup_service_integration!(org, params[:service_integration_identifier])
+          url = params[:url]
           webhook_sub = Webhookdb::WebhookSubscription.create(
             webhook_secret: params[:webhook_secret],
-            deliver_to_url: params[:url],
-            organization: sint ? nil : org,
+            deliver_to_url: url,
             service_integration: sint,
             created_by: current_customer,
           )
-          message = if sint
-                      "All webhooks for this #{sint.service_name} integration will be sent to #{params[:url]}"
-          else
-            "All webhooks for all integrations belonging to organization #{org.name} will be sent to #{params[:url]}."
-          end
+          message = "All notifications for this #{sint.service_name} integration will be sent to #{url}"
           status 200
           present webhook_sub, with: Webhookdb::API::WebhookSubscriptionEntity, message:
         end

data/lib/webhookdb/api.rb

@@ -60,13 +60,22 @@ module Webhookdb::API
             return org
           end
 
-          def ensure_admin!(org=nil, customer: nil)
+          # rubocop:disable Naming/PredicateName
+          def has_admin?(org=nil, customer: nil)
+            # rubocop:enable Naming/PredicateName
             customer ||= current_customer
             org ||= lookup_org!
             has_no_admin = org.verified_memberships_dataset.
               where(customer:, membership_role: Webhookdb::Role.admin_role).
               empty?
-            permission_error!("You don't have admin privileges with #{org.name}.") if has_no_admin
+            return !has_no_admin
+          end
+
+          def ensure_admin!(org=nil, customer: nil)
+            org ||= lookup_org!
+            admin = has_admin?(org, customer:)
+            # noinspection RubyNilAnalysis
+            permission_error!("You don't have admin privileges with #{org.name}.") unless admin
           end
         end
 

data/lib/webhookdb/apps.rb

@@ -17,6 +17,7 @@ require "webhookdb/api/install"
 require "webhookdb/api/me"
 require "webhookdb/api/organizations"
 require "webhookdb/api/replay"
+require "webhookdb/api/saved_queries"
 require "webhookdb/api/service_integrations"
 require "webhookdb/api/services"
 require "webhookdb/api/stripe"
@@ -68,6 +69,7 @@ module Webhookdb::Apps
     mount Webhookdb::API::Me
     mount Webhookdb::API::Organizations
     mount Webhookdb::API::Replay
+    mount Webhookdb::API::SavedQueries
     mount Webhookdb::API::ServiceIntegrations
     mount Webhookdb::API::Services
     mount Webhookdb::API::Stripe

data/lib/webhookdb/email_octopus.rb

@@ -2,6 +2,8 @@
 
 require "appydays/configurable"
 
+require "webhookdb/crypto"
+
 module Webhookdb::EmailOctopus
   include Appydays::Configurable
 

data/lib/webhookdb/envfixer.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Webhookdb
+  module Envfixer
+    # If DOCKER_DEV is set, replace 'localhost' urls with 'host.docker.internal'.
+    def self.replace_localhost_for_docker(env)
+      return unless env["DOCKER_DEV"]
+      env.each do |k, v|
+        begin
+          localhost = URI(v).host == "localhost"
+        rescue StandardError
+          next
+        end
+        next unless localhost
+        env[k] = v.gsub("localhost", "host.docker.internal")
+      end
+    end
+
+    # If MERGE_HEROKU_ENV, merge all of its environment vars into the current env
+    def self.merge_heroku_env(env)
+      return unless (heroku_app = env.fetch("MERGE_HEROKU_ENV", nil))
+      text = `heroku config -j --app=#{heroku_app}`
+      json = Oj.load(text)
+      json.each do |k, v|
+        env[k] = v
+      end
+    end
+  end
+end

data/lib/webhookdb/fixtures/saved_queries.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "faker"
+
+require "webhookdb"
+require "webhookdb/fixtures"
+
+module Webhookdb::Fixtures::SavedQueries
+  extend Webhookdb::Fixtures
+
+  fixtured_class Webhookdb::SavedQuery
+
+  base :saved_query do
+    self.description ||= Faker::Lorem.sentence
+    self.sql ||= "SELECT * FROM mytable"
+  end
+
+  before_saving do |instance|
+    instance.organization ||= Webhookdb::Fixtures.organization.create
+    instance
+  end
+
+  decorator :created_by do |c={}|
+    c = Webhookdb::Fixtures.customer.create(c) unless c.is_a?(Webhookdb::Customer)
+    self.created_by = c
+  end
+end

data/lib/webhookdb/google_calendar.rb

@@ -9,6 +9,12 @@ module Webhookdb::GoogleCalendar
     # How many calendars/events should we fetch in a single page?
     # Higher uses slightly more memory but fewer API calls.
     # Max of 2500.
+    #
+    # **NOTE:** Changing this in production will
+    # INVALIDATE EXISTING RESOURCES CAUSING A FULL RESYNC.
+    # You should, in general, avoid modifying this number once there is
+    # much Google Calendar data stored. Instead, page sizes will be automatically reduced
+    # as requests time out.
     setting :list_page_size, 2000
     # How many rows should we upsert at a time?
     # Higher is fewer upserts, but can create very large SQL strings,

data/lib/webhookdb/icalendar.rb

@@ -13,5 +13,11 @@ module Webhookdb::Icalendar
     # Do not store events older then this when syncing recurring events.
     # Many icalendar feeds are misconfigured and this prevents enumerating 2000+ years of recurrence.
     setting :oldest_recurring_event, "1990-01-01", convert: ->(s) { Date.parse(s) }
+    # Sync icalendar calendars only this often.
+    # Most services only update every day or so. Assume it takes 5s to sync each feed (request, parse, upsert).
+    # If you have 10,000 feeds, that is 50,000 seconds, or almost 14 hours of processing time,
+    # or two threads for 7 hours. The resyncs are spread out across the sync period
+    # (ie, no thundering herd every 8 hours), but it is still a good idea to sync as infrequently as possible.
+    setting :sync_period_hours, 6
   end
 end

data/lib/webhookdb/jobs/icalendar_enqueue_syncs.rb

@@ -3,6 +3,10 @@
 require "webhookdb/async/scheduled_job"
 require "webhookdb/jobs"
 
+# For every IcalendarCalendar row needing a sync (across all service integrations),
+# enqueue a +Webhookdb::Jobs::IcalendarSync+ job.
+# Jobs are 'splayed' over 1/4 of the configured calendar sync period (see +Webhookdb::Icalendar+)
+# to avoid a thundering herd.
 class Webhookdb::Jobs::IcalendarEnqueueSyncs
   extend Webhookdb::Async::ScheduledJob
 
@@ -10,12 +14,14 @@ class Webhookdb::Jobs::IcalendarEnqueueSyncs
   splay 30
 
   def _perform
+    max_splay = Webhookdb::Icalendar.sync_period_hours.hours.to_i / 4
     Webhookdb::ServiceIntegration.dataset.where_each(service_name: "icalendar_calendar_v1") do |sint|
       sint.replicator.admin_dataset do |ds|
         sint.replicator.rows_needing_sync(ds).each do |row|
           calendar_external_id = row.fetch(:external_id)
           self.with_log_tags(sint.log_tags) do
-            enqueued_job_id = Webhookdb::Jobs::IcalendarSync.perform_async(sint.id, calendar_external_id)
+            splay = rand(1..max_splay)
+            enqueued_job_id = Webhookdb::Jobs::IcalendarSync.perform_in(splay, sint.id, calendar_external_id)
             self.logger.info("enqueued_icalendar_sync", calendar_external_id:, enqueued_job_id:)
           end
         end

data/lib/webhookdb/jobs/prepare_database_connections.rb

@@ -15,8 +15,8 @@ class Webhookdb::Jobs::PrepareDatabaseConnections
     org.db.transaction do
       # If creating the public host fails, we end up with an orphaned database,
       # but that's not a big deal- we can eventually see it's empty/unlinked and drop it.
-      org.prepare_database_connections
-      org.create_public_host_cname
+      org.prepare_database_connections(safe: true)
+      org.create_public_host_cname(safe: true)
     end
   end
 end

data/lib/webhookdb/jobs/scheduled_backfills.rb

@@ -9,7 +9,7 @@ module Webhookdb::Jobs
   # Create a single way to do the common task of automatic scheduled backfills.
   # Each integration that needs automated backfills can add a specification here.
   module ScheduledBackfills
-    Spec = Struct.new(:klass, :service_name, :cron_expr, :splay, :incremental)
+    Spec = Struct.new(:klass, :service_name, :cron_expr, :splay, :incremental, :recursive)
 
     # @param spec [Spec]
     def self.install(spec)
@@ -21,7 +21,8 @@ module Webhookdb::Jobs
 
         define_method(:_perform) do
           Webhookdb::ServiceIntegration.dataset.where_each(service_name: spec.service_name) do |sint|
-            Webhookdb::BackfillJob.create(service_integration: sint, incremental: spec.incremental).enqueue
+            m = spec.recursive ? :create_recursive : :create
+            Webhookdb::BackfillJob.send(m, service_integration: sint, incremental: spec.incremental).enqueue
           end
         end
       end
@@ -31,19 +32,19 @@ module Webhookdb::Jobs
     [
       Spec.new(
         "ConvertkitBroadcastBackfill", "convertkit_broadcast_v1",
-        "10 * * * *", 2.minutes, false,
+        "10 * * * *", 2.minutes, false, false,
       ),
       Spec.new(
         "ConvertkitSubscriberBackfill", "convertkit_subscriber_v1",
-        "20 * * * *", 2.minutes, true,
+        "20 * * * *", 2.minutes, true, false,
       ),
       Spec.new(
         "ConvertkitTagBackfill", "convertkit_tag_v1",
-        "30 * * * *", 2.minutes, false,
+        "30 * * * *", 2.minutes, false, false,
       ),
       Spec.new(
         "EmailOctopusScheduledBackfill", "email_octopus_list_v1",
-        Webhookdb::EmailOctopus.cron_expression, 2.minutes, false,
+        Webhookdb::EmailOctopus.cron_expression, 2.minutes, false, true,
       ),
       Spec.new(
         "GithubRepoActivityScheduledBackfill", "github_repository_event_v1",
@@ -51,31 +52,31 @@ module Webhookdb::Jobs
       ),
       Spec.new(
         "IntercomScheduledBackfill", "intercom_marketplace_root_v1",
-        "*/1 * * * *", 0, false,
+        "*/1 * * * *", 0, false, true,
       ),
       Spec.new(
         "AtomSingleFeedPoller", "atom_single_feed_v1",
-        "11 * * * *", 10.seconds, true,
+        "11 * * * *", 10.seconds, true, false,
       ),
       Spec.new(
         "SponsyScheduledBackfill", "sponsy_publication_v1",
-        Webhookdb::Sponsy.cron_expression, 30.seconds, true,
+        Webhookdb::Sponsy.cron_expression, 30.seconds, true, true,
       ),
       Spec.new(
         "TransistorEpisodeBackfill", "transistor_episode_v1",
-        Webhookdb::Transistor.episode_cron_expression, 2.minutes, true,
+        Webhookdb::Transistor.episode_cron_expression, 2.minutes, true, true,
       ),
       Spec.new(
         "TransistorShowBackfill", "transistor_show_v1",
-        Webhookdb::Transistor.show_cron_expression, 2.minutes, true,
+        Webhookdb::Transistor.show_cron_expression, 2.minutes, true, false,
       ),
       Spec.new(
         "TwilioSmsBackfill", "twilio_sms_v1",
-        "*/1 * * * *", 0, true,
+        "*/1 * * * *", 0, true, false,
       ),
       Spec.new(
         "SignalwireMessageBackfill", "signalwire_message_v1",
-        "*/1 * * * *", 0, true,
+        "*/1 * * * *", 0, true, false,
       ),
     ].each { |sp| self.install(sp) }
   end

data/lib/webhookdb/organization.rb

@@ -34,6 +34,7 @@ class Webhookdb::Organization < Webhookdb::Postgres::Model(:organizations)
               adder: ->(om) { om.update(organization_id: id, verified: false) },
               order: :id
   one_to_many :service_integrations, class: "Webhookdb::ServiceIntegration", order: :id
+  one_to_many :saved_queries, class: "Webhookdb::SavedQuery", order: :id
   one_to_many :webhook_subscriptions, class: "Webhookdb::WebhookSubscription", order: :id
   many_to_many :feature_roles, class: "Webhookdb::Role", join_table: :feature_roles_organizations, right_key: :role_id
   one_to_many :all_webhook_subscriptions,
@@ -204,6 +205,7 @@ class Webhookdb::Organization < Webhookdb::Postgres::Model(:organizations)
   end
 
   # Build the org-specific users, database, and set our connection URLs to it.
+  # @param safe [*] If true, noop if connection urls are set.
   def prepare_database_connections(safe: false)
     self.db.transaction do
       self.lock!
@@ -220,13 +222,17 @@ class Webhookdb::Organization < Webhookdb::Postgres::Model(:organizations)
   end
 
   # Create a CNAME in Cloudflare for the currently configured connection urls.
-  def create_public_host_cname
+  # @param safe [*] If true, noop if the public host is set.
+  def create_public_host_cname(safe: false)
     self.db.transaction do
       self.lock!
       # We must have a host to create a CNAME to.
       raise Webhookdb::InvalidPrecondition, "connection urls must be set" if self.readonly_connection_url_raw.blank?
       # Should only be used once when creating the org DBs.
-      raise Webhookdb::InvalidPrecondition, "public_host must not be set" if self.public_host.present?
+      if self.public_host.present?
+        return if safe
+        raise Webhookdb::InvalidPrecondition, "public_host must not be set"
+      end
       # Use the raw URL, even though we know at this point
       # public_host is empty so raw and public host urls are the same.
       Webhookdb::Organization::DbBuilder.new(self).create_public_host_cname(self.readonly_connection_url_raw)

data/lib/webhookdb/postgres.rb

@@ -60,6 +60,7 @@ module Webhookdb::Postgres
     "webhookdb/organization/database_migration",
     "webhookdb/organization_membership",
     "webhookdb/role",
+    "webhookdb/saved_query",
     "webhookdb/service_integration",
     "webhookdb/subscription",
     "webhookdb/sync_target",
@@ -120,11 +121,13 @@ module Webhookdb::Postgres
   end
 
   def self.run_all_migrations(target: nil)
+    # :nocov:
     Sequel.extension :migration
     Webhookdb::Postgres.each_model_superclass do |cls|
       cls.install_all_extensions
       Sequel::Migrator.run(cls.db, Pathname(__FILE__).dirname.parent.parent + "db/migrations", target:)
     end
+    # :nocov:
   end
 
   # We can always register the models right away, since it does not have a side effect.

data/lib/webhookdb/replicator/front_signalwire_message_channel_app_v1.rb

@@ -263,7 +263,7 @@ All of this information can be found in the WebhookDB docs, at https://docs.webh
           item[:front_message_id] = front_response_body.fetch("message_uid")
         end
       else
-        messaged_at = Time.at(item.fetch(:data).fetch("created_at"))
+        messaged_at = Time.at(item.fetch(:data).fetch("payload").fetch("created_at"))
         if messaged_at < Webhookdb::Front.channel_sync_refreshness_cutoff.seconds.ago
           # Do not sync old rows, just mark them synced
           item[:signalwire_sid] = "skipped_due_to_age"

data/lib/webhookdb/replicator/icalendar_calendar_v1.rb

@@ -125,10 +125,9 @@ The secret to use for signing is:
   end
 
   CLEANUP_SERVICE_NAMES = ["icalendar_event_v1"].freeze
-  SYNC_PERIOD = 4.hours
 
   def rows_needing_sync(dataset, now: Time.now)
-    cutoff = now - SYNC_PERIOD
+    cutoff = now - Webhookdb::Icalendar.sync_period_hours.hours
     return dataset.where(Sequel[last_synced_at: nil] | Sequel.expr { last_synced_at < cutoff })
   end
 
@@ -215,7 +214,11 @@ The secret to use for signing is:
         expected_errors = [
           417, # If someone uses an Outlook HTML calendar, fetch gives us a 417
         ]
-        is_problem_error = (response_status > 404 || response_status < 400) &&
+        # For most client errors, we can't do anything about it. For example,
+        # and 'unshared' URL could result in a 401, 403, 404, or even a 405.
+        # For now, other client errors, we can raise on,
+        # in case it's something we can fix/work around.
+        is_problem_error = (response_status > 405 || response_status < 400) &&
           !expected_errors.include?(response_status)
         raise e if is_problem_error
         response_body = e.response.body.to_s

data/lib/webhookdb/replicator/signalwire_message_v1.rb

@@ -155,7 +155,9 @@ Press 'Show' next to the newly-created API token, and copy it.)
   def _fetch_backfill_page(pagination_token, last_backfilled:)
     urltail = pagination_token
     if pagination_token.blank?
-      date_send_max = Date.tomorrow
+      # We need to handle positive and negative UTC offset running locally (non-UTC).
+      # Using UTC + 1 day would give 'today' in some cases, we always want 'tomorrow the day after'.
+      date_send_max = (Time.now.utc + 2.days).to_date
       urltail = "/2010-04-01/Accounts/#{self.service_integration.backfill_key}/Messages.json" \
                 "?PageSize=100&DateSend%3C=#{date_send_max}"
     end

data/lib/webhookdb/replicator/twilio_sms_v1.rb

@@ -128,7 +128,9 @@ Both of these values should be visible from the homepage of your Twilio admin Da
   def _fetch_backfill_page(pagination_token, last_backfilled:)
     url = "https://api.twilio.com"
     if pagination_token.blank?
-      date_send_max = Date.tomorrow
+      # We need to handle positive and negative UTC offset running locally (non-UTC).
+      # Using UTC + 1 day would give 'today' in some cases, we always want 'tomorrow the day after'.
+      date_send_max = (Time.now.utc + 2.days).to_date
       url += "/2010-04-01/Accounts/#{self.service_integration.backfill_key}/Messages.json" \
              "?PageSize=100&DateSend%3C=#{date_send_max}"
     else

data/lib/webhookdb/saved_query.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class Webhookdb::SavedQuery < Webhookdb::Postgres::Model(:saved_queries)
+  plugin :timestamps
+
+  CLI_EDITABLE_FIELDS = ["description", "sql", "public"].freeze
+  INFO_FIELDS = {
+    "id" => :opaque_id,
+    "description" => :description,
+    "public" => :public,
+    "run_url" => :run_url,
+    "sql" => :sql,
+  }.freeze
+  DOCS_URL = "https://docs.webhookdb.com/docs/integrating/saved-queries.html"
+
+  many_to_one :organization, class: "Webhookdb::Organization"
+  many_to_one :created_by, class: "Webhookdb::Customer"
+
+  alias public? public
+  def private? = !self.public?
+
+  def before_create
+    self[:opaque_id] ||= Webhookdb::Id.new_opaque_id("svq")
+    super
+  end
+
+  def run_url = "#{Webhookdb.api_url}/v1/saved_queries/#{self.opaque_id}/run"
+end

data/lib/webhookdb/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Webhookdb
-  VERSION = "1.1.0"
+  VERSION = "1.2.0"
 end

data/lib/webhookdb.rb

@@ -8,29 +8,12 @@ require "money"
 require "pathname"
 require "phony"
 
+require "webhookdb/envfixer"
 require "webhookdb/json"
+require "webhookdb/method_utilities"
 
-if ENV["DOCKER_DEV"]
-  # If DOCKER_DEV is set, replace 'localhost' urls with 'host.docker.internal'.
-  ENV.each do |k, v|
-    begin
-      localhost = URI(v).host == "localhost"
-    rescue StandardError
-      next
-    end
-    next unless localhost
-    ENV[k] = v.gsub("localhost", "host.docker.internal")
-  end
-end
-
-if (heroku_app = ENV.fetch("MERGE_HEROKU_ENV", nil))
-  # If MERGE_HEROKU_ENV, merge all of its environment vars into the current env
-  text = `heroku config -j --app=#{heroku_app}`
-  json = Oj.load(text)
-  json.each do |k, v|
-    ENV[k] = v
-  end
-end
+Webhookdb::Envfixer.replace_localhost_for_docker(ENV)
+Webhookdb::Envfixer.merge_heroku_env(ENV)
 
 Money.locale_backend = :i18n
 Money.default_currency = "USD"
@@ -50,6 +33,7 @@ end
 module Webhookdb
   include Appydays::Loggable
   include Appydays::Configurable
+  extend Webhookdb::MethodUtilities
 
   # Error raised when we cannot take an action
   # because some condition has not been set up right.
@@ -86,6 +70,9 @@ module Webhookdb
 
   DATA_DIR = Pathname(__FILE__).dirname.parent + "data"
 
+  singleton_attr_reader :globals_cache
+  @globals_cache = {}
+
   configurable(:webhookdb) do
     setting :log_level_override,
             nil,
@@ -100,6 +87,10 @@ module Webhookdb
     setting :support_email, "hello@webhookdb.com"
     setting :use_globals_cache, false
     setting :regression_mode, false
+
+    after_configured do
+      globals_cache.clear
+    end
   end
 
   # Regression mode is true when we re replaying webhooks locally,
@@ -110,9 +101,6 @@ module Webhookdb
     return self.regression_mode
   end
 
-  require "webhookdb/method_utilities"
-  extend Webhookdb::MethodUtilities
-
   require "webhookdb/sentry"
 
   def self.load_app
@@ -129,9 +117,6 @@ module Webhookdb
   # :section: Globals cache
   #
 
-  singleton_attr_reader :globals_cache
-  @globals_cache = {}
-
   # If globals caching is enabled, see if there is a cached value under +key+
   # and return it if so. If there is not, evaluate the given block and store that value.
   # Generally used for looking up well-known database objects like certain roles.
@@ -141,7 +126,7 @@ module Webhookdb
       return result if result
     end
     result = yield()
-    self.globals_cache[key] = result
+    (self.globals_cache[key] = result) if self.use_globals_cache
     return result
   end
 
@@ -167,18 +152,6 @@ module Webhookdb
     return key
   end
 
-  #
-  # :section: Unambiguous/promo code chars
-  #
-
-  # Remove ambiguous characters (L, I, 1 or 0, O) and vowels from possible codes
-  # to avoid creating ambiguous codes or real words.
-  UNAMBIGUOUS_CHARS = "CDFGHJKMNPQRTVWXYZ23469".chars.freeze
-
-  def self.take_unambiguous_chars(n)
-    return Array.new(n) { UNAMBIGUOUS_CHARS.sample }.join
-  end
-
   # Convert a string into something we consistently use for slugs:
   # a-z, 0-9, and underscores only. Leading numbers are converted to words.
   #
@@ -206,6 +179,17 @@ module Webhookdb
     "9" => "nine",
   }.freeze
 
+  def self.parse_bool(s)
+    # rubocop:disable Style/NumericPredicate
+    return false if s == nil? || s.blank? || s == 0
+    # rubocop:enable Style/NumericPredicate
+    return true if s.is_a?(Integer)
+    sb = s.to_s.downcase
+    return true if ["true", "t", "yes", "y", "on", "1"].include?(sb)
+    return false if ["false", "f", "no", "n", "off", "0"].include?(sb)
+    raise ArgumentError, "unparseable bool: #{s.inspect}"
+  end
+
   # Return the request user and admin stored in TLS. See service.rb for implementation.
   #
   # Note that the second return value (the admin) will be nil if not authed as an admin,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: webhookdb
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - WebhookDB
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-01-19 00:00:00.000000000 Z
+date: 2024-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -853,6 +853,7 @@ files:
 - db/migrations/036_oauth.rb
 - db/migrations/037_oauth_used.rb
 - db/migrations/038_webhookdb_api_key.rb
+- db/migrations/039_saved_query.rb
 - integration/async_spec.rb
 - integration/auth_spec.rb
 - integration/database_spec.rb
@@ -883,6 +884,7 @@ files:
 - lib/webhookdb/api/me.rb
 - lib/webhookdb/api/organizations.rb
 - lib/webhookdb/api/replay.rb
+- lib/webhookdb/api/saved_queries.rb
 - lib/webhookdb/api/service_integrations.rb
 - lib/webhookdb/api/services.rb
 - lib/webhookdb/api/stripe.rb
@@ -918,6 +920,7 @@ files:
 - lib/webhookdb/developer_alert.rb
 - lib/webhookdb/email_octopus.rb
 - lib/webhookdb/enumerable.rb
+- lib/webhookdb/envfixer.rb
 - lib/webhookdb/fixtures.rb
 - lib/webhookdb/fixtures/backfill_jobs.rb
 - lib/webhookdb/fixtures/customers.rb
@@ -930,6 +933,7 @@ files:
 - lib/webhookdb/fixtures/organization_memberships.rb
 - lib/webhookdb/fixtures/organizations.rb
 - lib/webhookdb/fixtures/reset_codes.rb
+- lib/webhookdb/fixtures/saved_queries.rb
 - lib/webhookdb/fixtures/service_integrations.rb
 - lib/webhookdb/fixtures/subscriptions.rb
 - lib/webhookdb/fixtures/sync_targets.rb
@@ -1103,6 +1107,7 @@ files:
 - lib/webhookdb/replicator/webhook_request.rb
 - lib/webhookdb/replicator/webhookdb_customer_v1.rb
 - lib/webhookdb/role.rb
+- lib/webhookdb/saved_query.rb
 - lib/webhookdb/sentry.rb
 - lib/webhookdb/service.rb
 - lib/webhookdb/service/auth.rb