webhookdb 0.1.0

data/lib/webhookdb/tasks/fixture.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "rake/tasklib"
+
+require "webhookdb"
+require "webhookdb/postgres"
+
+module Webhookdb::Tasks
+  class Fixture < Rake::TaskLib
+    def initialize
+      super()
+      namespace :fixture do
+        desc "Create a bunch of fake integrations and fill them with data."
+        task :full do
+          require "webhookdb"
+          Webhookdb.load_app
+          require "webhookdb/fixtures"
+          Webhookdb::Fixtures.load_all
+          org = Webhookdb::Fixtures.organization.create
+          org.prepare_database_connections?
+          sints = Array.new(3) { Webhookdb::Fixtures.service_integration(organization: org).create }
+          now = Time.now
+          sints.each do |sint|
+            svc = sint.replicator
+            svc.create_table
+            Array.new(5000) do |i|
+              svc.upsert_webhook_body({"my_id" => i.to_s, "at" => (now - i.seconds).iso8601})
+            end
+          end
+          puts "Created Webhookdb::Organization[#{org.id}]"
+        end
+      end
+    end
+  end
+end

data/lib/webhookdb/tasks/message.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "rake/tasklib"
+
+module Webhookdb::Tasks
+  class Message < Rake::TaskLib
+    def initialize
+      super()
+      namespace :message do
+        desc "Render the specified message"
+        task :render, [:template_class, :out] do |_t, args|
+          template_class_name = args[:template_class] or
+            raise "Provide the template class name (NewCustomer or new_customer) as the first argument"
+          outpath = args[:out]
+          outpath = nil if outpath.blank? || outpath == "-"
+          if outpath
+            html_io = File.open(outpath, "w")
+            feedback_io = $stdout
+          else
+            html_io = $stdout
+            feedback_io = $stderr
+          end
+
+          ENV["WEBHOOKDB_DB_SLOW_QUERY_SECONDS"] ||= "1"
+          require "webhookdb"
+          Webhookdb.load_app
+          SemanticLogger.appenders.to_a.each { |a| SemanticLogger.remove_appender(a) }
+          SemanticLogger.add_appender(io: feedback_io)
+
+          commit = Webhookdb::RACK_ENV != "test"
+          delivery = Webhookdb::Message::Delivery.preview(template_class_name.classify, commit:)
+          feedback_io << "*** Created MessageDelivery: #{delivery.values}\n\n"
+          feedback_io << delivery.body_with_mediatype!("text/plain")&.content
+          feedback_io << "\n\n"
+          if outpath
+            feedback_io << "*** Writing HTML output to #{outpath}\n"
+          elsif html_io.tty?
+            feedback_io << "*** Writing HTML output to stdout.\n"
+            feedback_io << "*** Redirect it to a file (> temp.html), pass OUT to write it to a file (OUT=temp.html),\n"
+            feedback_io << "*** or view it at /admin_api/v1/message_deliveries/last\n\n"
+          else
+            feedback_io << "*** Writing HTML output to redirected stdout.\n"
+          end
+          html_io << delivery.body_with_mediatype!("text/html")&.content
+          html_io << "\n"
+        end
+      end
+    end
+  end
+end

data/lib/webhookdb/tasks/regress.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require "rake/tasklib"
+require "sequel"
+
+require "webhookdb"
+require "webhookdb/postgres"
+
+module Webhookdb::Tasks
+  class Regress < Rake::TaskLib
+    def initialize
+      super()
+      namespace :regress do
+        desc "Creates databases for all orgs that do not have them."
+        task :prepare do
+          Webhookdb.load_app
+          Webhookdb::Organization.where(readonly_connection_url_raw: nil).each do |org|
+            org.prepare_database_connections
+            org.migrate_replication_tables
+          end
+        end
+
+        desc "Prints out all service integrations that have untrimmed logged webhooks."
+        task :list_available do
+          Webhookdb.load_app
+          opaque_ids = Webhookdb::LoggedWebhook.where(truncated_at: nil).
+            distinct(:service_integration_opaque_id).
+            select_map(:service_integration_opaque_id)
+          sints = Webhookdb::ServiceIntegration.where(opaque_id: opaque_ids).all
+          self.print_service_integrations(sints)
+        end
+
+        desc "Replay the last :count webhooks going to the service integration with the given opaque id. " \
+             "Use -1 for all webhooks."
+        task :replay, [:opaque_id, :count] do |_, args|
+          opaque_id = args.fetch(:opaque_id)
+          count = args.fetch(:count).to_i
+          Webhookdb.load_app
+          ds = Webhookdb::LoggedWebhook.where(truncated_at: nil).
+            where(service_integration_opaque_id: opaque_id).
+            order(Sequel.asc(:inserted_at))
+          ds = ds.limit(count) if count >= 0
+          ds.paged_each do |lw|
+            good = lw.retry_one
+            puts "#{lw.pk} failed" unless good
+          end
+        end
+
+        desc "Prints out all service integrations that have backfill info available."
+        task :list_backfill do
+          Webhookdb.load_app
+          sints = Webhookdb::ServiceIntegration.exclude(backfill_key: nil).all
+          self.print_service_integrations(sints)
+        end
+
+        desc "Runs a backfill for the service integration with the given opaque id. " \
+             "Regression backfills are limited to one page."
+        task :backfill, [:opaque_id] do |_, args|
+          opaque_id = args.fetch(:opaque_id)
+          Webhookdb.load_app
+          sint = Webhookdb::ServiceIntegration[opaque_id:] or raise "No service integration for #{opaque_id}"
+          bfjob = Webhookdb::BackfillJob.create_recursive(service_integration: sint, incremental: false)
+          sint.replicator.backfill(bfjob)
+        end
+      end
+    end
+
+    def print_service_integrations(iter)
+      rows = iter.map do |sint|
+        name = "#{sint.organization.name} (#{sint.organization_id})"
+        [name, sint.service_name, sint.opaque_id, sint.table_name]
+      end
+      rows.sort!
+      self.table_print rows
+    end
+
+    def table_print(rows)
+      max_lengths = rows.each_with_object({}) do |row, m|
+        row.each_with_index { |value, idx| m[idx] = [m.fetch(idx, 0), value.to_s.length].max }
+      end
+      row_strings = rows.map do |row|
+        row.each_with_index.map { |value, idx| "%-#{max_lengths[idx]}s" % value }.join("\t")
+      end
+      puts row_strings
+    end
+  end
+end

data/lib/webhookdb/tasks/release.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "rake/tasklib"
+
+require "webhookdb"
+
+module Webhookdb::Tasks
+  class Release < Rake::TaskLib
+    def initialize
+      super()
+      desc "Migrate replication tables for each integration, ensure all columns and backfill new columns."
+      task :migrate_replication_tables do
+        Webhookdb.load_app
+        Webhookdb::Organization.enqueue_migrate_all_replication_tables
+      end
+
+      desc "Run the release script against the current environment."
+      task :release do
+        Rake::Task["db:migrate"].invoke
+        Rake::Task["migrate_replication_tables"].invoke
+        if ENV["HEROKU_APP_ID"] && (ENV["RUN_INTEGRATION_TESTS_ON_RELEASE"] == "true")
+          Rake::Task["specs:heroku_integration_step1"].invoke
+        end
+      end
+    end
+  end
+end

data/lib/webhookdb/tasks/sidekiq.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "rake/tasklib"
+
+require "webhookdb"
+
+module Webhookdb::Tasks
+  class Sidekiq < Rake::TaskLib
+    def initialize
+      super()
+      namespace :sidekiq do
+        desc "Clear the Sidekiq redis DB (flushdb). " \
+             "Only use on local, and only for legit reasons, " \
+             "not to paper over problems that will show on staging and prod " \
+             "(like removing a job class)."
+        task :reset do
+          require "webhookdb/async"
+          ::Sidekiq.redis(&:flushdb)
+        end
+      end
+    end
+  end
+end

data/lib/webhookdb/tasks/specs.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "rake/tasklib"
+require "stringio"
+
+require "webhookdb"
+
+module Webhookdb::Tasks
+  class Specs < Rake::TaskLib
+    def initialize
+      super()
+      namespace :specs do
+        desc "Run API integration tests"
+        task :integration do
+          require "rspec/core"
+          require "slack-notifier"
+          require "webhookdb/spec_helpers/integration"
+          require "webhookdb/spec_helpers/citest"
+          Webhookdb::SpecHelpers::Citest.run_tests("integration")
+        end
+
+        desc "The release process needs to finish quickly, so start the integration tests in another dyno."
+        task :heroku_integration_step1 do
+          require "webhookdb/heroku"
+          Webhookdb::Heroku.client.dyno.create(
+            Webhookdb::Heroku.app_name,
+            command: "bundle exec rake specs:heroku_integration_step2",
+            attach: false,
+            time_to_live: 1.minute.to_i,
+            type: "run",
+          )
+        end
+        desc "Sleep 20 seconds to wait for the **next** one-off dyno to have the new code."
+        task :heroku_integration_step2 do
+          sleep(20)
+          require "webhookdb/heroku"
+          Webhookdb::Heroku.client.dyno.create(
+            Webhookdb::Heroku.app_name,
+            command: "bundle exec rake specs:integration_step3",
+            env: {"INTEGRATION_TESTS" => "true"},
+            attach: false,
+            time_to_live: 10.minute.to_i,
+            type: "run",
+          )
+        end
+        desc "Run the actual integration tests."
+        task :heroku_integration_step3 do
+          require "webhookdb/heroku"
+          Rake::Task["specs:integration"].invoke
+        end
+
+        desc "Do a thing through Rake so it's easy to run under the debugger"
+        task :debugtask do
+          require "webhookdb"
+          Webhookdb.load_app
+          sint = Webhookdb::ServiceIntegration[84]
+          row_pk = 710
+          expiring_before = 1.week.from_now
+          sint.replicator.renew_watch_channel(row_pk:, expiring_before:)
+        end
+      end
+    end
+  end
+end

data/lib/webhookdb/theranest.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "appydays/configurable"
+
+module Webhookdb::Theranest
+  include Appydays::Configurable
+
+  configurable(:theranest) do
+    setting :cron_expression, "30 8 * * *" # default to midnight
+    setting :appointment_look_back_months, 12
+    setting :appointment_look_forward_months, 3
+    setting :page_size, 50
+    setting :http_timeout, 30
+  end
+end

data/lib/webhookdb/transistor.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "appydays/configurable"
+
+module Webhookdb::Transistor
+  include Appydays::Configurable
+
+  configurable(:transistor) do
+    setting :episode_cron_expression, "30 * * * *"
+    setting :show_cron_expression, "0 0 */1 * *"
+    setting :http_timeout, 30
+  end
+end

data/lib/webhookdb/twilio.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "appydays/configurable"
+require "appydays/loggable"
+
+module Webhookdb::Twilio
+  include Appydays::Configurable
+  include Appydays::Loggable
+
+  configurable(:twilio) do
+    setting :http_timeout, 30
+  end
+end

data/lib/webhookdb/typed_struct.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+class Webhookdb::TypedStruct
+  def initialize(**kwargs)
+    self._apply(self._defaults.merge(kwargs))
+  end
+
+  def _defaults
+    return {}
+  end
+
+  def [](k)
+    return self.send(k)
+  end
+
+  # Modify the receiver with kwargs.
+  def _apply(kwargs)
+    kwargs.each do |k, v|
+      raise TypeError, "invalid struct field #{k}" unless self.respond_to?(k)
+      self.instance_variable_set(:"@#{k}", v)
+    end
+  end
+
+  def change(**)
+    c = self.dup
+    c._apply(**)
+    return c
+  end
+
+  def as_json
+    return self.instance_values.as_json
+  end
+
+  protected def typecheck!(field, type, nullable: false)
+    value = self.send(field)
+    return if nullable && value.nil?
+    if type == :boolean
+      return if [true, false].include?(value)
+    elsif value.is_a?(type)
+      return
+    end
+    raise ArgumentError, "#{field} #{value.inspect} must be a #{type.name}"
+  end
+end

data/lib/webhookdb/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Webhookdb
+  VERSION = "0.1.0"
+end

data/lib/webhookdb/webhook_response.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "webhookdb/typed_struct"
+
+class Webhookdb::WebhookResponse < Webhookdb::TypedStruct
+  # The standard WebhookDB Secret header, in RFC header format (My-Hdr).
+  SECRET_HEADER_RFC = "Whdb-Webhook-Secret"
+  # The standard WebhookDB secret header, in Rack header format (HTTP_MY_HDR).
+  SECRET_HEADER_RACK = "HTTP_WHDB_WEBHOOK_SECRET"
+
+  # Compare the value of the SECRET_HEADER_RACK in the request header.
+  # @param [Rack::Request] request
+  # @param [String] webhook_secret
+  # @return [Webhookdb::WebhookResponse]
+  def self.for_standard_secret(request, webhook_secret, ok_status: 202)
+    hdr_secret = request.env[SECRET_HEADER_RACK]
+    return self.error("missing secret header") if hdr_secret.nil?
+    matches = ActiveSupport::SecurityUtils.secure_compare(webhook_secret, hdr_secret)
+    return self.error("secret mismatch") unless matches
+    return self.ok(status: ok_status)
+  end
+
+  attr_reader :status,
+              :headers,
+              :body,
+              :reason
+
+  def self.error(reason, status: 401)
+    return self.new(status:, json: {message: reason}, reason:)
+  end
+
+  def self.ok(json: {o: "k"}, status: 202)
+    return self.new(status:, json:)
+  end
+
+  def initialize(status:, body: nil, json: nil, reason: nil, headers: {})
+    raise "Reason must be provided if returning an error" if !reason && status >= 400
+    if json
+      body = json.to_json
+      headers["Content-Type"] = "application/json"
+    end
+    raise ":body or :json must be provided" if body.nil?
+    super(status:, body:, headers:, reason:)
+  end
+
+  # @return [Array<Integer, Hash, String>]
+  def to_rack
+    return [self.status, self.headers, self.body]
+  end
+end

data/lib/webhookdb/webhook_subscription/delivery.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "webhookdb/postgres/model"
+
+require "webhookdb/webhook_subscription"
+
+# Represents the attempted delivery of a rowupsert to a particular webhook subscription.
+# See WebhookSubscription for more details.
+class Webhookdb::WebhookSubscription::Delivery < Webhookdb::Postgres::Model(:webhook_subscription_deliveries)
+  plugin :timestamps
+
+  many_to_one :webhook_subscription, class: "Webhookdb::WebhookSubscription"
+
+  # See WebhookSubhscription#attempt_delivery
+  def attempt_delivery
+    self.webhook_subscription.attempt_delivery(self)
+  end
+
+  # Add an attempt to this instance.
+  def add_attempt(status:, at: Time.now)
+    self.attempt_timestamps << at
+    self.modified!(:attempt_timestamps)
+    self.attempt_http_response_statuses << status
+    self.modified!(:attempt_http_response_statuses)
+  end
+
+  # Create a list of Attempt instances.
+  def attempts
+    return self.attempt_timestamps.
+        zip(self.attempt_http_response_statuses).
+        map { |(at, status)| Attempt.new(at, status) }
+  end
+
+  # Fast path for getting the total attempt count.
+  def attempt_count
+    return self.attempt_timestamps.length
+  end
+
+  # Return the latest attempt, or nil if there have been no attempts.
+  def latest_attempt
+    cnt = self.attempt_count
+    return nil if cnt.zero?
+    ts = self.attempt_timestamps[cnt - 1]
+    status = self.attempt_http_response_statuses[cnt - 1]
+    return Attempt.new(ts, status)
+  end
+
+  # One of 'pending' (no attempts), 'success', or 'error'.
+  def latest_attempt_status
+    att = self.latest_attempt
+    return "pending" if att.nil?
+    return att.success ? "success" : "error"
+  end
+
+  class Attempt
+    attr_reader :at, :status, :success
+
+    def initialize(at, status)
+      @at = at
+      @status = status
+      @success = status < 300
+    end
+  end
+end
+
+# Table: webhook_subscription_deliveries
+# ------------------------------------------------------------------------------------------------------------------------------
+# Columns:
+#  id                             | integer                    | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY
+#  created_at                     | timestamp with time zone   | NOT NULL DEFAULT now()
+#  attempt_timestamps             | timestamp with time zone[] | NOT NULL DEFAULT ARRAY[]::timestamp with time zone[]
+#  attempt_http_response_statuses | smallint[]                 | NOT NULL DEFAULT ARRAY[]::smallint[]
+#  payload                        | jsonb                      | NOT NULL
+#  webhook_subscription_id        | integer                    | NOT NULL
+# Indexes:
+#  webhook_subscription_deliveries_pkey                          | PRIMARY KEY btree (id)
+#  webhook_subscription_deliveries_webhook_subscription_id_index | btree (webhook_subscription_id)
+# Check constraints:
+#  balanced_attempts | (array_length(attempt_timestamps, 1) = array_length(attempt_http_response_statuses, 1))
+# Foreign key constraints:
+#  webhook_subscription_deliveries_webhook_subscription_id_fkey | (webhook_subscription_id) REFERENCES webhook_subscriptions(id)
+# ------------------------------------------------------------------------------------------------------------------------------