webhookdb 0.1.0

data/lib/webhookdb/jobs/create_stripe_customer.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+require "stripe"
+require "webhookdb/stripe"
+
+class Webhookdb::Jobs::CreateStripeCustomer
+  extend Webhookdb::Async::Job
+
+  on "webhookdb.organization.created"
+
+  def _perform(event)
+    org = self.lookup_model(Webhookdb::Organization, event)
+    return if Webhookdb::Subscription.billing_disabled?
+    org.register_in_stripe
+  end
+end

data/lib/webhookdb/jobs/customer_created_notify_internal.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+
+class Webhookdb::Jobs::CustomerCreatedNotifyInternal
+  extend Webhookdb::Async::Job
+
+  on "webhookdb.customer.created"
+
+  def _perform(event)
+    customer = self.lookup_model(Webhookdb::Customer, event)
+    Webhookdb::DeveloperAlert.new(
+      subsystem: "Customer Created",
+      emoji: ":hook:",
+      fallback: "New customer created: #{customer.inspect}",
+      fields: [
+        {title: "Id", value: customer.id, short: true},
+        {title: "Email", value: customer.email, short: true},
+      ],
+    ).emit
+  end
+end

data/lib/webhookdb/jobs/demo_mode_sync_data.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+
+class Webhookdb::Jobs::DemoModeSyncData
+  extend Webhookdb::Async::Job
+
+  on "webhookdb.organization.syncdemodata"
+
+  def _perform(event)
+    org = self.lookup_model(Webhookdb::Organization, event)
+    if org.admin_connection_url.blank?
+      # If PrepareDatabaseConnections hasn't run, we can't sync yet.
+      # Retry every 1 second for a couple minutes until we have a database.
+      raise Amigo::Retry::OrDie.new(120, 1)
+    end
+    Webhookdb::DemoMode.sync_demo_data(org)
+  end
+end

data/lib/webhookdb/jobs/deprecated_jobs.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "amigo/deprecated_jobs"
+
+Amigo::DeprecatedJobs.install(
+  Webhookdb,
+  # Put jobs here to die. If you just remove a job in Sidekiq, it may be queued up
+  # (like if it's scheduled or retrying),
+  # and will fail if the class does not exist.
+  #
+  # So, make the class exist, but noop so it won't be scheduled and won't be retried.
+  # Then it can be deleted later.
+  "Jobs::Test::DeprecatedJob",
+  "Jobs::ConvertKitBroadcastBackfill",
+  "Jobs::ConvertKitSubscriberBackfill",
+  "Jobs::ConvertKitTagBackfill",
+  "Jobs::RssBackfillPoller",
+  "Jobs::TwilioScheduledBackfill",
+)

data/lib/webhookdb/jobs/developer_alert_handle.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+
+class Webhookdb::Jobs::DeveloperAlertHandle
+  extend Webhookdb::Async::Job
+
+  on "webhookdb.developeralert.emitted"
+
+  def _perform(event)
+    alert = Webhookdb::DeveloperAlert.new(**event.payload.first.symbolize_keys)
+    alert.handle
+  end
+end

data/lib/webhookdb/jobs/durable_job_recheck_poller.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "amigo/durable_job"
+require "webhookdb/async/scheduled_job"
+require "webhookdb/jobs"
+
+class Webhookdb::Jobs::DurableJobRecheckPoller
+  extend Webhookdb::Async::ScheduledJob
+
+  cron "*/3 * * * *"
+  splay 5
+
+  def _perform
+    return if ENV["DISABLE_DURABLE_JOBS_POLL"]
+    Amigo::DurableJob.poll_jobs(self)
+  end
+end

data/lib/webhookdb/jobs/emailer.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/scheduled_job"
+
+class Webhookdb::Jobs::Emailer
+  extend Webhookdb::Async::ScheduledJob
+
+  cron "* * * * *"
+  splay 5.seconds
+
+  def _perform
+    self.logger.info "Sending pending emails"
+    Webhookdb::Message.send_unsent
+  end
+end

data/lib/webhookdb/jobs/icalendar_enqueue_syncs.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/scheduled_job"
+require "webhookdb/jobs"
+
+class Webhookdb::Jobs::IcalendarEnqueueSyncs
+  extend Webhookdb::Async::ScheduledJob
+
+  cron "*/30 * * * *" # Every 30 minutes
+  splay 30
+
+  def _perform
+    Webhookdb::ServiceIntegration.dataset.where_each(service_name: "icalendar_calendar_v1") do |sint|
+      sint.replicator.admin_dataset do |ds|
+        sint.replicator.rows_needing_sync(ds).each do |row|
+          calendar_external_id = row.fetch(:external_id)
+          self.with_log_tags(sint.log_tags) do
+            enqueued_job_id = Webhookdb::Jobs::IcalendarSync.perform_async(sint.id, calendar_external_id)
+            self.logger.info("enqueued_icalendar_sync", calendar_external_id:, enqueued_job_id:)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/webhookdb/jobs/icalendar_sync.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+require "webhookdb/jobs"
+
+class Webhookdb::Jobs::IcalendarSync
+  extend Webhookdb::Async::Job
+
+  sidekiq_options retry: false
+
+  def perform(sint_id, calendar_external_id)
+    sint = self.lookup_model(Webhookdb::ServiceIntegration, sint_id)
+    self.with_log_tags(sint.log_tags.merge(calendar_external_id:)) do
+      row = sint.replicator.admin_dataset { |ds| ds[external_id: calendar_external_id] }
+      if row.nil?
+        self.logger.warn("icalendar_sync_row_miss", calendar_external_id:)
+        return
+      end
+      self.logger.info("icalendar_sync_start")
+      sint.replicator.sync_row(row)
+    end
+  end
+end

data/lib/webhookdb/jobs/logged_webhook_replay.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+require "webhookdb/jobs"
+
+class Webhookdb::Jobs::LoggedWebhooksReplay
+  extend Webhookdb::Async::Job
+
+  on "webhookdb.loggedwebhook.replay"
+
+  def _perform(event)
+    lwh = self.lookup_model(Webhookdb::LoggedWebhook, event)
+    self.with_log_tags(service_integration_opaque_id: lwh.service_integration_opaque_id) do
+      lwh.retry_one(truncate_successful: true)
+    end
+  end
+end

data/lib/webhookdb/jobs/logged_webhook_resilient_replay.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/scheduled_job"
+require "webhookdb/jobs"
+
+class Webhookdb::Jobs::LoggedWebhooksResilientReplay
+  extend Webhookdb::Async::ScheduledJob
+
+  cron "*/3 * * * *"
+  splay 5
+
+  def _perform
+    Webhookdb::LoggedWebhook.resilient_replay
+  end
+end

data/lib/webhookdb/jobs/message_dispatched.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+
+class Webhookdb::Jobs::MessageDispatched
+  extend Webhookdb::Async::Job
+
+  on "webhookdb.message.delivery.dispatched"
+
+  def _perform(event)
+    delivery = self.lookup_model(Webhookdb::Message::Delivery, event)
+    Webhookdb::Idempotency.once_ever.under_key("message-dispatched-#{delivery.id}") do
+      delivery.send!
+    end
+  end
+end

data/lib/webhookdb/jobs/organization_database_migration_notify_finished.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+require "webhookdb/messages/org_database_migration_finished"
+
+class Webhookdb::Jobs::OrganizationDatabaseMigrationNotifyFinished
+  extend Webhookdb::Async::Job
+
+  on "webhookdb.organization.databasemigration.updated"
+
+  def _perform(event)
+    dbm = self.lookup_model(Webhookdb::Organization::DatabaseMigration, event)
+    case event.payload[1]
+      when changed(:finished_at, from: nil)
+        Webhookdb::Idempotency.once_ever.under_key("org-dbmigration-finish-#{dbm.id}") do
+          msg = Webhookdb::Messages::OrgDatabaseMigrationFinished.new(dbm)
+          dbm.organization.admin_customers.each { |c| msg.dispatch_email(c) }
+        end
+    end
+  end
+end

data/lib/webhookdb/jobs/organization_database_migration_notify_started.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+require "webhookdb/messages/org_database_migration_started"
+
+class Webhookdb::Jobs::OrganizationDatabaseMigrationNotifyStarted
+  extend Webhookdb::Async::Job
+
+  on "webhookdb.organization.databasemigration.updated"
+
+  def _perform(event)
+    dbm = self.lookup_model(Webhookdb::Organization::DatabaseMigration, event)
+    case event.payload[1]
+      when changed(:started_at, from: nil)
+        Webhookdb::Idempotency.once_ever.under_key("org-dbmigration-start-#{dbm.id}") do
+          msg = Webhookdb::Messages::OrgDatabaseMigrationStarted.new(dbm)
+          dbm.organization.admin_customers.each { |c| msg.dispatch_email(c) }
+        end
+    end
+  end
+end

data/lib/webhookdb/jobs/organization_database_migration_run.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "amigo/durable_job"
+require "webhookdb/async/job"
+
+class Webhookdb::Jobs::OrganizationDatabaseMigrationRun
+  extend Webhookdb::Async::Job
+  include Amigo::DurableJob
+
+  on "webhookdb.organization.databasemigration.created"
+
+  def _perform(event)
+    dbm = self.lookup_model(Webhookdb::Organization::DatabaseMigration, event)
+    self.with_log_tags(
+      organization_id: dbm.organization.id,
+      organization_name: dbm.organization.name,
+      organization_database_migration_id: dbm.id,
+    ) do
+      dbm.migrate
+    rescue Webhookdb::Organization::DatabaseMigration::MigrationAlreadyFinished
+      self.logger.warn("org_database_migration_already_finished")
+    end
+  end
+end

data/lib/webhookdb/jobs/prepare_database_connections.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "amigo/durable_job"
+require "webhookdb/async/job"
+
+class Webhookdb::Jobs::PrepareDatabaseConnections
+  extend Webhookdb::Async::Job
+  include Amigo::DurableJob
+
+  on "webhookdb.organization.created"
+  sidekiq_options queue: "critical"
+
+  def _perform(event)
+    org = self.lookup_model(Webhookdb::Organization, event)
+    org.db.transaction do
+      # If creating the public host fails, we end up with an orphaned database,
+      # but that's not a big deal- we can eventually see it's empty/unlinked and drop it.
+      org.prepare_database_connections
+      org.create_public_host_cname
+    end
+  end
+end

data/lib/webhookdb/jobs/process_webhook.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "amigo/durable_job"
+require "amigo/queue_backoff_job"
+require "amigo/semaphore_backoff_job"
+require "webhookdb/async/job"
+
+class Webhookdb::Jobs::ProcessWebhook
+  extend Webhookdb::Async::Job
+  include Amigo::DurableJob
+  include Amigo::QueueBackoffJob
+  include Amigo::SemaphoreBackoffJob
+
+  on "webhookdb.serviceintegration.webhook"
+  sidekiq_options queue: "webhook" # This is usually overridden.
+
+  def semaphore_expiry = 5.minutes.to_i
+  def dependent_queues = ["critical"]
+
+  def before_perform(*args)
+    event = Amigo::Event.from_json(args[0])
+    @sint = self.lookup_model(Webhookdb::ServiceIntegration, event)
+  end
+
+  def _perform(event)
+    self.with_log_tags(@sint.log_tags) do
+      kw = event.payload[1].symbolize_keys
+      svc = Webhookdb::Replicator.create(@sint)
+      # kwargs contains: :headers, :body, :request_path, :request_method
+      req = Webhookdb::Replicator::WebhookRequest.new(
+        body: kw.fetch(:body),
+        headers: kw.fetch(:headers),
+        path: kw.fetch(:request_path),
+        method: kw.fetch(:request_method),
+      )
+      svc.upsert_webhook(req)
+    end
+  end
+
+  def semaphore_key
+    return "semaphore-procwebhook-#{@sint.organization_id}"
+  end
+
+  def semaphore_size
+    return @sint.organization.job_semaphore_size
+  end
+end

data/lib/webhookdb/jobs/renew_watch_channel.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+require "webhookdb/jobs"
+
+# Generic helper to renew watch channels, enqueued by replicator-specific jobs
+# like RenewGoogleWatchChannels.
+# Must be emitted with [service integration id, {row_pk:, expirng_before:}]
+# Calls #renew_watch_channel(row_pk:, expiring_before:).
+class Webhookdb::Jobs::RenewWatchChannel
+  extend Webhookdb::Async::Job
+
+  on "webhookdb.serviceintegration.renewwatchchannel"
+
+  def _perform(event)
+    sint = self.lookup_model(Webhookdb::ServiceIntegration, event)
+    self.with_log_tags(sint.log_tags) do
+      opts = event.payload[1]
+      row_pk = opts.fetch("row_pk")
+      expiring_before = Time.parse(opts.fetch("expiring_before"))
+      sint.replicator.renew_watch_channel(row_pk:, expiring_before:)
+    end
+  end
+end

data/lib/webhookdb/jobs/replication_migration.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# See Organization::enqueue_migrate_all_replication_tables for more info.
+class Webhookdb::Jobs::ReplicationMigration
+  include Sidekiq::Worker
+
+  def perform(org_id, target_release_created_at)
+    (org = Webhookdb::Organization[org_id]) or raise "Organization[#{org_id}] does not exist"
+    target_rca = Time.parse(target_release_created_at)
+    current_rca = Time.parse(Webhookdb::RELEASE_CREATED_AT)
+    if target_rca == current_rca
+      self.class.migrate_org(org)
+    elsif target_rca > current_rca
+      self.class.perform_in(1, org_id, target_release_created_at)
+    else
+      self.logger.warn("stale_replication_migration_job", target_release_created_at:)
+    end
+  end
+
+  # To make mocking easier.
+  def self.migrate_org(org)
+    org.migrate_replication_tables
+  end
+end

data/lib/webhookdb/jobs/reset_code_create_dispatch.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+require "webhookdb/messages/verification"
+
+class Webhookdb::Jobs::ResetCodeCreateDispatch
+  extend Webhookdb::Async::Job
+
+  on "webhookdb.customer.resetcode.created"
+
+  def _perform(event)
+    code = self.lookup_model(Webhookdb::Customer::ResetCode, event)
+    Webhookdb::Idempotency.once_ever.under_key("reset-code-#{code.customer_id}-#{code.id}") do
+      msg = Webhookdb::Messages::Verification.new(code)
+      case code.transport
+        when "email"
+          msg.dispatch_email(code.customer)
+      else
+          raise "Unknown transport for #{code.inspect}"
+      end
+    end
+  end
+end

data/lib/webhookdb/jobs/scheduled_backfills.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Webhookdb::Jobs
+  # Create a single way to do the common task of automatic scheduled backfills.
+  # Each integration that needs automated backfills can add a specification here.
+  module ScheduledBackfills
+    Spec = Struct.new(:klass, :service_name, :cron_expr, :splay, :incremental)
+
+    # @param spec [Spec]
+    def self.install(spec)
+      cls = Class.new do
+        extend Webhookdb::Async::ScheduledJob
+
+        cron spec.cron_expr
+        splay spec.splay
+
+        define_method(:_perform) do
+          Webhookdb::ServiceIntegration.dataset.where_each(service_name: spec.service_name) do |sint|
+            Webhookdb::BackfillJob.create(service_integration: sint, incremental: spec.incremental).enqueue
+          end
+        end
+      end
+      Webhookdb::Jobs.const_set(spec.klass, cls)
+    end
+
+    [
+      Spec.new(
+        "ConvertkitBroadcastBackfill", "convertkit_broadcast_v1",
+        "10 * * * *", 2.minutes, false,
+      ),
+      Spec.new(
+        "ConvertkitSubscriberBackfill", "convertkit_subscriber_v1",
+        "20 * * * *", 2.minutes, true,
+      ),
+      Spec.new(
+        "ConvertkitTagBackfill", "convertkit_tag_v1",
+        "30 * * * *", 2.minutes, false,
+      ),
+      Spec.new(
+        "EmailOctopusScheduledBackfill", "email_octopus_list_v1",
+        Webhookdb::EmailOctopus.cron_expression, 2.minutes, false,
+      ),
+      Spec.new(
+        "GithubRepoActivityScheduledBackfill", "github_repository_event_v1",
+        Webhookdb::Github.activity_cron_expression, 30.seconds, false,
+      ),
+      Spec.new(
+        "IntercomScheduledBackfill", "intercom_marketplace_root_v1",
+        "*/1 * * * *", 0, false,
+      ),
+      Spec.new(
+        "AtomSingleFeedPoller", "atom_single_feed_v1",
+        "11 * * * *", 10.seconds, true,
+      ),
+      Spec.new(
+        "SponsyScheduledBackfill", "sponsy_publication_v1",
+        Webhookdb::Sponsy.cron_expression, 30.seconds, true,
+      ),
+      Spec.new(
+        "TransistorEpisodeBackfill", "transistor_episode_v1",
+        Webhookdb::Transistor.episode_cron_expression, 2.minutes, true,
+      ),
+      Spec.new(
+        "TransistorShowBackfill", "transistor_show_v1",
+        Webhookdb::Transistor.show_cron_expression, 2.minutes, true,
+      ),
+      Spec.new(
+        "TwilioSmsBackfill", "twilio_sms_v1",
+        "*/1 * * * *", 0, true,
+      ),
+      Spec.new(
+        "SignalwireMessageBackfill", "signalwire_message_v1",
+        "*/1 * * * *", 0, true,
+      ),
+    ].each { |sp| self.install(sp) }
+  end
+end

data/lib/webhookdb/jobs/send_invite.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+require "webhookdb/messages/invite"
+
+class Webhookdb::Jobs::SendInvite
+  extend Webhookdb::Async::Job
+
+  on "webhookdb.organizationmembership.invite"
+
+  def _perform(event)
+    membership = self.lookup_model(Webhookdb::OrganizationMembership, event)
+    Webhookdb::Messages::Invite.new(membership).dispatch(membership.customer)
+  end
+end

data/lib/webhookdb/jobs/send_test_webhook.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "amigo/queue_backoff_job"
+require "webhookdb/async/job"
+
+class Webhookdb::Jobs::SendTestWebhook
+  extend Webhookdb::Async::Job
+  include Amigo::QueueBackoffJob
+
+  on "webhookdb.webhooksubscription.test"
+  sidekiq_options queue: "netout"
+
+  # If this job fails for a programmer error,
+  # we don't want to retry and randomly send a payload later.
+  sidekiq_options retry: false
+
+  def dependent_queues
+    return ["critical"]
+  end
+
+  def _perform(event)
+    webhook_sub = self.lookup_model(Webhookdb::WebhookSubscription, event)
+    webhook_sub.deliver_test_event
+  end
+end

data/lib/webhookdb/jobs/send_webhook.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "amigo/durable_job"
+require "webhookdb/async/job"
+require "webhookdb/jobs"
+
+class Webhookdb::Jobs::SendWebhook
+  extend Webhookdb::Async::Job
+  include Amigo::DurableJob
+
+  def _perform(event)
+    sint = self.lookup_model(Webhookdb::ServiceIntegration, event)
+    self.with_log_tags(sint.log_tags) do
+      sint.all_webhook_subscriptions_dataset.to_notify.each do |sub|
+        payload = {service_name: sint.service_name, table_name: sint.table_name, **event.payload[1]}
+        sub.enqueue_delivery(payload)
+      end
+    end
+  end
+end

data/lib/webhookdb/jobs/sync_target_enqueue_scheduled.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/job"
+
+class Webhookdb::Jobs::SyncTargetEnqueueScheduled
+  extend Webhookdb::Async::ScheduledJob
+
+  cron "*/1 * * * *"
+  splay 0
+
+  def _perform
+    Webhookdb::SyncTarget.due_for_sync(as_of: Time.now).select(:id, :period_seconds).each do |st|
+      Webhookdb::Jobs::SyncTargetRunSync.perform_in(st.jitter, st.id)
+    end
+  end
+end

data/lib/webhookdb/jobs/sync_target_run_sync.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "amigo/queue_backoff_job"
+require "webhookdb/async/job"
+
+class Webhookdb::Jobs::SyncTargetRunSync
+  extend Webhookdb::Async::Job
+  include Amigo::QueueBackoffJob
+
+  sidekiq_options queue: "netout"
+
+  def dependent_queues
+    return ["critical"]
+  end
+
+  def perform(sync_target_id)
+    stgt = Webhookdb::SyncTarget[sync_target_id]
+    if stgt.nil?
+      # A sync target may be enqueued, but destroyed before the sync runs.
+      # If so, log a warning. We see this on staging a lot,
+      # but it does happen on production too, and should be expected.
+      self.logger.info("missing_sync_target", sync_target_id:)
+      return
+    end
+    self.with_log_tags(
+      sync_target_id: stgt.id,
+      sync_target_connection_url: stgt.displaysafe_connection_url,
+      sync_target_service_integration_service: stgt.service_integration.service_name,
+      sync_target_service_integration_table: stgt.service_integration.table_name,
+    ) do
+      stgt.run_sync(now: Time.now)
+    rescue Webhookdb::SyncTarget::SyncInProgress
+      self.logger.info("sync_target_already_in_progress")
+    rescue Webhookdb::SyncTarget::Deleted
+      self.logger.info("sync_target_deleted")
+    end
+  end
+end

data/lib/webhookdb/jobs/trim_logged_webhooks.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "webhookdb/async/scheduled_job"
+require "webhookdb/jobs"
+
+class Webhookdb::Jobs::TrimLoggedWebhooks
+  extend Webhookdb::Async::ScheduledJob
+
+  cron "17 8 * * *"
+  splay 300
+
+  def _perform
+    Webhookdb::LoggedWebhook.trim
+  end
+end