munawaba 0.1.0

data/app/services/munawaba/conflicts/finder.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Munawaba
+  module Conflicts
+    class Finder
+      # One bound statement provides one snapshot for both conflict branches.
+      SQL = <<~SQL.freeze
+        WITH proposed_slots AS (
+          SELECT * FROM jsonb_to_recordset($1::jsonb) AS proposed(
+            target_schedule_id bigint, target_shift_id bigint, coverage_revision bigint,
+            boundary_index bigint, effective_person_id bigint, starts_at timestamptz, ends_at timestamptz
+          )
+        ), pairs AS (
+          SELECT p.target_schedule_id AS a_schedule, p.target_shift_id AS a_shift,
+            p.coverage_revision AS a_revision, p.boundary_index AS a_boundary,
+            e.schedule_id AS b_schedule, e.id AS b_shift, e.coverage_revision AS b_revision,
+            e.boundary_index AS b_boundary, p.effective_person_id,
+            greatest(p.starts_at, e.starts_at) AS overlap_start, least(p.ends_at,e.ends_at) AS overlap_end
+          FROM proposed_slots p JOIN munawaba_shifts e
+            ON e.effective_person_id = p.effective_person_id AND e.schedule_id <> p.target_schedule_id
+            AND e.canceled_at IS NULL AND tstzrange(e.starts_at,e.ends_at,'[)') && tstzrange(p.starts_at,p.ends_at,'[)')
+          WHERE NOT EXISTS (SELECT 1 FROM proposed_slots r WHERE r.target_shift_id = e.id)
+          UNION ALL
+          SELECT l.target_schedule_id, l.target_shift_id, l.coverage_revision, l.boundary_index,
+            r.target_schedule_id, r.target_shift_id, r.coverage_revision, r.boundary_index, l.effective_person_id,
+            greatest(l.starts_at,r.starts_at), least(l.ends_at,r.ends_at)
+          FROM proposed_slots l JOIN proposed_slots r ON l.effective_person_id = r.effective_person_id
+            AND l.target_schedule_id <> r.target_schedule_id
+            AND (l.target_schedule_id,l.coverage_revision,l.boundary_index) < (r.target_schedule_id,r.coverage_revision,r.boundary_index)
+            AND tstzrange(l.starts_at,l.ends_at,'[)') && tstzrange(r.starts_at,r.ends_at,'[)')
+        ), oriented AS (
+          SELECT CASE WHEN (a_schedule,a_revision,a_boundary) < (b_schedule,b_revision,b_boundary)
+            THEN ARRAY[a_schedule,a_shift,a_revision,a_boundary,b_schedule,b_shift,b_revision,b_boundary,effective_person_id,
+              (extract(epoch FROM overlap_start)*1000000)::bigint,(extract(epoch FROM overlap_end)*1000000)::bigint]
+            ELSE ARRAY[b_schedule,b_shift,b_revision,b_boundary,a_schedule,a_shift,a_revision,a_boundary,effective_person_id,
+              (extract(epoch FROM overlap_start)*1000000)::bigint,(extract(epoch FROM overlap_end)*1000000)::bigint]
+            END AS fingerprint FROM pairs
+        ) SELECT array_to_json(fingerprint)::text AS fingerprint FROM oriented ORDER BY fingerprint;
+      SQL
+
+      def self.call(slots:)
+        return [] if slots.empty?
+
+        vector = slots.map do |slot|
+          { target_schedule_id: slot.fetch(:schedule_id), target_shift_id: slot[:id],
+            coverage_revision: slot.fetch(:coverage_revision), boundary_index: slot.fetch(:boundary_index),
+            effective_person_id: slot.fetch(:effective_person_id), starts_at: Canonical.instant(slot.fetch(:starts_at)), ends_at: Canonical.instant(slot.fetch(:ends_at)) }
+        end
+        bind = ActiveRecord::Relation::QueryAttribute.new("proposed_slots", JSON.generate(vector), ActiveRecord::Type::String.new)
+        rows = Shift.connection.exec_query(SQL, "Munawaba conflicts", [bind])
+        rows.map { |row| JSON.parse(row.fetch("fingerprint")) }.uniq.sort_by { |entry|
+          entry.map { |value|
+            value.nil? ? -1 : value
+          }
+        }.freeze
+      end
+    end
+  end
+end

data/app/services/munawaba/integrations/planner_effects.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Munawaba
+  module Integrations
+    class PlannerEffects
+      def self.replace_epoch(schedule, old_intents, now)
+        planned_count = 0
+        old_intents.each do |row|
+          valid = Notifications::Validity.call(row, now: now, allow_old_notification_revision: true)
+          status = valid.status == :canceled ? :canceled : :stale
+          if row.kind != "test" && valid.valid? && %w[assignment_change next_assignment_change].include?(row.kind)
+            values = row.attributes.slice("shift_id", "kind", "coverage_revision", "assignment_version",
+                                          "timing_version", "rotation_revision", "expires_at")
+            context = Notifications::Context::V1.validate!(kind: row.kind, context: row.context)
+            NotificationDelivery.create!(**values, schedule_id: schedule.id, notification_revision: schedule.notification_revision,
+                                                   event_key: row.event_key.sub(/notification:\d+/, "notification:#{schedule.notification_revision}"), context: context,
+                                                   due_at: now, next_attempt_at: now, status: "pending")
+            planned_count += 1
+          end
+          row.finish!(status: status, code: valid.code || "notification_changed", now: now)
+        end
+        planned = Notifications::Planner.call(schedule: schedule, now: now, operation: "integration")
+        Notifications::WakeDispatcher.call if planned_count.positive? && planned.zero?
+      end
+    end
+  end
+end

data/app/services/munawaba/integrations.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Munawaba
+  module Integrations
+    class Invalid < Munawaba::Error; end
+    SETTINGS = %w[slack_enabled notify_advance advance_notice_seconds notify_shift_start notify_assignment_change
+                  notify_next_assignment_change].freeze
+
+    def self.call(operation:, schedule:, attributes: {}, actor: nil)
+      operation = operation.to_s
+      attributes = attributes.to_h.stringify_keys.slice(*SETTINGS, "slack_webhook_url", "lock_version")
+      Schedule.transaction(requires_new: true) do
+        current = Schedule.where(id: schedule.id).lock("FOR NO KEY UPDATE").first!
+        check_version = attributes.key?("lock_version") && (operation == "update" || attributes["lock_version"])
+        expected = attributes.delete("lock_version")
+        if check_version && Integer(expected, exception: false) != current.lock_version
+          message = operation == "update" ? "Review and save again." : "Review them again."
+          next Result.new(status: 409, record: current, errors: ["Integration settings changed. #{message}"])
+        end
+
+        now = Time.current
+        case operation
+        when "update" then update(current, attributes, actor, now)
+        when "remove" then remove(current, actor, now)
+        when "test" then test(current, actor, now)
+        else raise ArgumentError, "Unknown integration operation"
+        end
+        Result.new(status: 303, record: current, errors: [])
+      end
+    rescue ActiveRecord::RecordInvalid => error
+      Result.new(status: 422, record: operation == "update" ? error.record : schedule,
+                 errors: error.record.errors.full_messages)
+    rescue Invalid => error
+      Result.new(status: 422, record: schedule, errors: [error.message])
+    end
+
+    def self.update(current, attributes, actor, now)
+      before = current.attributes.slice(*SETTINGS)
+      old_timestamp = current.slack_webhook_configured_at
+      webhook = attributes.delete("slack_webhook_url")
+      if webhook.present?
+        current.slack_webhook_url = webhook
+        current.slack_webhook_configured_at = now
+      end
+      current.assign_attributes(attributes)
+      return unless current.changed?
+
+      save(current, now)
+      changed = SETTINGS.select { |key| before[key] != current.public_send(key) }
+      event_type = if webhook.present?
+                     old_timestamp ? "slack.replaced" : "slack.configured"
+                   else
+                     "slack.settings_changed"
+                   end
+      Audit::Recorder.record!(event_type: event_type, schedule: current, actor: actor, occurred_at: now,
+                              metadata: { changed_fields: changed, before: before.slice(*changed),
+                                          after: current.attributes.slice(*changed) })
+    end
+
+    def self.remove(current, actor, now)
+      raise Invalid, "No webhook is configured." unless current.slack_webhook_configured_at
+
+      enabled = current.slack_enabled
+      current.assign_attributes(slack_webhook_url: nil, slack_webhook_configured_at: nil, slack_enabled: false)
+      save(current, now)
+      Audit::Recorder.record!(event_type: "slack.removed", schedule: current, actor: actor, occurred_at: now,
+                              metadata: { enabled_before: enabled })
+    end
+
+    def self.test(current, actor, now)
+      unless current.slack_enabled && current.slack_webhook_configured_at
+        raise Invalid, "Configure and enable Slack first."
+      end
+
+      row = NotificationDelivery.create!(schedule: current, kind: "test", status: "pending",
+                                         event_key: "schedule:#{current.id}:test:notification:#{current.notification_revision}:#{SecureRandom.uuid}",
+                                         notification_revision: current.notification_revision, context: Notifications::Context::V1.build(kind: "test"),
+                                         due_at: now, next_attempt_at: now, expires_at: now + 10.minutes)
+      Audit::Recorder.record!(event_type: "slack.test_requested", schedule: current, actor: actor, occurred_at: now,
+                              metadata: { delivery_id: row.id })
+      Notifications::WakeDispatcher.call
+    end
+
+    def self.save(current, now)
+      old_intents = current.notification_deliveries.unsent.order(:id).lock("FOR NO KEY UPDATE").to_a
+      current.notification_revision += 1
+      current.save!
+      PlannerEffects.replace_epoch(current, old_intents, now)
+    end
+
+    private_class_method :update, :remove, :test, :save
+  end
+end

data/app/services/munawaba/notifications/context/v1.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Munawaba
+  module Notifications
+    module Context
+      class V1
+        class Invalid < Munawaba::Error
+          def initialize(*) = super("Invalid delivery context")
+        end
+
+        OVERRIDES = {
+          "override_created" => %w[override_id],
+          "override_superseded" => %w[override_id ended_override_id],
+          "override_revoked" => %w[ended_override_id],
+          "override_restored_to_base" => %w[ended_override_id]
+        }.freeze
+
+        def self.validate!(kind:, context:)
+          raise Invalid unless context.is_a?(Hash) && context.keys.all? { |key| key.is_a?(String) }
+          raise Invalid unless context["schema_version"].is_a?(Integer) && context["schema_version"] == 1
+
+          keys = ["schema_version"]
+          case kind.to_s
+          when "advance_reminder", "shift_start", "test"
+            nil
+          when "assignment_change"
+            transition = OVERRIDES[context["transition_type"]]
+            raise Invalid unless transition
+
+            keys += %w[transition_type previous_person_id new_person_id] + transition
+            raise Invalid if context["previous_person_id"] == context["new_person_id"]
+          when "next_assignment_change"
+            # Accept the old activity reference without requiring activity rows to exist.
+            context = context.except("operation_id")
+            keys += %w[previous_next_person_id new_next_person_id described_shift_id]
+            raise Invalid if context["previous_next_person_id"] == context["new_next_person_id"]
+          else
+            raise Invalid
+          end
+          raise Invalid unless keys.sort == context.keys.sort
+
+          keys.grep(/_id\z/).each do |key|
+            raise Invalid unless context[key].is_a?(Integer) && context[key] > 0
+          end
+          raise Invalid if JSON.generate(context).bytesize > 16_384
+
+          context.slice(*keys).freeze
+        rescue JSON::GeneratorError, EncodingError
+          raise Invalid
+        end
+
+        def self.valid?(kind:, context:)
+          validate!(kind: kind, context: context)
+          true
+        rescue Invalid
+          false
+        end
+
+        def self.build(kind:, **values)
+          validate!(kind: kind, context: { "schema_version" => 1 }.merge(values.stringify_keys))
+        end
+      end
+    end
+  end
+end

data/app/services/munawaba/notifications/deliver.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Munawaba
+  module Notifications
+    class Deliver
+      def self.call(delivery_id, claim_token)
+        row = NotificationDelivery.find_by(id: delivery_id, claim_token: claim_token, status: "enqueued")
+        return unless row && row.lease_expires_at > Time.current
+        unless Munawaba.config.notifications_enabled?
+          return Dispatcher.reset(delivery_id, claim_token, Time.current, signal: false)
+        end
+
+        check = Validity.call(row)
+        unless check.valid?
+          reject_claim(delivery_id, claim_token, check)
+          return
+        end
+        payload = Slack::Renderer.call(delivery: row, schedule: check.schedule, shift: check.shift)
+        webhook = check.schedule.slack_webhook_url
+        deadline = Slack::Client.monotonic + Defaults::SLACK_TOTAL_TIMEOUT.to_f
+        authorized = NotificationDelivery.transaction do
+          current = NotificationDelivery.where(id: delivery_id, claim_token: claim_token,
+                                               status: "enqueued").lock("FOR NO KEY UPDATE").first
+          now = Time.current
+          next false unless current && current.lease_expires_at > now
+
+          unless Munawaba.config.notifications_enabled?
+            current.mark_pending!(at: now)
+            next false
+          end
+          final = Validity.call(current, now: now)
+          unless final.valid?
+            current.finish!(status: final.status, code: final.code, now: now)
+            request_timing_maintenance(current.schedule_id) if final.code == "timing_rules_changed"
+            next false
+          end
+          current.update!(status: "processing", processing_at: now, lease_expires_at: now + Defaults::NOTIFICATION_PROCESSING_LEASE,
+                          attempt_count: current.attempt_count + 1, last_attempt_at: now, last_http_status: nil,
+                          last_error_code: current.last_error_code == "delivery_outcome_unknown" ? current.last_error_code : nil)
+          true
+        end
+        return unless authorized
+
+        outcome = Slack::Client.call(webhook: webhook, payload: payload, deadline: deadline)
+        complete(delivery_id, claim_token, outcome)
+      rescue Context::V1::Invalid, ActiveRecord::Encryption::Errors::Base, Slack::WebhookValidator::Invalid
+        reject_claim(delivery_id, claim_token, Validity::Result.new(status: :stale, code: "invalid_delivery_context"))
+      end
+
+      def self.complete(id, token, outcome)
+        NotificationDelivery.transaction do
+          row = NotificationDelivery.where(id: id, claim_token: token,
+                                           status: "processing").lock("FOR NO KEY UPDATE").first
+          unless row
+            Munawaba.signal("notification_late_result")
+            next
+          end
+          now = Time.current
+          row.last_http_status = outcome.http_status
+          if outcome.status == :delivered
+            row.finish!(status: :delivered, now: now, http_status: outcome.http_status)
+          elsif outcome.status == :failed
+            row.last_error_code = "delivery_outcome_unknown" if outcome.unknown
+            row.finish!(status: :failed, code: outcome.error_code, now: now, http_status: outcome.http_status)
+          else
+            row.retry_later!(now: now, code: outcome.error_code, retry_after: outcome.retry_after,
+                             unknown: outcome.unknown)
+          end
+        end
+      end
+
+      def self.reject_claim(id, token, result)
+        NotificationDelivery.transaction do
+          row = NotificationDelivery.where(id: id, claim_token: token,
+                                           status: "enqueued").lock("FOR NO KEY UPDATE").first
+          if row
+            row.finish!(status: result.status, code: result.code)
+            request_timing_maintenance(row.schedule_id) if result.code == "timing_rules_changed"
+          end
+        end
+      end
+
+      def self.request_timing_maintenance(schedule_id)
+        ActiveRecord.after_all_transactions_commit do
+          begin
+            MaintenanceJob.perform_later(schedule_id)
+          rescue StandardError
+            Munawaba.signal("timing_wake_failed")
+          end
+        end
+      end
+    end
+  end
+end

data/app/services/munawaba/notifications/dispatcher.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Munawaba
+  module Notifications
+    class Dispatcher
+      def self.call(now: Time.current)
+        return 0 unless Munawaba.config.notifications_enabled?
+
+        limit = Defaults::NOTIFICATION_BATCH_SIZE
+        NotificationDelivery.transaction do
+          NotificationDelivery.where(status: "pending").where("expires_at <= ?", now)
+                              .order(:expires_at, :id).limit(limit).lock("FOR UPDATE SKIP LOCKED").each do |delivery|
+            delivery.finish!(status: :stale, code: "expired", now: now)
+          end
+        end
+        claimed = NotificationDelivery.transaction do
+          next [] unless Munawaba.config.notifications_enabled?
+
+          NotificationDelivery.where(status: "pending").where("next_attempt_at <= ?", now)
+                              .order(:next_attempt_at, :id).limit(limit).lock("FOR UPDATE SKIP LOCKED").map do |delivery|
+            token = SecureRandom.uuid
+            delivery.update!(status: "enqueued", next_attempt_at: nil, claim_token: token,
+                             enqueued_at: now, processing_at: nil, lease_expires_at: now + Defaults::NOTIFICATION_ENQUEUE_LEASE)
+            [delivery.id, token]
+          end
+        end
+        claimed.each do |id, token|
+          begin
+            job = DeliverNotificationJob.perform_later(id, token)
+            reset(id, token, now) unless job && job.successfully_enqueued?
+          rescue StandardError
+            reset(id, token, now)
+          end
+        end
+        claimed.size
+      end
+
+      def self.reset(id, token, now, signal: true)
+        NotificationDelivery.transaction do
+          row = NotificationDelivery.where(id: id, claim_token: token,
+                                           status: "enqueued").lock("FOR NO KEY UPDATE").first
+          row.mark_pending!(at: now) if row
+        end
+        Munawaba.signal("notification_enqueue_failed") if signal
+      rescue ActiveRecord::ActiveRecordError
+        Munawaba.signal("notification_enqueue_reset_failed")
+      end
+    end
+  end
+end

data/app/services/munawaba/notifications/planner.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module Munawaba
+  module Notifications
+    class Planner
+      def self.call(schedule:, now:, operation:, shifts: [], previous_person_id: nil, override: nil,
+                    ended_override: nil, invalidation_shift_ids: nil)
+        new(schedule, now).call(operation: operation.to_s, shifts: shifts, previous_person_id: previous_person_id,
+                                override: override, ended_override: ended_override, invalidation_shift_ids: invalidation_shift_ids)
+      end
+
+      def initialize(schedule, now)
+        @schedule, @now = schedule, now
+        @planned_count = 0
+      end
+
+      def call(operation:, shifts:, previous_person_id:, override:, ended_override:,
+               invalidation_shift_ids: nil)
+        invalidate(invalidation_shift_ids)
+        rows = shifts.present? ? shifts : @schedule.shifts.live.where(coverage_revision: @schedule.coverage_revision).where(
+          "ends_at > ?", @now
+        ).order(:id)
+        rows = rows.to_a
+        rows.each { |shift| plan_shift(shift) }
+        if %w[override revoke restore_to_base].include?(operation) && previous_person_id && rows.one?
+          plan_assignment(rows.first, previous_person_id, override, ended_override)
+        elsif %w[rotation deactivate].include?(operation) && previous_person_id
+          shift = @schedule.shifts.live.where(coverage_revision: @schedule.coverage_revision).where("starts_at > ?", @now).order(
+            :starts_at, :id
+          ).first
+          plan_next(shift, previous_person_id) if shift
+        end
+        WakeDispatcher.call if @planned_count.positive?
+        @planned_count
+      end
+
+      def enabled?(kind)
+        return false unless @schedule.slack_webhook_configured_at
+        return false unless @schedule.slack_enabled
+
+        setting = Validity::KIND_SETTINGS[kind]
+        return false if setting && !@schedule.public_send(setting)
+
+        true
+      end
+
+      def invalidate(shift_ids = nil)
+        scope = @schedule.notification_deliveries.unsent
+        if shift_ids
+          # Compare JSON IDs as text so malformed context cannot cause a SQL cast error.
+          scope = scope.where(
+            "shift_id IN (?) OR (kind = 'next_assignment_change' AND context->>'described_shift_id' IN (?))", shift_ids, shift_ids.map(&:to_s)
+          )
+        end
+        scope.order(:id).lock("FOR NO KEY UPDATE").each do |delivery|
+          validity = Validity.call(delivery, now: @now)
+          next if validity.valid?
+
+          replan_transition(delivery) if validity.code == "timing_changed"
+          delivery.finish!(status: validity.status, code: validity.code, now: @now)
+        end
+      end
+
+      def plan_shift(shift)
+        return if shift.canceled_at || shift.ends_at <= @now || shift.coverage_revision != @schedule.coverage_revision
+        return unless %w[scheduled active pausing].include?(@schedule.state)
+
+        create_shift_intent(shift, "advance_reminder", due_at: shift.starts_at - @schedule.advance_notice_seconds,
+                                                       expires_at: shift.starts_at)
+        create_shift_intent(shift, "shift_start", due_at: shift.starts_at,
+                                                  expires_at: shift.starts_at + Munawaba::Defaults::START_NOTIFICATION_GRACE_PERIOD)
+      end
+
+      def create_shift_intent(shift, kind, due_at:, expires_at:, context: Context::V1.build(kind: kind))
+        return unless enabled?(kind)
+        return if expires_at <= @now
+
+        key = "shift:#{shift.id}:#{kind}:assignment:#{shift.assignment_version}:timing:#{shift.timing_version}:notification:#{@schedule.notification_revision}"
+        create(kind: kind, event_key: key, shift_id: shift.id, coverage_revision: shift.coverage_revision,
+               assignment_version: shift.assignment_version, timing_version: shift.timing_version,
+               context: context, due_at: [due_at, expires_at - 0.000001].min, expires_at: expires_at)
+      end
+
+      def plan_assignment(shift, previous, override, ended)
+        return if previous.to_i == shift.effective_person_id
+
+        transition = ended ? "override_#{ended.end_reason}" : "override_created"
+        values = { transition_type: transition, previous_person_id: previous.to_i,
+                   new_person_id: shift.effective_person_id }
+        values[:override_id] = override.id if override
+        values[:ended_override_id] = ended.id if ended
+        origin = override&.created_at || ended&.ended_at
+        return unless origin
+
+        context = Context::V1.build(kind: "assignment_change", **values)
+        create_shift_intent(shift, "assignment_change", due_at: origin,
+                                                        expires_at: [shift.ends_at, origin + 24.hours].min, context: context)
+      end
+
+      def plan_next(shift, previous)
+        return if previous.to_i == shift.base_person_id
+        return unless enabled?("next_assignment_change")
+
+        expires_at = [shift.starts_at, @now + 24.hours].min
+        return if expires_at <= @now
+
+        context = Context::V1.build(kind: "next_assignment_change",
+                                    previous_next_person_id: previous.to_i, new_next_person_id: shift.base_person_id, described_shift_id: shift.id)
+        create(kind: "next_assignment_change", event_key: next_key(shift), context: context, coverage_revision: shift.coverage_revision,
+               rotation_revision: @schedule.rotation_revision, timing_version: shift.timing_version, due_at: @now, expires_at: expires_at)
+      end
+
+      def next_key(shift)
+        "schedule:#{@schedule.id}:next_assignment_change:coverage:#{@schedule.coverage_revision}:rotation:#{@schedule.rotation_revision}:timing:#{shift.timing_version}:notification:#{@schedule.notification_revision}"
+      end
+
+      def replan_transition(delivery)
+        return unless %w[assignment_change next_assignment_change].include?(delivery.kind)
+        return unless enabled?(delivery.kind)
+
+        context = Context::V1.validate!(kind: delivery.kind, context: delivery.context)
+        shift = Shift.find(delivery.shift_id || context.fetch("described_shift_id"))
+        candidate = delivery.dup
+        candidate.timing_version = shift.timing_version
+        candidate.notification_revision = @schedule.notification_revision
+        return unless Validity.call(candidate, now: @now).valid?
+
+        if delivery.kind == "assignment_change"
+          transition = ShiftOverride.find(context["override_id"] || context.fetch("ended_override_id"))
+          origin = context["override_id"] ? transition.created_at : transition.ended_at
+          create_shift_intent(shift, delivery.kind, due_at: origin, expires_at: [shift.ends_at, origin + 24.hours].min,
+                                                    context: context)
+        else
+          expiry = [shift.starts_at, delivery.created_at + 24.hours].min
+          return if expiry <= @now
+
+          create(kind: delivery.kind, event_key: next_key(shift), context: context, coverage_revision: shift.coverage_revision,
+                 rotation_revision: @schedule.rotation_revision, timing_version: shift.timing_version, due_at: delivery.due_at, expires_at: expiry)
+        end
+      rescue Context::V1::Invalid, ActiveRecord::RecordNotFound
+        nil
+      end
+
+      def create(**attributes)
+        return if NotificationDelivery.exists?(event_key: attributes.fetch(:event_key))
+
+        row = NotificationDelivery.create!(**attributes, schedule_id: @schedule.id, notification_revision: @schedule.notification_revision,
+                                                         status: "pending", next_attempt_at: [attributes.fetch(:due_at), @now].max)
+        @planned_count += 1
+        row
+      end
+    end
+  end
+end

data/app/services/munawaba/notifications/reclaim_expired_leases.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Munawaba
+  module Notifications
+    class ReclaimExpiredLeases
+      def self.call(now: Time.current)
+        NotificationDelivery.transaction do
+          NotificationDelivery.where(status: %w[enqueued processing]).where("lease_expires_at <= ?", now)
+                              .order(:lease_expires_at, :id).limit(Defaults::NOTIFICATION_BATCH_SIZE)
+                              .lock("FOR UPDATE SKIP LOCKED").each do |row|
+            was_processing = row.status == "processing"
+            row.last_error_code = "delivery_outcome_unknown" if was_processing
+            validity = Validity.call(row, now: now)
+            if !validity.valid?
+              row.finish!(status: validity.status, code: validity.code, now: now)
+              Deliver.request_timing_maintenance(row.schedule_id) if validity.code == "timing_rules_changed"
+            elsif was_processing
+              row.retry_later!(now: now, code: "delivery_outcome_unknown", unknown: true, validity: validity)
+            else
+              row.mark_pending!(at: now)
+            end
+          end.size
+        end
+      end
+    end
+  end
+end

data/app/services/munawaba/notifications/retry_failed.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Munawaba
+  module Notifications
+    class RetryFailed
+      def self.call(delivery:, actor: nil, acknowledge_duplicate: false)
+        reference = NotificationDelivery.uncached { NotificationDelivery.find(delivery.id) }
+        parsed = Context::V1.validate!(kind: reference.kind, context: reference.context)
+        described_shift_id = reference.shift_id || parsed["described_shift_id"]
+        result = nil
+        NotificationDelivery.transaction(requires_new: true) do
+          schedule = Schedule.where(id: reference.schedule_id).lock("FOR NO KEY UPDATE").first!
+          shift = Shift.where(id: described_shift_id).lock("FOR NO KEY UPDATE").first! if described_shift_id
+          original = NotificationDelivery.where(id: reference.id).lock("FOR NO KEY UPDATE").first!
+          authoritative_context = Context::V1.validate!(kind: original.kind, context: original.context)
+          unless original.schedule_id == schedule.id &&
+                 (original.shift_id || authoritative_context["described_shift_id"]) == described_shift_id &&
+                 (!shift || shift.schedule_id == schedule.id)
+            raise Context::V1::Invalid
+          end
+
+          now = Time.current
+          validity = Validity.call(original, now: now, allow_old_notification_revision: true)
+          if original.status != "failed" || !validity.valid? || NotificationDelivery.exists?(retry_of_delivery_id: original.id)
+            result = Result.new(status: 422, record: original,
+                                errors: ["This delivery is no longer eligible for retry."])
+            next
+          end
+          if original.last_error_code == "delivery_outcome_unknown" && !ActiveModel::Type::Boolean.new.cast(acknowledge_duplicate)
+            result = Result.new(status: 422, record: original,
+                                errors: ["Acknowledge that retrying may post a duplicate message."])
+            next
+          end
+          values = original.attributes.slice("shift_id", "kind", "coverage_revision", "assignment_version",
+                                             "timing_version", "rotation_revision", "expires_at")
+          successor = NotificationDelivery.create!(**values, schedule: schedule, retry_of_delivery_id: original.id,
+                                                             notification_revision: schedule.notification_revision, context: authoritative_context, event_key: "retry:#{original.id}:#{SecureRandom.uuid}",
+                                                             status: "pending", due_at: now, next_attempt_at: now)
+          Audit::Recorder.record!(event_type: "notification.manual_retry_requested", schedule: schedule, shift_id: original.shift_id,
+                                  actor: actor, occurred_at: now, metadata: { predecessor_delivery_id: original.id, successor_delivery_id: successor.id,
+                                                                              kind: original.kind })
+          WakeDispatcher.call
+          result = Result.new(status: 303, record: successor, errors: [])
+        end
+        result
+      rescue Context::V1::Invalid, ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique, ActiveRecord::RecordNotFound
+        Result.new(status: 422, record: delivery, errors: ["This delivery cannot be retried."])
+      end
+    end
+  end
+end