cronwatch 0.3.1 → 0.4.0

data/lib/cronwatch/alerts/resend.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Cronwatch
+  module Alerts
+    # Sends alerts as email through Resend
+    # (https://resend.com/docs/api-reference/emails/send-email).
+    #
+    #   Cronwatch::Alerts::Resend.new(api_key: ENV.fetch("RESEND_API_KEY"),
+    #                                 from: "CronWatch <alerts@example.com>", to: "ops@example.com")
+    class Resend
+      ENDPOINT = "https://api.resend.com/emails"
+
+      attr_reader :name
+
+      def initialize(api_key:, from:, to:, subject_prefix: nil, link: nil, http: HTTP.default)
+        @api_key = Provider.require_credential(api_key, "Cronwatch::Alerts::Resend needs an api_key")
+        @to = Email.recipients("Resend", from, to)
+        @from = from
+        @subject_prefix = subject_prefix
+        @link = link
+        @http = http
+        @name = "resend"
+      end
+
+      def call(alert)
+        email = Email.compose(alert, from: @from, to: @to, subject_prefix: @subject_prefix, link: @link)
+        headers = {
+          "content-type" => "application/json",
+          "authorization" => "Bearer #{@api_key}",
+          # The same alert sent twice within 24 hours is delivered once.
+          "idempotency-key" => "cronwatch-#{Provider.alert_id(alert)}",
+        }
+        body = JS.json({ "from" => email.from, "to" => email.to, "subject" => email.subject, "text" => email.text, "html" => email.html })
+        Provider.post(@http, "Resend", ENDPOINT, headers, body, [@api_key])
+        nil
+      end
+    end
+  end
+end

data/lib/cronwatch/alerts/rollbar.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Cronwatch
+  module Alerts
+    # Reports alerts to Rollbar, one item per job and alert type
+    # (https://docs.rollbar.com/reference/create-item). `access_token` needs
+    # the post_server_item scope. Recoveries go too, as info items, unless
+    # `recovered: false`.
+    class Rollbar
+      ENDPOINT = "https://api.rollbar.com/api/1/item/"
+
+      attr_reader :name
+
+      def initialize(access_token:, environment: nil, recovered: true, link: nil, http: HTTP.default)
+        @access_token = Provider.require_credential(access_token, "Cronwatch::Alerts::Rollbar needs an access_token")
+        @environment = environment
+        @recovered = recovered
+        @link = link
+        @http = http
+        @name = "rollbar"
+      end
+
+      def call(alert)
+        return if alert.type == :recovered && @recovered == false
+
+        link = Provider.link_for(@link, alert)
+        type = alert.type.to_s
+        custom = { "job" => alert.job, "type" => type }
+        custom["triage"] = alert.triage if Provider.present?(alert.triage)
+        custom["link"] = link if link
+        custom["details"] = Naming.to_json_value(alert.details)
+        custom["run"] = Provider.run_summary(alert)
+        item = {
+          "data" => {
+            "environment" => Provider.cut(@environment.nil? ? "production" : @environment.to_s, 255),
+            "level" => Provider.severity(alert.type),
+            "timestamp" => alert.at.div(1000),
+            "title" => Provider.cut(alert.title, 255),
+            # Rollbar hashes a fingerprint longer than 40 characters itself.
+            "fingerprint" => "cronwatch:#{alert.job}:#{type}",
+            "uuid" => Provider.as_uuid(Provider.alert_id(alert)),
+            "body" => { "message" => { "body" => alert.message } },
+            "custom" => custom,
+            "notifier" => { "name" => "cronwatch" },
+          },
+        }
+        headers = { "content-type" => "application/json", "x-rollbar-access-token" => @access_token }
+        Provider.post(@http, "Rollbar", ENDPOINT, headers, JS.json(item), [@access_token])
+        nil
+      end
+    end
+  end
+end

data/lib/cronwatch/alerts/sendgrid.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Cronwatch
+  module Alerts
+    # Sends alerts as email through SendGrid
+    # (https://www.twilio.com/docs/sendgrid/api-reference/mail-send/mail-send).
+    # `region: "eu"` for an EU regional subuser.
+    class Sendgrid
+      attr_reader :name
+
+      def initialize(api_key:, from:, to:, region: nil, subject_prefix: nil, link: nil, http: HTTP.default)
+        @api_key = Provider.require_credential(api_key, "Cronwatch::Alerts::Sendgrid needs an api_key")
+        @to = Email.recipients("Sendgrid", from, to)
+        @from = from
+        @url = region.to_s == "eu" ? "https://api.eu.sendgrid.com/v3/mail/send" : "https://api.sendgrid.com/v3/mail/send"
+        @subject_prefix = subject_prefix
+        @link = link
+        @http = http
+        @name = "sendgrid"
+      end
+
+      def call(alert)
+        email = Email.compose(alert, from: @from, to: @to, subject_prefix: @subject_prefix, link: @link)
+        body = JS.json({
+          "personalizations" => [{ "to" => email.to.map { |a| Email.parse_address(a) } }],
+          "from" => Email.parse_address(email.from),
+          "subject" => email.subject,
+          # text/plain must come before text/html.
+          "content" => [
+            { "type" => "text/plain", "value" => email.text },
+            { "type" => "text/html", "value" => email.html },
+          ],
+          "categories" => ["cronwatch"],
+        })
+        headers = { "content-type" => "application/json", "authorization" => "Bearer #{@api_key}" }
+        Provider.post(@http, "SendGrid", @url, headers, body, [@api_key])
+        nil
+      end
+    end
+  end
+end

data/lib/cronwatch/alerts/sentry.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Cronwatch
+  module Alerts
+    # Sends alerts to Sentry as events through the envelope endpoint, one
+    # issue per job and alert type. Recoveries go too, as info events, unless
+    # `recovered: false`.
+    #
+    #   Cronwatch::Alerts::Sentry.new(dsn: ENV.fetch("SENTRY_DSN"), environment: "production")
+    #
+    # Envelopes: https://develop.sentry.dev/sdk/data-model/envelopes/
+    # Event payload: https://develop.sentry.dev/sdk/data-model/event-payloads/
+    class Sentry
+      Dsn = Struct.new(:endpoint, :public_key, keyword_init: true)
+
+      attr_reader :name
+
+      def initialize(dsn:, environment: nil, release: nil, recovered: true, link: nil, http: HTTP.default)
+        dsn = Provider.require_credential(dsn, "Cronwatch::Alerts::Sentry needs a dsn")
+        parsed = Sentry.parse_dsn(dsn)
+        @endpoint = parsed.endpoint
+        @public_key = parsed.public_key
+        @environment = environment
+        @release = release
+        @recovered = recovered
+        @link = link
+        @http = http
+        @name = "sentry"
+      end
+
+      # "https://<key>@<host>/<project>" as the envelope endpoint and the public key.
+      def self.parse_dsn(dsn)
+        uri = begin
+          URI.parse(dsn.to_s)
+        rescue URI::Error
+          nil
+        end
+        raise ArgumentError, "Cronwatch::Alerts::Sentry needs a valid dsn" unless uri&.scheme && uri.host
+
+        segments = uri.path.to_s.split("/").reject(&:empty?)
+        project = segments.pop
+        user = uri.user
+        if user.nil? || user.empty? || project.nil? || !/\A\d+\z/.match?(project)
+          raise ArgumentError, "Cronwatch::Alerts::Sentry needs a dsn like https://<key>@<host>/<project>"
+        end
+
+        prefix = segments.empty? ? "" : "/#{segments.join("/")}"
+        Dsn.new(endpoint: "#{uri.scheme.downcase}://#{SigV4.host(uri).downcase}#{prefix}/api/#{project}/envelope/",
+                public_key: URI.decode_uri_component(user))
+      end
+
+      def call(alert)
+        return if alert.type == :recovered && @recovered == false
+
+        event_id = Provider.alert_id(alert)
+        link = Provider.link_for(@link, alert)
+        event = {
+          "event_id" => event_id,
+          "timestamp" => alert.at / 1000.0,
+          "platform" => "other",
+          "level" => Provider.severity(alert.type),
+          "logger" => "cronwatch",
+          "transaction" => alert.job,
+          "environment" => @environment.nil? ? "production" : @environment,
+        }
+        event["release"] = @release if Provider.present?(@release)
+        # The first line is the issue title.
+        event["logentry"] = { "formatted" => Provider.cut("#{alert.title}\n\n#{alert.message}", 8192) }
+        event["fingerprint"] = ["cronwatch", alert.job, alert.type.to_s]
+        event["tags"] = { "job" => Provider.cut(alert.job, 199), "type" => alert.type.to_s }
+        extra = {}
+        extra["triage"] = alert.triage if Provider.present?(alert.triage)
+        extra["link"] = link if link
+        extra["details"] = Naming.to_json_value(alert.details)
+        extra["run"] = Provider.run_summary(alert)
+        event["extra"] = extra
+        payload = JS.json(event)
+        envelope = [
+          JS.json({ "event_id" => event_id }),
+          JS.json({ "type" => "event", "content_type" => "application/json", "length" => payload.bytesize }),
+          payload,
+        ].join("\n") + "\n"
+        headers = {
+          "content-type" => "application/x-sentry-envelope",
+          "x-sentry-auth" => "Sentry sentry_version=7, sentry_key=#{@public_key}, sentry_client=cronwatch",
+        }
+        Provider.post(@http, "Sentry", @endpoint, headers, envelope, [@public_key])
+        nil
+      end
+    end
+  end
+end

data/lib/cronwatch/alerts/ses.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Cronwatch
+  module Alerts
+    # Sends alerts as email through Amazon SES, API v2 SendEmail
+    # (https://docs.aws.amazon.com/ses/latest/APIReference-V2/API_SendEmail.html),
+    # signed with AWS Signature Version 4 (see SigV4), so no AWS SDK is
+    # needed. The from identity must be verified in `region`.
+    #
+    #   Cronwatch::Alerts::Ses.new(region: "us-east-1", access_key_id: ENV.fetch("AWS_ACCESS_KEY_ID"),
+    #                              secret_access_key: ENV.fetch("AWS_SECRET_ACCESS_KEY"),
+    #                              from: "alerts@example.com", to: "ops@example.com")
+    class Ses
+      attr_reader :name
+
+      # now: the clock used to sign requests, a callable returning epoch milliseconds. For tests.
+      def initialize(region:, access_key_id:, secret_access_key:, from:, to:, session_token: nil, configuration_set_name: nil,
+                     subject_prefix: nil, link: nil, now: nil, http: HTTP.default)
+        region = Provider.require_option(region, "Cronwatch::Alerts::Ses needs a region")
+        raise ArgumentError, "Cronwatch::Alerts::Ses needs a region like us-east-1" unless /\A[a-z0-9-]+\z/.match?(region)
+        # A pasted credential often carries a stray space or newline, which would spoil the signature.
+        access_key_id = Provider.trimmed(access_key_id)
+        secret_access_key = Provider.trimmed(secret_access_key)
+        if access_key_id.empty? || secret_access_key.empty?
+          raise ArgumentError, "Cronwatch::Alerts::Ses needs an access_key_id and secret_access_key"
+        end
+
+        @to = Email.recipients("Ses", from, to)
+        @region = region
+        @access_key_id = access_key_id
+        @secret_access_key = secret_access_key
+        session_token = Provider.trimmed(session_token)
+        @session_token = session_token.empty? ? nil : session_token
+        @configuration_set_name = configuration_set_name
+        @from = from
+        @subject_prefix = subject_prefix
+        @link = link
+        @now = now || -> { Process.clock_gettime(Process::CLOCK_REALTIME, :millisecond) }
+        @url = "https://email.#{region}.amazonaws.com/v2/email/outbound-emails"
+        @http = http
+        @name = "ses"
+      end
+
+      def call(alert)
+        email = Email.compose(alert, from: @from, to: @to, subject_prefix: @subject_prefix, link: @link)
+        payload = {
+          "FromEmailAddress" => email.from,
+          "Destination" => { "ToAddresses" => email.to },
+          "Content" => {
+            "Simple" => {
+              "Subject" => { "Data" => email.subject, "Charset" => "UTF-8" },
+              "Body" => {
+                "Text" => { "Data" => email.text, "Charset" => "UTF-8" },
+                "Html" => { "Data" => email.html, "Charset" => "UTF-8" },
+              },
+            },
+          },
+        }
+        payload["ConfigurationSetName"] = @configuration_set_name if Provider.present?(@configuration_set_name)
+        payload["EmailTags"] = [{ "Name" => "source", "Value" => "cronwatch" }]
+        body = JS.json(payload)
+        headers = SigV4.sign(
+          method: "POST", url: @url, headers: { "content-type" => "application/json" }, body: body, region: @region,
+          service: "ses", now: @now.call, access_key_id: @access_key_id, secret_access_key: @secret_access_key,
+          session_token: @session_token
+        )
+        Provider.post(@http, "SES", @url, headers, body, [@secret_access_key, @session_token])
+        nil
+      end
+    end
+  end
+end

data/lib/cronwatch/alerts/sigv4.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "openssl"
+require "uri"
+
+module Cronwatch
+  module Alerts
+    # AWS Signature Version 4 with OpenSSL, for the SES channel, as the SDK's
+    # alerts/sigv4.ts. Spec:
+    # https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html
+    # Checked against the AWS SigV4 test suite (test/provider_channels_test.rb).
+    module SigV4
+      SPACES = Regexp.new("[#{JS::WHITESPACE}]+")
+
+      module_function
+
+      # Returns the headers to send: the given ones (lowercased) plus
+      # x-amz-date, the session token when there is one, and authorization.
+      # Host is signed but not returned; Net::HTTP sets it.
+      def sign(method:, url:, headers:, body:, region:, service:, now:, access_key_id:, secret_access_key:, session_token: nil)
+        uri = URI(url)
+        amz_date = JS.iso(now).delete("-:").sub(/\.\d{3}/, "")
+        day = amz_date[0, 8]
+        out = {}
+        headers.each { |name, value| out[name.to_s.downcase] = value.to_s }
+        out["x-amz-date"] = amz_date
+        out["x-amz-security-token"] = session_token if Provider.present?(session_token)
+
+        signed = out.merge("host" => host(uri))
+        names = signed.keys.sort
+        canonical_headers = names.map { |n| "#{n}:#{JS.trim(signed[n]).gsub(SPACES, " ")}\n" }.join
+        signed_headers = names.join(";")
+        canonical_request = [
+          method.to_s.upcase,
+          canonical_uri(uri.path),
+          canonical_query(uri.query),
+          canonical_headers,
+          signed_headers,
+          Provider.sha256_hex(body),
+        ].join("\n")
+        scope = "#{day}/#{region}/#{service}/aws4_request"
+        string_to_sign = ["AWS4-HMAC-SHA256", amz_date, scope, Provider.sha256_hex(canonical_request)].join("\n")
+
+        key = hmac("AWS4#{secret_access_key}", day)
+        key = hmac(key, region)
+        key = hmac(key, service)
+        key = hmac(key, "aws4_request")
+        signature = OpenSSL::HMAC.hexdigest("SHA256", key, string_to_sign)
+
+        out["authorization"] = "AWS4-HMAC-SHA256 Credential=#{access_key_id}/#{scope}, SignedHeaders=#{signed_headers}, Signature=#{signature}"
+        out
+      end
+
+      # URL#host: the host, and the port when it is not the scheme's default.
+      def host(uri)
+        uri.port && uri.port != uri.default_port ? "#{uri.host}:#{uri.port}" : uri.host
+      end
+
+      def hmac(key, data)
+        OpenSSL::HMAC.digest("SHA256", key, data)
+      end
+
+      # RFC 3986 encoding of every byte but the unreserved characters.
+      def uri_encode(text)
+        text.to_s.b.gsub(/[^A-Za-z0-9\-_.~]/n) { |c| format("%%%02X", c.ord) }.force_encoding(Encoding::UTF_8)
+      end
+
+      def canonical_uri(path)
+        return "/" if path.nil? || path.empty?
+
+        # The path is already encoded once; every AWS service but S3 expects each segment encoded again.
+        path.split("/", -1).map { |segment| uri_encode(segment) }.join("/")
+      end
+
+      # The query as URLSearchParams reads it, each name and value encoded, sorted.
+      def canonical_query(query)
+        return "" if query.nil? || query.empty?
+
+        pairs = URI.decode_www_form(query).map { |name, value| [uri_encode(name), uri_encode(value)] }
+        pairs.sort.map { |name, value| "#{name}=#{value}" }.join("&")
+      end
+    end
+  end
+end

data/lib/cronwatch/alerts/twilio.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Cronwatch
+  module Alerts
+    # Texts alerts through Twilio
+    # (https://www.twilio.com/docs/messaging/api/message-resource), form
+    # encoded with basic auth, one message per number, to every number at once.
+    #
+    #   Cronwatch::Alerts::Twilio.new(account_sid: ENV.fetch("TWILIO_ACCOUNT_SID"),
+    #                                 auth_token: ENV.fetch("TWILIO_AUTH_TOKEN"),
+    #                                 from: "+15005550006", to: ["+15551110000"])
+    #
+    # Sign with auth_token, or with api_key_sid and api_key_secret (each
+    # trimmed of the spaces and newlines a paste leaves). Send from a number,
+    # or through messaging_service_sid. Recoveries are not texted unless
+    # `recovered: true`: a text is for what needs a person. A message fits in
+    # `segments` SMS segments (default 3, 1 to 10).
+    #
+    # The alert counts as delivered when any number took it; each number that
+    # refused it is reported through the client's channel context (on_error).
+    # It fails only when every number did.
+    class Twilio
+      # The GSM 03.38 alphabet: a message in it takes 153 characters a segment
+      # (when split), anything else is UCS-2 at 67. The extension table costs two.
+      GSM = Set.new("@£$¥èéùìòÇ\nØø\rÅåΔ_ΦΓΛΩΠΨΣΘΞÆæßÉ !\"\#¤%&'()*+,-./0123456789:;<=>?¡ABCDEFGHIJKLMNOPQRSTUVWXYZÄÖÑܧ¿" \
+                    "abcdefghijklmnopqrstuvwxyzäöñüà".chars).freeze
+      GSM_EXTENDED = Set.new("^{}\\[~]|€\f".chars).freeze
+      # The most segments a message may use, which keeps it inside Twilio's 1600 character Body limit.
+      MAX_SEGMENTS = 10
+      # Twilio refuses a Body longer than this.
+      MAX_BODY = 1600
+
+      attr_reader :name
+
+      def initialize(account_sid:, to:, auth_token: nil, api_key_sid: nil, api_key_secret: nil, from: nil,
+                     messaging_service_sid: nil, recovered: false, segments: 3, link: nil, http: HTTP.default)
+        # A pasted credential often carries a stray space or newline, which the Authorization header would refuse or send.
+        account_sid = Provider.require_credential(account_sid, "Cronwatch::Alerts::Twilio needs an account_sid")
+        api_key_sid = Provider.trimmed(api_key_sid)
+        user = api_key_sid.empty? ? account_sid : api_key_sid
+        password = Provider.trimmed(api_key_sid.empty? ? auth_token : api_key_secret)
+        if password.empty?
+          raise ArgumentError, "Cronwatch::Alerts::Twilio needs an auth_token, or an api_key_sid and api_key_secret"
+        end
+        unless Provider.present?(from) || Provider.present?(messaging_service_sid)
+          raise ArgumentError, "Cronwatch::Alerts::Twilio needs a from number or a messaging_service_sid"
+        end
+
+        @to = (to.is_a?(Array) ? to : [to]).select { |n| n.is_a?(String) && JS.trim(n) != "" }.map { |n| JS.trim(n) }
+        raise ArgumentError, "Cronwatch::Alerts::Twilio needs at least one to number" if @to.empty?
+
+        @url = "https://api.twilio.com/2010-04-01/Accounts/#{Provider.encode_uri_component(account_sid)}/Messages.json"
+        @password = password
+        @authorization = Provider.basic_auth(user, @password)
+        @from = from
+        @messaging_service_sid = messaging_service_sid
+        @recovered = recovered
+        @segments = Twilio.segment_budget(segments)
+        @link = link
+        @http = http
+        # How long to wait for the numbers' requests, each already bounded by its own HTTP deadline.
+        @deadline_s = HTTP::TIMEOUT + 1
+        @name = "twilio"
+      end
+
+      # `context` is the client's channel context: each number that refused
+      # the alert, when another took it, is reported through its on_error.
+      def call(alert, context = nil)
+        return if alert.type == :recovered && !@recovered
+
+        body = Twilio.sms_body(alert, Provider.link_for(@link, alert), @segments)
+        errors = send_all(body)
+        failed = @to.each_index.filter_map { |i| [@to[i], errors[i]] if errors[i] }
+        return if failed.empty?
+
+        if failed.length == @to.length
+          message = failed[0][1].message
+          raise(@to.length > 1 ? "#{message} (#{failed.length} of #{@to.length} numbers failed)" : message)
+        end
+
+        # Delivered to someone: counted as sent, so a retry never texts the numbers that took it again.
+        took = @to.length - failed.length
+        failed.each do |number, error|
+          report = RuntimeError.new("#{error.message} (to #{Twilio.mask_number(number)}; #{took} of #{@to.length} numbers took the alert)")
+          if context.respond_to?(:on_error)
+            context.on_error(report)
+          else
+            warn "[cronwatch] alert channel twilio: #{report.message}"
+          end
+        end
+        nil
+      end
+
+      # A number with all but its last four digits hidden, for an error message.
+      def self.mask_number(number)
+        length = JS.length16(number)
+        length <= 4 ? number : "#{"*" * [length - 4, 8].min}#{JS.tail16(number, 4)}"
+      end
+
+      # A segment count clamped to 1 to MAX_SEGMENTS; 3 for anything not a number.
+      def self.segment_budget(segments)
+        n = segments.is_a?(Numeric) && segments.real? && JS.finite?(segments) ? segments.floor.to_i : 3
+        [MAX_SEGMENTS, [1, n].max].min
+      end
+
+      # The segments `text` takes. A character is never split across two: an
+      # extension character (two septets) or a surrogate pair (two UCS-2
+      # units) that would straddle a boundary starts the next segment, as
+      # phones pack them.
+      def self.sms_segments(text)
+        units = []
+        gsm = true
+        text.each_char do |ch|
+          if GSM.include?(ch) then units << 1
+          elsif GSM_EXTENDED.include?(ch) then units << 2
+          else
+            gsm = false
+            break
+          end
+        end
+        single, per, sizes = gsm ? [160, 153, units] : [70, 67, text.each_char.map { |ch| ch.ord > 0xFFFF ? 2 : 1 }]
+        return 1 if sizes.sum <= single
+
+        count = 1
+        used = 0
+        sizes.each do |u|
+          if used + u > per
+            count += 1
+            used = 0
+          end
+          used += u
+        end
+        count
+      end
+
+      # Whether `text` fits in `segments` SMS segments and Twilio's Body limit.
+      def self.fits?(text, segments)
+        JS.length16(text) <= MAX_BODY && sms_segments(text) <= segments
+      end
+
+      # The title, then as many lines of the message (and the triage) as fit,
+      # then the link. The link is kept whole; the text before it is cut to
+      # make room. `segments` is clamped to 1 to 10.
+      def self.sms_body(alert, link, segments = 3)
+        budget = segment_budget(segments)
+        tail = Provider.present?(link) ? "\n#{link}" : ""
+        lines = [alert.title, *alert.message.split("\n", -1).reject { |l| JS.trim(l) == "" }]
+        lines << "Triage: #{alert.triage}" if Provider.present?(alert.triage)
+        text = +""
+        lines.each do |line|
+          following = text.empty? ? line : "#{text}\n#{line}"
+          if fits?(following + tail, budget)
+            text = following
+            next
+          end
+          # Part of this line, cut on a code point and marked.
+          chars = line.chars
+          lo = 0
+          hi = chars.length
+          while lo < hi
+            mid = (lo + hi + 1) / 2
+            candidate = (text.empty? ? "" : "#{text}\n") + chars[0, mid].join + "..."
+            if fits?(candidate + tail, budget)
+              lo = mid
+            else
+              hi = mid - 1
+            end
+          end
+          text = (text.empty? ? "" : "#{text}\n") + chars[0, lo].join + "..." if lo.positive?
+          break
+        end
+        # Only a link too long for any budget gets here too long; Twilio would refuse it whole.
+        Provider.cut(text + tail, MAX_BODY)
+      end
+
+      private
+
+      # Posts to every number at once, each in its own thread. Returns the
+      # error for each number, by index, or nil where it took the alert.
+      def send_all(body)
+        errors = Array.new(@to.length)
+        threads = @to.each_with_index.map do |number, i|
+          Thread.new do
+            Thread.current.report_on_exception = false
+            pairs = [["To", number]]
+            pairs << (Provider.present?(@messaging_service_sid) ? ["MessagingServiceSid", @messaging_service_sid] : ["From", @from])
+            pairs << ["Body", body]
+            headers = { "content-type" => "application/x-www-form-urlencoded", "authorization" => @authorization }
+            Provider.post(@http, "Twilio", @url, headers, Provider.form(pairs), [@password])
+          rescue StandardError => e
+            errors[i] = e
+          end
+        end
+        deadline = HTTP.monotonic + @deadline_s
+        threads.each_with_index do |thread, i|
+          next if thread.join([deadline - HTTP.monotonic, 0].max)
+
+          errors[i] = HTTP::TimeoutError.new
+        end
+        errors
+      end
+    end
+  end
+end

data/lib/cronwatch/alerts/webhook.rb

@@ -7,7 +7,10 @@ module Cronwatch
     # POSTs the alert as JSON to any URL. The body is the alert's JSON:
     # { type, run, details, job, definition, title, message, at, triage }.
     # With a secret, each request carries `X-CronWatch-Signature: sha256=<hex>`,
-    # the HMAC-SHA256 of the raw body, so the receiver can verify it.
+    # the HMAC-SHA256 of the raw body, so the receiver can verify it. Header
+    # values are trimmed. A redirect is an error, not followed (the headers
+    # and the signature would go with it): point the url at where the
+    # receiver really is.
     class Webhook
       attr_reader :name
 
@@ -23,7 +26,9 @@ module Cronwatch
 
       def call(alert)
         body = JS.json(alert.to_h)
-        headers = { "content-type" => "application/json", "user-agent" => "cronwatch" }.merge(@headers.transform_keys(&:to_s))
+        headers = { "content-type" => "application/json", "user-agent" => "cronwatch" }
+        # A pasted Authorization value often carries a stray space or newline, which a header would refuse.
+        @headers.each { |name, value| headers[name.to_s] = value.is_a?(String) ? JS.trim(value) : value }
         if @secret && !@secret.empty?
           headers["x-cronwatch-signature"] = "sha256=#{OpenSSL::HMAC.hexdigest("SHA256", @secret, body)}"
         end