bouncy 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +24 -0
- data/README.md +185 -93
- data/guides/README.md +20 -0
- data/guides/admin.md +2 -2
- data/guides/amazon-ses.md +70 -9
- data/guides/api.md +129 -0
- data/guides/bounce-handling.md +38 -1
- data/guides/compatibility.md +16 -2
- data/guides/configuration.md +83 -0
- data/guides/delivery.md +46 -8
- data/guides/migrating.md +3 -3
- data/guides/operations.md +104 -0
- data/guides/privacy.md +1 -1
- data/guides/troubleshooting.md +1 -1
- data/lib/bouncy/address_lock.rb +321 -0
- data/lib/bouncy/body_limit.rb +55 -0
- data/lib/bouncy/configuration.rb +38 -2
- data/lib/bouncy/event.rb +8 -3
- data/lib/bouncy/ingestor.rb +5 -3
- data/lib/bouncy/interceptor.rb +85 -16
- data/lib/bouncy/notification.rb +70 -0
- data/lib/bouncy/providers/ses.rb +57 -2
- data/lib/bouncy/providers/ses_parser.rb +20 -0
- data/lib/bouncy/providers/sns_verifier.rb +74 -6
- data/lib/bouncy/prune_job.rb +3 -0
- data/lib/bouncy/scope_lock.rb +3 -2
- data/lib/bouncy/store.rb +4 -0
- data/lib/bouncy/sync_job.rb +4 -0
- data/lib/bouncy/version.rb +1 -1
- data/lib/bouncy/webhook.rb +60 -9
- data/lib/bouncy.rb +34 -2
- data/lib/generators/bouncy/templates/initializer.rb.tt +14 -0
- metadata +13 -3
data/lib/bouncy/interceptor.rb
CHANGED
|
@@ -1,30 +1,83 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Bouncy
|
|
4
|
+
# Checks the recipients of every mail where Mail hands it to the transport.
|
|
5
|
+
#
|
|
6
|
+
# Mail calls delivering_email before each delivery. It stamps the configuration-set header and
|
|
7
|
+
# wraps the message's delivery method in a Gate. Mail's deliver and deliver! both end in that
|
|
8
|
+
# delivery method, after every interceptor and every Action Mailer callback, so the Gate
|
|
9
|
+
# decides on the recipients the transport would really get. In :drop mode it removes blocked
|
|
10
|
+
# recipients, and when none is left it does not call the transport: deliver_now!,
|
|
11
|
+
# deliver_later! and their plain forms all send nothing.
|
|
4
12
|
class Interceptor
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
return Bouncy.unconfigured!("interception") unless Bouncy.configured?
|
|
13
|
+
CONFIGURATION_SET_HEADER = "X-SES-CONFIGURATION-SET"
|
|
14
|
+
RECIPIENT_FIELDS = %i[to cc bcc].freeze
|
|
8
15
|
|
|
9
|
-
|
|
16
|
+
# The delivery method of a message that Bouncy checks. Mail builds it through
|
|
17
|
+
# Message#delivery_method(Gate, transport), its public way to set a delivery method.
|
|
18
|
+
class Gate
|
|
19
|
+
attr_reader :transport
|
|
20
|
+
|
|
21
|
+
def initialize(transport)
|
|
22
|
+
@transport = transport
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def settings = transport.settings
|
|
26
|
+
|
|
27
|
+
# Bypass, scope and mode are read again at each delivery: a message delivered once keeps
|
|
28
|
+
# its gate, and can be delivered again under Bouncy.unblocked or another mode.
|
|
29
|
+
def deliver!(message)
|
|
30
|
+
return transport.deliver!(message) unless Interceptor.active?
|
|
31
|
+
return if Interceptor.new.call(message) == :undeliverable
|
|
32
|
+
|
|
33
|
+
transport.deliver!(message)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def respond_to_missing?(name, include_private = false) = transport.respond_to?(name, include_private) || super
|
|
37
|
+
def method_missing(name, ...) = transport.respond_to?(name) ? transport.public_send(name, ...) : super
|
|
10
38
|
end
|
|
11
39
|
|
|
40
|
+
class << self
|
|
41
|
+
def delivering_email(message)
|
|
42
|
+
stamp_configuration_set(message)
|
|
43
|
+
return if bypassed?
|
|
44
|
+
return Bouncy.unconfigured!("interception") unless Bouncy.configured?
|
|
45
|
+
|
|
46
|
+
transport = message.delivery_method
|
|
47
|
+
message.delivery_method(Gate, transport) unless transport.is_a?(Gate)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def active? = !bypassed? && Bouncy.configured?
|
|
51
|
+
|
|
52
|
+
private
|
|
53
|
+
|
|
54
|
+
def bypassed? = Bouncy.configuration.interception == :off || ActiveSupport::IsolatedExecutionState[:bouncy_unblocked]
|
|
55
|
+
|
|
56
|
+
# SES reads the set from this header and removes it before delivery. A mail that names its
|
|
57
|
+
# own set keeps it. The header does not depend on the interception mode: turning
|
|
58
|
+
# interception off must not stop the events that SES publishes for the set.
|
|
59
|
+
def stamp_configuration_set(message)
|
|
60
|
+
name = Bouncy.configuration.ses.configuration_set.presence
|
|
61
|
+
message[CONFIGURATION_SET_HEADER] = name if name && message.header[CONFIGURATION_SET_HEADER].nil?
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Returns :undeliverable when :drop removed every recipient, else nil.
|
|
12
66
|
def call(message)
|
|
13
67
|
envelope = Array(message.smtp_envelope_to).dup
|
|
14
|
-
|
|
15
|
-
keys = (envelope +
|
|
16
|
-
rows = Bouncy.blocked.where(email: keys).to_a
|
|
68
|
+
fields = RECIPIENT_FIELDS.to_h { |name| [name, addresses(message[name])] }
|
|
69
|
+
keys = (envelope + fields.values.flatten.map(&:address)).filter_map { |entry| normalize(entry) }.uniq
|
|
70
|
+
rows = Bouncy.current_state(Bouncy.blocked).where(email: keys).to_a
|
|
17
71
|
return if rows.empty?
|
|
18
72
|
|
|
19
73
|
mode = Bouncy.configuration.interception
|
|
20
|
-
# Provider-derived evidence is only enforced while a complete sync is fresh
|
|
21
|
-
# and legacy soft holds are local policy and
|
|
22
|
-
# rule applies in :log mode so that its
|
|
23
|
-
|
|
74
|
+
# Provider-derived evidence is only enforced while a complete sync is fresh, unless the host
|
|
75
|
+
# set require_fresh_sync to false. Manual holds and legacy soft holds are local policy and
|
|
76
|
+
# do not depend on provider freshness. The same rule applies in :log mode so that its
|
|
77
|
+
# preview matches what :drop would do.
|
|
78
|
+
fresh = !Bouncy.configuration.require_fresh_sync || Bouncy.sync_fresh?
|
|
24
79
|
enforceable = rows.select { |row| fresh || row.manual_blocked_at || (row.soft_blocked_until && row.soft_blocked_until > Time.current) }
|
|
25
80
|
blocked = enforceable.map(&:email)
|
|
26
|
-
remaining = envelope.reject { |email| blocked.include?(normalize(email)) }
|
|
27
|
-
changed_headers = headers.transform_values { |values| values.reject { |email| blocked.include?(normalize(email)) } }
|
|
28
81
|
# Persist all skip evidence before changing the message. A DB outage must leave it intact.
|
|
29
82
|
Event.transaction do
|
|
30
83
|
rows.each do |row|
|
|
@@ -37,19 +90,35 @@ module Bouncy
|
|
|
37
90
|
end
|
|
38
91
|
return unless mode == :drop && blocked.any?
|
|
39
92
|
|
|
40
|
-
|
|
93
|
+
# Each kept entry keeps its own text, display name included; only the lookup uses the
|
|
94
|
+
# bare address.
|
|
95
|
+
fields.each do |name, field_addresses|
|
|
96
|
+
kept = field_addresses.reject { |address| blocked.include?(normalize(address.address)) }
|
|
97
|
+
message[name] = kept.empty? ? nil : kept.map(&:format) if kept.size < field_addresses.size
|
|
98
|
+
end
|
|
99
|
+
remaining = envelope.reject { |entry| blocked.include?(normalize(entry)) }
|
|
41
100
|
message.smtp_envelope_to = remaining
|
|
42
|
-
|
|
101
|
+
return if remaining.any?
|
|
102
|
+
|
|
103
|
+
message.perform_deliveries = false
|
|
104
|
+
:undeliverable
|
|
43
105
|
rescue ActiveRecord::ActiveRecordError => e
|
|
44
106
|
raise unless Bouncy.database_unavailable?(e)
|
|
45
107
|
|
|
46
108
|
ActiveSupport::Notifications.instrument("unavailable.bouncy", operation: "interception")
|
|
109
|
+
nil
|
|
47
110
|
end
|
|
48
111
|
|
|
49
112
|
private
|
|
50
113
|
|
|
114
|
+
# The parsed addresses of a To, Cc or Bcc field, or none for a field Mail cannot parse.
|
|
115
|
+
def addresses(field)
|
|
116
|
+
field.respond_to?(:addrs) ? field.addrs : []
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# "Ada <ADA@example.com>" and "ada@example.com" are the same lookup key.
|
|
51
120
|
def normalize(value)
|
|
52
|
-
Identity.normalize(Mail::Address.new(value).address)
|
|
121
|
+
Identity.normalize(Mail::Address.new(value.to_s).address)
|
|
53
122
|
rescue InvalidAddress, Mail::Field::ParseError
|
|
54
123
|
nil
|
|
55
124
|
end
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bouncy
|
|
4
|
+
# One SES notification, as config.on_notification receives it: every event type SES
|
|
5
|
+
# publishes to the topic, Send and Delivery included, after the receiver committed what it
|
|
6
|
+
# records from it.
|
|
7
|
+
#
|
|
8
|
+
# config.on_notification = lambda do |notification|
|
|
9
|
+
# notification.type # => "Delivery"
|
|
10
|
+
# notification.message_id # => the SES message id of the mail
|
|
11
|
+
# notification.recipients # => ["ada@example.com"]
|
|
12
|
+
# notification.occurred_at # => the time SES gives for this event
|
|
13
|
+
# notification.tags # => { "order" => ["42"] }
|
|
14
|
+
# notification.raw # => the SES JSON object, as a Hash
|
|
15
|
+
# end
|
|
16
|
+
#
|
|
17
|
+
# SNS repeats and reorders notifications, so the hook must be safe to run again and in any
|
|
18
|
+
# order.
|
|
19
|
+
class Notification
|
|
20
|
+
# The SES object that holds an event's own details, where it is not the type in camelCase.
|
|
21
|
+
SECTIONS = { "Rendering Failure" => "failure", "RenderingFailure" => "failure" }.freeze
|
|
22
|
+
|
|
23
|
+
attr_reader :raw
|
|
24
|
+
|
|
25
|
+
def initialize(raw, sns_timestamp: nil)
|
|
26
|
+
@raw = raw
|
|
27
|
+
@sns_timestamp = sns_timestamp
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# The SES event type as SES names it: "Send", "Delivery", "Bounce", "Complaint", ...
|
|
31
|
+
def type = raw["eventType"] || raw["notificationType"]
|
|
32
|
+
|
|
33
|
+
def message_id = mail["messageId"]
|
|
34
|
+
|
|
35
|
+
# The SES message tags of the mail: each name maps to an array of values.
|
|
36
|
+
def tags = mail["tags"].is_a?(Hash) ? mail["tags"] : {}
|
|
37
|
+
|
|
38
|
+
# The addresses this event is about, exactly as SES spells them: the bounced, complained,
|
|
39
|
+
# delivered or delayed recipients, or the mail's destination for other events.
|
|
40
|
+
def recipients
|
|
41
|
+
values = case type
|
|
42
|
+
when "Bounce" then section["bouncedRecipients"]
|
|
43
|
+
when "Complaint" then section["complainedRecipients"]
|
|
44
|
+
when "Delivery" then section["recipients"]
|
|
45
|
+
when "DeliveryDelay" then section["delayedRecipients"]
|
|
46
|
+
else mail["destination"]
|
|
47
|
+
end
|
|
48
|
+
Array(values).filter_map do |recipient|
|
|
49
|
+
address = recipient.is_a?(Hash) ? recipient["emailAddress"] : recipient
|
|
50
|
+
address if address.is_a?(String)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# The time of this event: its own timestamp, else the mail's, else the time SNS published it.
|
|
55
|
+
def occurred_at
|
|
56
|
+
[section["timestamp"], mail["timestamp"], @sns_timestamp].each do |value|
|
|
57
|
+
return Time.iso8601(value) if value.is_a?(String)
|
|
58
|
+
rescue ArgumentError
|
|
59
|
+
next
|
|
60
|
+
end
|
|
61
|
+
nil
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
private
|
|
65
|
+
|
|
66
|
+
def mail = object(raw["mail"])
|
|
67
|
+
def section = object(raw[SECTIONS.fetch(type.to_s) { type.to_s.sub(/\A./, &:downcase) }])
|
|
68
|
+
def object(value) = value.is_a?(Hash) ? value : {}
|
|
69
|
+
end
|
|
70
|
+
end
|
data/lib/bouncy/providers/ses.rb
CHANGED
|
@@ -3,6 +3,24 @@
|
|
|
3
3
|
module Bouncy
|
|
4
4
|
module Providers
|
|
5
5
|
class Ses < Base
|
|
6
|
+
# The SNS host domain of each AWS partition, for the certificate and SubscribeURL checks.
|
|
7
|
+
SNS_DOMAINS = { "aws" => "amazonaws.com", "aws-us-gov" => "amazonaws.com", "aws-cn" => "amazonaws.com.cn" }.freeze
|
|
8
|
+
# SNS answers a SubscribeURL visit with a short XML document.
|
|
9
|
+
CONFIRMATION_LIMIT = 64 * 1024
|
|
10
|
+
|
|
11
|
+
# The SubscribeURL as a URI when it is HTTPS on the topic's own regional SNS host, port 443,
|
|
12
|
+
# with no userinfo and no fragment; nil for any other value.
|
|
13
|
+
def self.subscribe_uri(envelope, region)
|
|
14
|
+
uri = URI.parse(envelope["SubscribeURL"].to_s)
|
|
15
|
+
domain = SNS_DOMAINS[envelope["TopicArn"].to_s.split(":")[1]]
|
|
16
|
+
return unless domain && uri.is_a?(URI::HTTPS) && uri.host == "sns.#{region}.#{domain}" && uri.port == 443
|
|
17
|
+
return if uri.userinfo || uri.fragment
|
|
18
|
+
|
|
19
|
+
uri
|
|
20
|
+
rescue URI::InvalidURIError
|
|
21
|
+
nil
|
|
22
|
+
end
|
|
23
|
+
|
|
6
24
|
def initialize(configuration = Bouncy.configuration)
|
|
7
25
|
super()
|
|
8
26
|
@configuration = configuration
|
|
@@ -70,8 +88,12 @@ module Bouncy
|
|
|
70
88
|
def control(envelope) # rubocop:disable Naming/PredicateMethod -- Adapter operation with side effects.
|
|
71
89
|
case envelope.fetch("Type")
|
|
72
90
|
when "SubscriptionConfirmation"
|
|
73
|
-
|
|
74
|
-
|
|
91
|
+
if @settings.subscription_confirmation == :subscribe_url
|
|
92
|
+
visit_subscribe_url(envelope)
|
|
93
|
+
else
|
|
94
|
+
request { sns_client.confirm_subscription(topic_arn: envelope.fetch("TopicArn"), token: envelope.fetch("Token")) }
|
|
95
|
+
end
|
|
96
|
+
Store.event!("subscription_confirmed", source: "webhook", details: { "via" => @settings.subscription_confirmation.to_s })
|
|
75
97
|
true
|
|
76
98
|
when "UnsubscribeConfirmation"
|
|
77
99
|
Store.event!("subscription_unsubscribed", source: "webhook")
|
|
@@ -85,6 +107,11 @@ module Bouncy
|
|
|
85
107
|
SesParser.new(@configuration).call(envelope)
|
|
86
108
|
end
|
|
87
109
|
|
|
110
|
+
# The notification for config.on_notification. parse has already checked the message.
|
|
111
|
+
def notification(envelope)
|
|
112
|
+
Notification.new(JSON.parse(envelope.fetch("Message"), max_nesting: 20), sns_timestamp: envelope["Timestamp"])
|
|
113
|
+
end
|
|
114
|
+
|
|
88
115
|
def doctor
|
|
89
116
|
policy = policy_check
|
|
90
117
|
{ "scope" => @configuration.scope, "policy_verified" => policy.verified, "policy_reason" => policy.reason,
|
|
@@ -94,6 +121,33 @@ module Bouncy
|
|
|
94
121
|
|
|
95
122
|
private
|
|
96
123
|
|
|
124
|
+
# Confirms the subscription by visiting the SubscribeURL that SNS signed, so the credentials
|
|
125
|
+
# need no sns:ConfirmSubscription grant. Only an HTTPS URL on the topic's own regional SNS
|
|
126
|
+
# host is visited; any other URL is malformed for good. A transport failure or an answer
|
|
127
|
+
# other than 2xx is a ProviderError: the receiver answers 503, and SNS sends the
|
|
128
|
+
# confirmation again.
|
|
129
|
+
def visit_subscribe_url(envelope)
|
|
130
|
+
require "net/http"
|
|
131
|
+
uri = self.class.subscribe_uri(envelope, @settings.region)
|
|
132
|
+
raise MalformedMessage, "SubscribeURL is not on the topic's SNS host" unless uri
|
|
133
|
+
|
|
134
|
+
code = nil
|
|
135
|
+
Net::HTTP.start(uri.host, uri.port, use_ssl: true, open_timeout: 3, read_timeout: 5,
|
|
136
|
+
verify_mode: OpenSSL::SSL::VERIFY_PEER, max_retries: 0) do |http|
|
|
137
|
+
http.request(Net::HTTP::Get.new(uri.request_uri)) do |response|
|
|
138
|
+
code = response.code
|
|
139
|
+
size = 0
|
|
140
|
+
response.read_body do |chunk|
|
|
141
|
+
size += chunk.bytesize
|
|
142
|
+
raise ProviderError, "SNS confirmation response exceeds limit" if size > CONFIRMATION_LIMIT
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
raise ProviderError, "SNS did not confirm the subscription (HTTP #{code})" unless code.to_s.match?(/\A2\d\d\z/)
|
|
147
|
+
rescue Timeout::Error, SocketError, IOError, SystemCallError, OpenSSL::SSL::SSLError
|
|
148
|
+
raise ProviderError, "SNS confirmation transport unavailable"
|
|
149
|
+
end
|
|
150
|
+
|
|
97
151
|
def require_sdk(service)
|
|
98
152
|
require "aws-sdk-#{service}"
|
|
99
153
|
rescue LoadError
|
|
@@ -160,6 +214,7 @@ module Bouncy
|
|
|
160
214
|
end
|
|
161
215
|
|
|
162
216
|
sets = @settings.configuration_sets.dup
|
|
217
|
+
sets << @settings.configuration_set if @settings.configuration_set.present?
|
|
163
218
|
@settings.identities.each do |identity|
|
|
164
219
|
response = request { ses.get_email_identity(email_identity: identity) }
|
|
165
220
|
sets << response.configuration_set_name if response.configuration_set_name.present?
|
|
@@ -79,6 +79,8 @@ module Bouncy
|
|
|
79
79
|
end
|
|
80
80
|
|
|
81
81
|
def classify(type, data)
|
|
82
|
+
return strict_classification(type, data) if @configuration.webhook_blocking == :strict && %w[Bounce Complaint].include?(type)
|
|
83
|
+
|
|
82
84
|
case type
|
|
83
85
|
when "Bounce"
|
|
84
86
|
case data["bounceSubType"]
|
|
@@ -107,6 +109,24 @@ module Bouncy
|
|
|
107
109
|
else %w[ignored unsupported_type]
|
|
108
110
|
end
|
|
109
111
|
end
|
|
112
|
+
|
|
113
|
+
# config.webhook_blocking = :strict: nothing reads the account list, so every Permanent
|
|
114
|
+
# bounce blocks, whatever its subtype, and so does every recipient a complaint names.
|
|
115
|
+
# SES cannot say which one complained: most ISPs remove that address, and SES lists the
|
|
116
|
+
# recipients who might have sent the complaint. Undetermined and Transient bounces stay
|
|
117
|
+
# record-only, and a not-spam report says the opposite of a complaint.
|
|
118
|
+
def strict_classification(type, data)
|
|
119
|
+
if type == "Bounce"
|
|
120
|
+
return ["hard_bounce", data["bounceSubType"].presence || "unspecified"] if data["bounceType"] == "Permanent"
|
|
121
|
+
|
|
122
|
+
return ["soft_bounce", data["bounceSubType"]] if %w[Transient Undetermined].include?(data["bounceType"])
|
|
123
|
+
|
|
124
|
+
return %w[unknown unrecognized_bounce]
|
|
125
|
+
end
|
|
126
|
+
return %w[ignored not_spam] if data["complaintFeedbackType"] == "not-spam"
|
|
127
|
+
|
|
128
|
+
["complaint", data["complaintSubType"].presence || "named"]
|
|
129
|
+
end
|
|
110
130
|
end
|
|
111
131
|
end
|
|
112
132
|
end
|
|
@@ -7,13 +7,71 @@ module Bouncy
|
|
|
7
7
|
class SnsVerifier
|
|
8
8
|
BODY_LIMIT = 2 * 1024 * 1024
|
|
9
9
|
|
|
10
|
+
# Signing certificates, shared by every receiver in the process: each request builds a new
|
|
11
|
+
# adapter, and an app that subscribes to Send and Delivery events gets one SNS message per
|
|
12
|
+
# mail. At most 16 certificate URLs, each kept for an hour. Only a certificate that the
|
|
13
|
+
# allowlisted, bounded download returned is kept; a failed download keeps nothing.
|
|
14
|
+
class CertificateCache
|
|
15
|
+
Entry = Data.define(:pem, :fetched_at)
|
|
16
|
+
|
|
17
|
+
def initialize(size:, ttl:)
|
|
18
|
+
@size = size
|
|
19
|
+
@ttl = ttl
|
|
20
|
+
@entries = {}
|
|
21
|
+
@mutex = Mutex.new
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# The download runs outside the lock, so one slow fetch does not stop other requests.
|
|
25
|
+
def fetch(url)
|
|
26
|
+
entry = @mutex.synchronize { @entries[url] }
|
|
27
|
+
return entry.pem if entry && Time.now - entry.fetched_at < @ttl
|
|
28
|
+
|
|
29
|
+
pem = yield
|
|
30
|
+
@mutex.synchronize do
|
|
31
|
+
@entries.delete(url)
|
|
32
|
+
@entries[url] = Entry.new(pem: pem, fetched_at: Time.now)
|
|
33
|
+
@entries.shift while @entries.size > @size
|
|
34
|
+
end
|
|
35
|
+
pem
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Drops a certificate that failed a verification, so the next one fetches it again. A
|
|
39
|
+
# certificate fetched less than `after` seconds ago stays: a stream of forged messages
|
|
40
|
+
# must not turn into a stream of downloads.
|
|
41
|
+
def forget(url, after:)
|
|
42
|
+
@mutex.synchronize do
|
|
43
|
+
entry = @entries[url]
|
|
44
|
+
next false unless entry && Time.now - entry.fetched_at >= after
|
|
45
|
+
|
|
46
|
+
@entries.delete(url)
|
|
47
|
+
true
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def size = @mutex.synchronize { @entries.size }
|
|
52
|
+
def clear = @mutex.synchronize { @entries.clear }
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
CERTIFICATES = CertificateCache.new(size: 16, ttl: 3600)
|
|
56
|
+
# A cached certificate older than this is fetched again, once, when a signature fails.
|
|
57
|
+
REFETCH_AFTER = 60
|
|
58
|
+
|
|
10
59
|
class BoundedVerifier < Aws::SNS::MessageVerifier
|
|
11
60
|
private
|
|
12
61
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
62
|
+
# The SDK's download_pem checks the URL and the PEM framing, and calls the bounded
|
|
63
|
+
# https_get below. A certificate is cached only once it parses and carries the RSA key
|
|
64
|
+
# that SNS signs with; any other body is refused and never kept, so the next request
|
|
65
|
+
# fetches again at once.
|
|
66
|
+
def pem(uri) = CERTIFICATES.fetch(uri.to_s) { usable(download_pem(uri)) }
|
|
67
|
+
|
|
68
|
+
def usable(pem)
|
|
69
|
+
key = OpenSSL::X509::Certificate.new(pem).public_key
|
|
70
|
+
raise VerificationError, "The SNS signing certificate has no RSA key" unless key.is_a?(OpenSSL::PKey::RSA)
|
|
71
|
+
|
|
72
|
+
pem
|
|
73
|
+
rescue OpenSSL::X509::CertificateError, OpenSSL::PKey::PKeyError
|
|
74
|
+
raise VerificationError, "The SNS signing certificate does not parse"
|
|
17
75
|
end
|
|
18
76
|
|
|
19
77
|
def https_get(uri, _failed_attempts = 0)
|
|
@@ -65,7 +123,7 @@ module Bouncy
|
|
|
65
123
|
end
|
|
66
124
|
|
|
67
125
|
authorize!(envelope)
|
|
68
|
-
|
|
126
|
+
verify(raw_body, envelope.fetch("SigningCertURL"))
|
|
69
127
|
envelope
|
|
70
128
|
rescue JSON::ParserError, URI::InvalidURIError
|
|
71
129
|
raise MalformedMessage, "Malformed SNS envelope"
|
|
@@ -75,6 +133,16 @@ module Bouncy
|
|
|
75
133
|
|
|
76
134
|
private
|
|
77
135
|
|
|
136
|
+
# A certificate that SNS rotated at the same URL fails once from the cache; it is fetched
|
|
137
|
+
# again, once, and the message is checked with the new one.
|
|
138
|
+
def verify(raw_body, url)
|
|
139
|
+
@mutex.synchronize { @verifier.authenticate!(raw_body) }
|
|
140
|
+
rescue Aws::SNS::MessageVerifier::VerificationError, OpenSSL::OpenSSLError
|
|
141
|
+
raise unless @verifier.is_a?(BoundedVerifier) && CERTIFICATES.forget(url, after: REFETCH_AFTER)
|
|
142
|
+
|
|
143
|
+
@mutex.synchronize { @verifier.authenticate!(raw_body) }
|
|
144
|
+
end
|
|
145
|
+
|
|
78
146
|
def authorize!(envelope)
|
|
79
147
|
topic = envelope.fetch("TopicArn")
|
|
80
148
|
settings = @configuration.ses
|
|
@@ -84,7 +152,7 @@ module Bouncy
|
|
|
84
152
|
raise AuthenticationError, "SNS topic is not authorized"
|
|
85
153
|
end
|
|
86
154
|
|
|
87
|
-
suffix =
|
|
155
|
+
suffix = Ses::SNS_DOMAINS[parts[1]]
|
|
88
156
|
uri = URI.parse(envelope.fetch("SigningCertURL"))
|
|
89
157
|
unless suffix && uri.scheme == "https" && uri.host == "sns.#{settings.region}.#{suffix}" &&
|
|
90
158
|
uri.port == 443 && !uri.userinfo && !uri.query && !uri.fragment &&
|
data/lib/bouncy/prune_job.rb
CHANGED
|
@@ -4,7 +4,10 @@ module Bouncy
|
|
|
4
4
|
class PruneJob < ActiveJob::Base
|
|
5
5
|
queue_as :default
|
|
6
6
|
|
|
7
|
+
# Like SyncJob, a no-op until config.scope is set.
|
|
7
8
|
def perform
|
|
9
|
+
return Rails.logger&.info("[bouncy] Bouncy::PruneJob skipped: config.scope is not set") unless Bouncy.configured?
|
|
10
|
+
|
|
8
11
|
Bouncy.configuration.validate!
|
|
9
12
|
# Keep state rows, including inactive release fences, indefinitely.
|
|
10
13
|
Bouncy.events.where("created_at < ?", Bouncy.configuration.retention.ago).in_batches.delete_all
|
data/lib/bouncy/scope_lock.rb
CHANGED
|
@@ -13,7 +13,8 @@ module Bouncy
|
|
|
13
13
|
key = Digest::SHA256.hexdigest(Bouncy.scope)[0, 15].to_i(16)
|
|
14
14
|
case connection.adapter_name
|
|
15
15
|
when "PostgreSQL"
|
|
16
|
-
|
|
16
|
+
# Uncached: a cached answer would report the lock without taking it.
|
|
17
|
+
acquired = connection.uncached { connection.select_value("SELECT pg_try_advisory_lock(#{key})") }
|
|
17
18
|
raise ProviderError, "Another sync holds this scope lock" unless [true, "t"].include?(acquired)
|
|
18
19
|
|
|
19
20
|
begin
|
|
@@ -23,7 +24,7 @@ module Bouncy
|
|
|
23
24
|
end
|
|
24
25
|
when /Mysql|Trilogy/i
|
|
25
26
|
name = Digest::SHA256.hexdigest("bouncy:#{connection.pool.db_config.database}:#{Bouncy.scope}")
|
|
26
|
-
acquired = connection.select_value("SELECT GET_LOCK('#{name}', 0)")
|
|
27
|
+
acquired = connection.uncached { connection.select_value("SELECT GET_LOCK('#{name}', 0)") }
|
|
27
28
|
raise ProviderError, "Another sync holds this scope lock" unless acquired == 1
|
|
28
29
|
|
|
29
30
|
begin
|
data/lib/bouncy/store.rb
CHANGED
|
@@ -5,11 +5,15 @@ module Bouncy
|
|
|
5
5
|
module_function
|
|
6
6
|
|
|
7
7
|
# Retry outside the transaction: PostgreSQL aborts a transaction on a unique violation.
|
|
8
|
+
# The address lock comes first, before any row lock: a sender inside Bouncy.hold sees this
|
|
9
|
+
# change either committed or not started.
|
|
8
10
|
def change(email)
|
|
9
11
|
key = Identity.normalize(email)
|
|
12
|
+
Record.connection_pool.with_connection { |connection| AddressLock.settle(connection) }
|
|
10
13
|
attempts = 0
|
|
11
14
|
begin
|
|
12
15
|
Suppression.transaction(requires_new: true) do
|
|
16
|
+
AddressLock.claim!(key)
|
|
13
17
|
row = Suppression.find_or_create_by!(scope: Bouncy.scope, email: key) do |record|
|
|
14
18
|
record.provider = Bouncy.configuration.provider.to_s
|
|
15
19
|
end
|
data/lib/bouncy/sync_job.rb
CHANGED
|
@@ -6,7 +6,11 @@ module Bouncy
|
|
|
6
6
|
class SyncJob < ActiveJob::Base
|
|
7
7
|
queue_as :default
|
|
8
8
|
|
|
9
|
+
# A recurring job in an app that has not set config.scope does nothing, instead of failing
|
|
10
|
+
# every run. Bouncy.sync! itself still raises.
|
|
9
11
|
def perform
|
|
12
|
+
return Rails.logger&.info("[bouncy] Bouncy::SyncJob skipped: config.scope is not set") unless Bouncy.configured?
|
|
13
|
+
|
|
10
14
|
Bouncy.sync!
|
|
11
15
|
end
|
|
12
16
|
end
|
data/lib/bouncy/version.rb
CHANGED
data/lib/bouncy/webhook.rb
CHANGED
|
@@ -3,17 +3,24 @@
|
|
|
3
3
|
module Bouncy
|
|
4
4
|
class Webhook
|
|
5
5
|
BODY_LIMIT = 2 * 1024 * 1024
|
|
6
|
+
METHODS = %w[GET HEAD PUT PATCH DELETE OPTIONS TRACE CONNECT].freeze
|
|
7
|
+
INVALID = "<invalid>"
|
|
8
|
+
TOPIC_ARN = /\Aarn:aws[a-z-]*:sns:[a-z]{2}(?:-[a-z]+)+-\d+:\d{12}:[A-Za-z0-9_-]{1,256}\z/
|
|
6
9
|
|
|
7
10
|
def call(environment)
|
|
8
|
-
|
|
11
|
+
method = environment["REQUEST_METHOD"]
|
|
12
|
+
unless method == "POST"
|
|
13
|
+
shown = METHODS.include?(method) ? method : INVALID
|
|
14
|
+
return refuse(405, "method #{shown} is not POST")
|
|
15
|
+
end
|
|
9
16
|
|
|
10
17
|
# An unconfigured receiver answers 503 so the provider retries later instead of
|
|
11
18
|
# treating the endpoint as broken or, worse, as having accepted the notification.
|
|
12
19
|
Bouncy.configuration.validate!
|
|
13
20
|
body = environment.fetch("rack.input").read(BODY_LIMIT + 1).to_s
|
|
14
|
-
return
|
|
21
|
+
return refuse(400, "empty body") if body.empty?
|
|
15
22
|
|
|
16
|
-
return
|
|
23
|
+
return refuse(413, "body over #{BODY_LIMIT} bytes") if body.bytesize > BODY_LIMIT
|
|
17
24
|
|
|
18
25
|
adapter = Bouncy.adapter
|
|
19
26
|
envelope = adapter.authenticate(raw_body: body, headers: environment)
|
|
@@ -21,18 +28,62 @@ module Bouncy
|
|
|
21
28
|
raise MalformedMessage, "Unsupported SNS control type" unless envelope["Type"] == "Notification"
|
|
22
29
|
|
|
23
30
|
adapter.parse(envelope).each { |event| Ingestor.new.call(event) }
|
|
31
|
+
return response(503) unless notified?(adapter, envelope)
|
|
24
32
|
end
|
|
25
33
|
response(200)
|
|
26
|
-
rescue AuthenticationError
|
|
27
|
-
|
|
28
|
-
rescue MalformedMessage
|
|
29
|
-
|
|
30
|
-
rescue ProviderError, ConfigurationError
|
|
31
|
-
|
|
34
|
+
rescue AuthenticationError => e
|
|
35
|
+
refuse(401, e.message, body)
|
|
36
|
+
rescue MalformedMessage => e
|
|
37
|
+
refuse(400, e.message, body)
|
|
38
|
+
rescue ProviderError, ConfigurationError => e
|
|
39
|
+
refuse(503, e.message, body)
|
|
40
|
+
rescue ActiveRecord::ActiveRecordError => e
|
|
41
|
+
# A database message can quote a statement and its values, so only the class is logged.
|
|
42
|
+
refuse(503, e.class.name, body)
|
|
32
43
|
end
|
|
33
44
|
|
|
34
45
|
private
|
|
35
46
|
|
|
47
|
+
# One warning per refused message: how a dead subscription is found in production. It names
|
|
48
|
+
# the reason and where the message claims to come from. The message is not authenticated, so
|
|
49
|
+
# a topic is logged only when it has the strict SNS ARN syntax and a SubscribeURL host only
|
|
50
|
+
# when it passes the SNS host check; anything else is written as <invalid>. No raw field of
|
|
51
|
+
# the body is ever copied into the log.
|
|
52
|
+
def refuse(status, reason, body = nil)
|
|
53
|
+
Rails.logger&.warn("[bouncy] refused an SNS message (#{status}): #{reason}#{origin(body)}")
|
|
54
|
+
response(status)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def origin(body)
|
|
58
|
+
envelope = JSON.parse(body.to_s, max_nesting: 30)
|
|
59
|
+
return "" unless envelope.is_a?(Hash)
|
|
60
|
+
|
|
61
|
+
parts = []
|
|
62
|
+
topic = envelope["TopicArn"] if envelope["TopicArn"].is_a?(String) && envelope["TopicArn"].match?(TOPIC_ARN)
|
|
63
|
+
parts << "topic #{topic || INVALID}" if envelope.key?("TopicArn")
|
|
64
|
+
if envelope["Type"] == "SubscriptionConfirmation" && envelope.key?("SubscribeURL")
|
|
65
|
+
uri = Providers::Ses.subscribe_uri(envelope, Bouncy.configuration.ses.region) if topic
|
|
66
|
+
parts << "SubscribeURL host #{uri&.host || INVALID}"
|
|
67
|
+
end
|
|
68
|
+
parts.map { |part| "; #{part}" }.join
|
|
69
|
+
rescue JSON::ParserError
|
|
70
|
+
""
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# config.on_notification runs after the state committed, outside Bouncy's locks, and before
|
|
74
|
+
# the answer. A failure answers 503, so SNS delivers the notification again; the repeat
|
|
75
|
+
# deduplicates and calls the hook again.
|
|
76
|
+
def notified?(adapter, envelope)
|
|
77
|
+
return true unless adapter.respond_to?(:notification)
|
|
78
|
+
|
|
79
|
+
Bouncy.configuration.on_notification.call(adapter.notification(envelope))
|
|
80
|
+
true
|
|
81
|
+
rescue StandardError => e
|
|
82
|
+
ActiveSupport::Notifications.instrument("hook_error.bouncy", hook: :on_notification, error_class: e.class.name)
|
|
83
|
+
Rails.logger&.error("[bouncy] config.on_notification raised #{e.class.name}; answering 503 so SNS retries")
|
|
84
|
+
false
|
|
85
|
+
end
|
|
86
|
+
|
|
36
87
|
def response(status)
|
|
37
88
|
[status, { "content-type" => "text/plain", "cache-control" => "no-store" }, [Rack::Utils::HTTP_STATUS_CODES.fetch(status)]]
|
|
38
89
|
end
|