bouncy 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +37 -0
- data/README.md +195 -92
- data/guides/README.md +20 -0
- data/guides/admin.md +2 -2
- data/guides/amazon-ses.md +70 -9
- data/guides/api.md +148 -0
- data/guides/bounce-handling.md +38 -1
- data/guides/compatibility.md +16 -2
- data/guides/configuration.md +83 -0
- data/guides/delivery.md +46 -8
- data/guides/migrating.md +8 -7
- data/guides/operations.md +104 -0
- data/guides/privacy.md +1 -1
- data/guides/recovery.md +1 -1
- data/guides/troubleshooting.md +1 -1
- data/lib/bouncy/address_lock.rb +321 -0
- data/lib/bouncy/body_limit.rb +55 -0
- data/lib/bouncy/configuration.rb +38 -2
- data/lib/bouncy/event.rb +10 -4
- data/lib/bouncy/importer.rb +100 -0
- data/lib/bouncy/ingestor.rb +12 -3
- data/lib/bouncy/interceptor.rb +85 -16
- data/lib/bouncy/notification.rb +70 -0
- data/lib/bouncy/providers/ses.rb +57 -2
- data/lib/bouncy/providers/ses_parser.rb +20 -0
- data/lib/bouncy/providers/sns_verifier.rb +74 -6
- data/lib/bouncy/prune_job.rb +3 -0
- data/lib/bouncy/scope_lock.rb +3 -2
- data/lib/bouncy/status.rb +3 -0
- data/lib/bouncy/store.rb +9 -1
- data/lib/bouncy/sync_job.rb +4 -0
- data/lib/bouncy/version.rb +1 -1
- data/lib/bouncy/webhook.rb +60 -9
- data/lib/bouncy.rb +44 -3
- data/lib/generators/bouncy/templates/initializer.rb.tt +14 -0
- metadata +14 -3
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bouncy
|
|
4
|
+
# Records a block that another system recorded first, for Bouncy.import!.
|
|
5
|
+
#
|
|
6
|
+
# A bounce or a complaint becomes the same evidence that a webhook records, and a manual block
|
|
7
|
+
# becomes a manual hold. The time is the time the other system recorded the block. It is kept
|
|
8
|
+
# as given, however old: config.maximum_event_age is for webhook feedback that arrives late,
|
|
9
|
+
# and an old block that another system kept still stops mail.
|
|
10
|
+
#
|
|
11
|
+
# An import is not new evidence, so it calls no hook. It is idempotent: the same address,
|
|
12
|
+
# reason and time a second time writes nothing. It never removes or weakens a block that is
|
|
13
|
+
# in force, and a release recorded at or after its time fences it.
|
|
14
|
+
class Importer
|
|
15
|
+
REASONS = %w[hard_bounce complaint manual].freeze
|
|
16
|
+
FUTURE_SKEW = 5.minutes
|
|
17
|
+
# The attributes that an import can change. A difference in them is a change to record.
|
|
18
|
+
CHANGES = %w[manual_blocked_at manual_note event_blocked_at event_reason last_event_at blocked_at details].freeze
|
|
19
|
+
|
|
20
|
+
def call(email, reason:, at:, note:, actor:)
|
|
21
|
+
reason = reason.to_s
|
|
22
|
+
raise ArgumentError, "reason must be :hard_bounce, :complaint or :manual" unless REASONS.include?(reason)
|
|
23
|
+
|
|
24
|
+
time = time_of(at)
|
|
25
|
+
exact = email.to_s.strip
|
|
26
|
+
key = Identity.normalize(email)
|
|
27
|
+
dedupe_key = Digest::SHA256.hexdigest(JSON.generate(["import", Bouncy.scope, key, reason, time.iso8601(6)]))
|
|
28
|
+
Store.change(key) do |row|
|
|
29
|
+
next row if Bouncy.events.exists?(dedupe_key: dedupe_key)
|
|
30
|
+
|
|
31
|
+
record(row, reason, time, dedupe_key, exact: exact, note: note.to_s.strip.truncate(1000).presence,
|
|
32
|
+
actor: actor.to_s.truncate(200))
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def record(row, reason, time, dedupe_key, exact:, note:, actor:)
|
|
39
|
+
was_blocked = row.blocked?
|
|
40
|
+
released_at = fence(row, reason)
|
|
41
|
+
fenced = !released_at.nil? && time <= released_at
|
|
42
|
+
unless fenced
|
|
43
|
+
before = row.attributes.slice(*CHANGES).deep_dup
|
|
44
|
+
reason == "manual" ? hold(row, time, note) : feedback(row, reason, time, exact)
|
|
45
|
+
row.blocked_at = [row.blocked_at, time].compact.min
|
|
46
|
+
return row if row.attributes.slice(*CHANGES) == before
|
|
47
|
+
|
|
48
|
+
row.summarize!
|
|
49
|
+
end
|
|
50
|
+
Store.event!("imported", email: row.email, source: "import", occurred_at: time, dedupe_key: dedupe_key, details: {
|
|
51
|
+
"reason" => reason, "note" => note, "actor" => actor,
|
|
52
|
+
"ignored_for_policy" => fenced, "became_blocked" => !was_blocked && row.blocked?
|
|
53
|
+
})
|
|
54
|
+
row
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# A release clears manual holds and records "soft_released_before". A provider release, or a
|
|
58
|
+
# sync that no longer finds webhook evidence, clears it and records released_before. An
|
|
59
|
+
# imported block at or before that time is older than the release, so it changes nothing.
|
|
60
|
+
def fence(row, reason)
|
|
61
|
+
return row.released_before unless reason == "manual"
|
|
62
|
+
|
|
63
|
+
Time.iso8601(row.details["soft_released_before"].to_s)
|
|
64
|
+
rescue ArgumentError
|
|
65
|
+
nil
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Keeps an earlier hold and its support note.
|
|
69
|
+
def hold(row, time, note)
|
|
70
|
+
return if row.manual_blocked_at && row.manual_blocked_at <= time
|
|
71
|
+
|
|
72
|
+
row.manual_note = note unless row.manual_blocked_at
|
|
73
|
+
row.manual_blocked_at = time
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Webhook evidence keeps the time of its newest block and the stronger reason: a complaint
|
|
77
|
+
# outranks a hard bounce, so an imported hard bounce never replaces a complaint.
|
|
78
|
+
def feedback(row, reason, time, exact)
|
|
79
|
+
previous = row.event_reason if row.event_blocked_at
|
|
80
|
+
return if previous == reason && row.event_blocked_at >= time
|
|
81
|
+
return if previous == "complaint" && row.event_blocked_at >= time
|
|
82
|
+
|
|
83
|
+
row.event_reason = previous == "complaint" ? "complaint" : reason
|
|
84
|
+
row.event_blocked_at = [row.event_blocked_at, time].compact.max
|
|
85
|
+
row.last_event_at = [row.last_event_at, time].compact.max
|
|
86
|
+
row.details = row.details.except("absence_count").reverse_merge("exact_email" => exact)
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Microsecond precision, as every supported database stores it, so a second import of the
|
|
90
|
+
# same time compares and deduplicates as equal.
|
|
91
|
+
def time_of(value)
|
|
92
|
+
raise ArgumentError, "at must be a Time" unless value.is_a?(Time) || value.is_a?(DateTime)
|
|
93
|
+
|
|
94
|
+
time = value.to_time.utc.floor(6)
|
|
95
|
+
raise ArgumentError, "at must not be in the future" if time > FUTURE_SKEW.from_now
|
|
96
|
+
|
|
97
|
+
time
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
data/lib/bouncy/ingestor.rb
CHANGED
|
@@ -1,8 +1,15 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Bouncy
|
|
4
|
+
# Internal: one webhook observation for Ingestor. See Ingestor.
|
|
4
5
|
Observation = Data.define(:email, :kind, :provider_event_id, :message_id, :occurred_at, :details, :provider_reason, :status_code, :diagnostic)
|
|
5
6
|
|
|
7
|
+
# Internal: the receiver's step for one webhook observation. It is not a public API. Use
|
|
8
|
+
# Bouncy.import! to record a block from another system. The call shape of Ingestor#call and
|
|
9
|
+
# Observation stays the same in 0.2.x, because apps on 0.2.0 call them to carry blocks.
|
|
10
|
+
#
|
|
11
|
+
# config.maximum_event_age applies here only. It keeps a late webhook notification from
|
|
12
|
+
# blocking. Bouncy.import! and Bouncy.block! do not apply it.
|
|
6
13
|
class Ingestor
|
|
7
14
|
def call(observation)
|
|
8
15
|
email = observation.email && Identity.normalize(observation.email)
|
|
@@ -44,9 +51,11 @@ module Bouncy
|
|
|
44
51
|
details = attributes.fetch(:details).merge("ignored_for_policy" => [too_old, fenced, out_of_order].any?)
|
|
45
52
|
if row && !details["ignored_for_policy"]
|
|
46
53
|
was_blocked = row.blocked?
|
|
47
|
-
#
|
|
48
|
-
# marks it "confirmed"); candidates wait for the provider's own list
|
|
49
|
-
|
|
54
|
+
# Under :cautious a complaint blocks locally only when the provider named exactly one
|
|
55
|
+
# recipient (SesParser marks it "confirmed"); candidates wait for the provider's own list
|
|
56
|
+
# at the next sync. Under :strict every named recipient blocks.
|
|
57
|
+
complaint = attributes[:kind] == "complaint" && (details["certainty"] == "confirmed" || Bouncy.configuration.webhook_blocking == :strict)
|
|
58
|
+
if attributes[:kind] == "hard_bounce" || complaint
|
|
50
59
|
row.event_blocked_at = time
|
|
51
60
|
row.event_reason = attributes[:kind]
|
|
52
61
|
row.last_event_at = [row.last_event_at, time].compact.max
|
data/lib/bouncy/interceptor.rb
CHANGED
|
@@ -1,30 +1,83 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Bouncy
|
|
4
|
+
# Checks the recipients of every mail where Mail hands it to the transport.
|
|
5
|
+
#
|
|
6
|
+
# Mail calls delivering_email before each delivery. It stamps the configuration-set header and
|
|
7
|
+
# wraps the message's delivery method in a Gate. Mail's deliver and deliver! both end in that
|
|
8
|
+
# delivery method, after every interceptor and every Action Mailer callback, so the Gate
|
|
9
|
+
# decides on the recipients the transport would really get. In :drop mode it removes blocked
|
|
10
|
+
# recipients, and when none is left it does not call the transport: deliver_now!,
|
|
11
|
+
# deliver_later! and their plain forms all send nothing.
|
|
4
12
|
class Interceptor
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
return Bouncy.unconfigured!("interception") unless Bouncy.configured?
|
|
13
|
+
CONFIGURATION_SET_HEADER = "X-SES-CONFIGURATION-SET"
|
|
14
|
+
RECIPIENT_FIELDS = %i[to cc bcc].freeze
|
|
8
15
|
|
|
9
|
-
|
|
16
|
+
# The delivery method of a message that Bouncy checks. Mail builds it through
|
|
17
|
+
# Message#delivery_method(Gate, transport), its public way to set a delivery method.
|
|
18
|
+
class Gate
|
|
19
|
+
attr_reader :transport
|
|
20
|
+
|
|
21
|
+
def initialize(transport)
|
|
22
|
+
@transport = transport
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def settings = transport.settings
|
|
26
|
+
|
|
27
|
+
# Bypass, scope and mode are read again at each delivery: a message delivered once keeps
|
|
28
|
+
# its gate, and can be delivered again under Bouncy.unblocked or another mode.
|
|
29
|
+
def deliver!(message)
|
|
30
|
+
return transport.deliver!(message) unless Interceptor.active?
|
|
31
|
+
return if Interceptor.new.call(message) == :undeliverable
|
|
32
|
+
|
|
33
|
+
transport.deliver!(message)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def respond_to_missing?(name, include_private = false) = transport.respond_to?(name, include_private) || super
|
|
37
|
+
def method_missing(name, ...) = transport.respond_to?(name) ? transport.public_send(name, ...) : super
|
|
10
38
|
end
|
|
11
39
|
|
|
40
|
+
class << self
|
|
41
|
+
def delivering_email(message)
|
|
42
|
+
stamp_configuration_set(message)
|
|
43
|
+
return if bypassed?
|
|
44
|
+
return Bouncy.unconfigured!("interception") unless Bouncy.configured?
|
|
45
|
+
|
|
46
|
+
transport = message.delivery_method
|
|
47
|
+
message.delivery_method(Gate, transport) unless transport.is_a?(Gate)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def active? = !bypassed? && Bouncy.configured?
|
|
51
|
+
|
|
52
|
+
private
|
|
53
|
+
|
|
54
|
+
def bypassed? = Bouncy.configuration.interception == :off || ActiveSupport::IsolatedExecutionState[:bouncy_unblocked]
|
|
55
|
+
|
|
56
|
+
# SES reads the set from this header and removes it before delivery. A mail that names its
|
|
57
|
+
# own set keeps it. The header does not depend on the interception mode: turning
|
|
58
|
+
# interception off must not stop the events that SES publishes for the set.
|
|
59
|
+
def stamp_configuration_set(message)
|
|
60
|
+
name = Bouncy.configuration.ses.configuration_set.presence
|
|
61
|
+
message[CONFIGURATION_SET_HEADER] = name if name && message.header[CONFIGURATION_SET_HEADER].nil?
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Returns :undeliverable when :drop removed every recipient, else nil.
|
|
12
66
|
def call(message)
|
|
13
67
|
envelope = Array(message.smtp_envelope_to).dup
|
|
14
|
-
|
|
15
|
-
keys = (envelope +
|
|
16
|
-
rows = Bouncy.blocked.where(email: keys).to_a
|
|
68
|
+
fields = RECIPIENT_FIELDS.to_h { |name| [name, addresses(message[name])] }
|
|
69
|
+
keys = (envelope + fields.values.flatten.map(&:address)).filter_map { |entry| normalize(entry) }.uniq
|
|
70
|
+
rows = Bouncy.current_state(Bouncy.blocked).where(email: keys).to_a
|
|
17
71
|
return if rows.empty?
|
|
18
72
|
|
|
19
73
|
mode = Bouncy.configuration.interception
|
|
20
|
-
# Provider-derived evidence is only enforced while a complete sync is fresh
|
|
21
|
-
# and legacy soft holds are local policy and
|
|
22
|
-
# rule applies in :log mode so that its
|
|
23
|
-
|
|
74
|
+
# Provider-derived evidence is only enforced while a complete sync is fresh, unless the host
|
|
75
|
+
# set require_fresh_sync to false. Manual holds and legacy soft holds are local policy and
|
|
76
|
+
# do not depend on provider freshness. The same rule applies in :log mode so that its
|
|
77
|
+
# preview matches what :drop would do.
|
|
78
|
+
fresh = !Bouncy.configuration.require_fresh_sync || Bouncy.sync_fresh?
|
|
24
79
|
enforceable = rows.select { |row| fresh || row.manual_blocked_at || (row.soft_blocked_until && row.soft_blocked_until > Time.current) }
|
|
25
80
|
blocked = enforceable.map(&:email)
|
|
26
|
-
remaining = envelope.reject { |email| blocked.include?(normalize(email)) }
|
|
27
|
-
changed_headers = headers.transform_values { |values| values.reject { |email| blocked.include?(normalize(email)) } }
|
|
28
81
|
# Persist all skip evidence before changing the message. A DB outage must leave it intact.
|
|
29
82
|
Event.transaction do
|
|
30
83
|
rows.each do |row|
|
|
@@ -37,19 +90,35 @@ module Bouncy
|
|
|
37
90
|
end
|
|
38
91
|
return unless mode == :drop && blocked.any?
|
|
39
92
|
|
|
40
|
-
|
|
93
|
+
# Each kept entry keeps its own text, display name included; only the lookup uses the
|
|
94
|
+
# bare address.
|
|
95
|
+
fields.each do |name, field_addresses|
|
|
96
|
+
kept = field_addresses.reject { |address| blocked.include?(normalize(address.address)) }
|
|
97
|
+
message[name] = kept.empty? ? nil : kept.map(&:format) if kept.size < field_addresses.size
|
|
98
|
+
end
|
|
99
|
+
remaining = envelope.reject { |entry| blocked.include?(normalize(entry)) }
|
|
41
100
|
message.smtp_envelope_to = remaining
|
|
42
|
-
|
|
101
|
+
return if remaining.any?
|
|
102
|
+
|
|
103
|
+
message.perform_deliveries = false
|
|
104
|
+
:undeliverable
|
|
43
105
|
rescue ActiveRecord::ActiveRecordError => e
|
|
44
106
|
raise unless Bouncy.database_unavailable?(e)
|
|
45
107
|
|
|
46
108
|
ActiveSupport::Notifications.instrument("unavailable.bouncy", operation: "interception")
|
|
109
|
+
nil
|
|
47
110
|
end
|
|
48
111
|
|
|
49
112
|
private
|
|
50
113
|
|
|
114
|
+
# The parsed addresses of a To, Cc or Bcc field, or none for a field Mail cannot parse.
|
|
115
|
+
def addresses(field)
|
|
116
|
+
field.respond_to?(:addrs) ? field.addrs : []
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# "Ada <ADA@example.com>" and "ada@example.com" are the same lookup key.
|
|
51
120
|
def normalize(value)
|
|
52
|
-
Identity.normalize(Mail::Address.new(value).address)
|
|
121
|
+
Identity.normalize(Mail::Address.new(value.to_s).address)
|
|
53
122
|
rescue InvalidAddress, Mail::Field::ParseError
|
|
54
123
|
nil
|
|
55
124
|
end
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bouncy
|
|
4
|
+
# One SES notification, as config.on_notification receives it: every event type SES
|
|
5
|
+
# publishes to the topic, Send and Delivery included, after the receiver committed what it
|
|
6
|
+
# records from it.
|
|
7
|
+
#
|
|
8
|
+
# config.on_notification = lambda do |notification|
|
|
9
|
+
# notification.type # => "Delivery"
|
|
10
|
+
# notification.message_id # => the SES message id of the mail
|
|
11
|
+
# notification.recipients # => ["ada@example.com"]
|
|
12
|
+
# notification.occurred_at # => the time SES gives for this event
|
|
13
|
+
# notification.tags # => { "order" => ["42"] }
|
|
14
|
+
# notification.raw # => the SES JSON object, as a Hash
|
|
15
|
+
# end
|
|
16
|
+
#
|
|
17
|
+
# SNS repeats and reorders notifications, so the hook must be safe to run again and in any
|
|
18
|
+
# order.
|
|
19
|
+
class Notification
|
|
20
|
+
# The SES object that holds an event's own details, where it is not the type in camelCase.
|
|
21
|
+
SECTIONS = { "Rendering Failure" => "failure", "RenderingFailure" => "failure" }.freeze
|
|
22
|
+
|
|
23
|
+
attr_reader :raw
|
|
24
|
+
|
|
25
|
+
def initialize(raw, sns_timestamp: nil)
|
|
26
|
+
@raw = raw
|
|
27
|
+
@sns_timestamp = sns_timestamp
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# The SES event type as SES names it: "Send", "Delivery", "Bounce", "Complaint", ...
|
|
31
|
+
def type = raw["eventType"] || raw["notificationType"]
|
|
32
|
+
|
|
33
|
+
def message_id = mail["messageId"]
|
|
34
|
+
|
|
35
|
+
# The SES message tags of the mail: each name maps to an array of values.
|
|
36
|
+
def tags = mail["tags"].is_a?(Hash) ? mail["tags"] : {}
|
|
37
|
+
|
|
38
|
+
# The addresses this event is about, exactly as SES spells them: the bounced, complained,
|
|
39
|
+
# delivered or delayed recipients, or the mail's destination for other events.
|
|
40
|
+
def recipients
|
|
41
|
+
values = case type
|
|
42
|
+
when "Bounce" then section["bouncedRecipients"]
|
|
43
|
+
when "Complaint" then section["complainedRecipients"]
|
|
44
|
+
when "Delivery" then section["recipients"]
|
|
45
|
+
when "DeliveryDelay" then section["delayedRecipients"]
|
|
46
|
+
else mail["destination"]
|
|
47
|
+
end
|
|
48
|
+
Array(values).filter_map do |recipient|
|
|
49
|
+
address = recipient.is_a?(Hash) ? recipient["emailAddress"] : recipient
|
|
50
|
+
address if address.is_a?(String)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# The time of this event: its own timestamp, else the mail's, else the time SNS published it.
|
|
55
|
+
def occurred_at
|
|
56
|
+
[section["timestamp"], mail["timestamp"], @sns_timestamp].each do |value|
|
|
57
|
+
return Time.iso8601(value) if value.is_a?(String)
|
|
58
|
+
rescue ArgumentError
|
|
59
|
+
next
|
|
60
|
+
end
|
|
61
|
+
nil
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
private
|
|
65
|
+
|
|
66
|
+
def mail = object(raw["mail"])
|
|
67
|
+
def section = object(raw[SECTIONS.fetch(type.to_s) { type.to_s.sub(/\A./, &:downcase) }])
|
|
68
|
+
def object(value) = value.is_a?(Hash) ? value : {}
|
|
69
|
+
end
|
|
70
|
+
end
|
data/lib/bouncy/providers/ses.rb
CHANGED
|
@@ -3,6 +3,24 @@
|
|
|
3
3
|
module Bouncy
|
|
4
4
|
module Providers
|
|
5
5
|
class Ses < Base
|
|
6
|
+
# The SNS host domain of each AWS partition, for the certificate and SubscribeURL checks.
|
|
7
|
+
SNS_DOMAINS = { "aws" => "amazonaws.com", "aws-us-gov" => "amazonaws.com", "aws-cn" => "amazonaws.com.cn" }.freeze
|
|
8
|
+
# SNS answers a SubscribeURL visit with a short XML document.
|
|
9
|
+
CONFIRMATION_LIMIT = 64 * 1024
|
|
10
|
+
|
|
11
|
+
# The SubscribeURL as a URI when it is HTTPS on the topic's own regional SNS host, port 443,
|
|
12
|
+
# with no userinfo and no fragment; nil for any other value.
|
|
13
|
+
def self.subscribe_uri(envelope, region)
|
|
14
|
+
uri = URI.parse(envelope["SubscribeURL"].to_s)
|
|
15
|
+
domain = SNS_DOMAINS[envelope["TopicArn"].to_s.split(":")[1]]
|
|
16
|
+
return unless domain && uri.is_a?(URI::HTTPS) && uri.host == "sns.#{region}.#{domain}" && uri.port == 443
|
|
17
|
+
return if uri.userinfo || uri.fragment
|
|
18
|
+
|
|
19
|
+
uri
|
|
20
|
+
rescue URI::InvalidURIError
|
|
21
|
+
nil
|
|
22
|
+
end
|
|
23
|
+
|
|
6
24
|
def initialize(configuration = Bouncy.configuration)
|
|
7
25
|
super()
|
|
8
26
|
@configuration = configuration
|
|
@@ -70,8 +88,12 @@ module Bouncy
|
|
|
70
88
|
def control(envelope) # rubocop:disable Naming/PredicateMethod -- Adapter operation with side effects.
|
|
71
89
|
case envelope.fetch("Type")
|
|
72
90
|
when "SubscriptionConfirmation"
|
|
73
|
-
|
|
74
|
-
|
|
91
|
+
if @settings.subscription_confirmation == :subscribe_url
|
|
92
|
+
visit_subscribe_url(envelope)
|
|
93
|
+
else
|
|
94
|
+
request { sns_client.confirm_subscription(topic_arn: envelope.fetch("TopicArn"), token: envelope.fetch("Token")) }
|
|
95
|
+
end
|
|
96
|
+
Store.event!("subscription_confirmed", source: "webhook", details: { "via" => @settings.subscription_confirmation.to_s })
|
|
75
97
|
true
|
|
76
98
|
when "UnsubscribeConfirmation"
|
|
77
99
|
Store.event!("subscription_unsubscribed", source: "webhook")
|
|
@@ -85,6 +107,11 @@ module Bouncy
|
|
|
85
107
|
SesParser.new(@configuration).call(envelope)
|
|
86
108
|
end
|
|
87
109
|
|
|
110
|
+
# The notification for config.on_notification. parse has already checked the message.
|
|
111
|
+
def notification(envelope)
|
|
112
|
+
Notification.new(JSON.parse(envelope.fetch("Message"), max_nesting: 20), sns_timestamp: envelope["Timestamp"])
|
|
113
|
+
end
|
|
114
|
+
|
|
88
115
|
def doctor
|
|
89
116
|
policy = policy_check
|
|
90
117
|
{ "scope" => @configuration.scope, "policy_verified" => policy.verified, "policy_reason" => policy.reason,
|
|
@@ -94,6 +121,33 @@ module Bouncy
|
|
|
94
121
|
|
|
95
122
|
private
|
|
96
123
|
|
|
124
|
+
# Confirms the subscription by visiting the SubscribeURL that SNS signed, so the credentials
|
|
125
|
+
# need no sns:ConfirmSubscription grant. Only an HTTPS URL on the topic's own regional SNS
|
|
126
|
+
# host is visited; any other URL is malformed for good. A transport failure or an answer
|
|
127
|
+
# other than 2xx is a ProviderError: the receiver answers 503, and SNS sends the
|
|
128
|
+
# confirmation again.
|
|
129
|
+
def visit_subscribe_url(envelope)
|
|
130
|
+
require "net/http"
|
|
131
|
+
uri = self.class.subscribe_uri(envelope, @settings.region)
|
|
132
|
+
raise MalformedMessage, "SubscribeURL is not on the topic's SNS host" unless uri
|
|
133
|
+
|
|
134
|
+
code = nil
|
|
135
|
+
Net::HTTP.start(uri.host, uri.port, use_ssl: true, open_timeout: 3, read_timeout: 5,
|
|
136
|
+
verify_mode: OpenSSL::SSL::VERIFY_PEER, max_retries: 0) do |http|
|
|
137
|
+
http.request(Net::HTTP::Get.new(uri.request_uri)) do |response|
|
|
138
|
+
code = response.code
|
|
139
|
+
size = 0
|
|
140
|
+
response.read_body do |chunk|
|
|
141
|
+
size += chunk.bytesize
|
|
142
|
+
raise ProviderError, "SNS confirmation response exceeds limit" if size > CONFIRMATION_LIMIT
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
raise ProviderError, "SNS did not confirm the subscription (HTTP #{code})" unless code.to_s.match?(/\A2\d\d\z/)
|
|
147
|
+
rescue Timeout::Error, SocketError, IOError, SystemCallError, OpenSSL::SSL::SSLError
|
|
148
|
+
raise ProviderError, "SNS confirmation transport unavailable"
|
|
149
|
+
end
|
|
150
|
+
|
|
97
151
|
def require_sdk(service)
|
|
98
152
|
require "aws-sdk-#{service}"
|
|
99
153
|
rescue LoadError
|
|
@@ -160,6 +214,7 @@ module Bouncy
|
|
|
160
214
|
end
|
|
161
215
|
|
|
162
216
|
sets = @settings.configuration_sets.dup
|
|
217
|
+
sets << @settings.configuration_set if @settings.configuration_set.present?
|
|
163
218
|
@settings.identities.each do |identity|
|
|
164
219
|
response = request { ses.get_email_identity(email_identity: identity) }
|
|
165
220
|
sets << response.configuration_set_name if response.configuration_set_name.present?
|
|
@@ -79,6 +79,8 @@ module Bouncy
|
|
|
79
79
|
end
|
|
80
80
|
|
|
81
81
|
def classify(type, data)
|
|
82
|
+
return strict_classification(type, data) if @configuration.webhook_blocking == :strict && %w[Bounce Complaint].include?(type)
|
|
83
|
+
|
|
82
84
|
case type
|
|
83
85
|
when "Bounce"
|
|
84
86
|
case data["bounceSubType"]
|
|
@@ -107,6 +109,24 @@ module Bouncy
|
|
|
107
109
|
else %w[ignored unsupported_type]
|
|
108
110
|
end
|
|
109
111
|
end
|
|
112
|
+
|
|
113
|
+
# config.webhook_blocking = :strict: nothing reads the account list, so every Permanent
|
|
114
|
+
# bounce blocks, whatever its subtype, and so does every recipient a complaint names.
|
|
115
|
+
# SES cannot say which one complained: most ISPs remove that address, and SES lists the
|
|
116
|
+
# recipients who might have sent the complaint. Undetermined and Transient bounces stay
|
|
117
|
+
# record-only, and a not-spam report says the opposite of a complaint.
|
|
118
|
+
def strict_classification(type, data)
|
|
119
|
+
if type == "Bounce"
|
|
120
|
+
return ["hard_bounce", data["bounceSubType"].presence || "unspecified"] if data["bounceType"] == "Permanent"
|
|
121
|
+
|
|
122
|
+
return ["soft_bounce", data["bounceSubType"]] if %w[Transient Undetermined].include?(data["bounceType"])
|
|
123
|
+
|
|
124
|
+
return %w[unknown unrecognized_bounce]
|
|
125
|
+
end
|
|
126
|
+
return %w[ignored not_spam] if data["complaintFeedbackType"] == "not-spam"
|
|
127
|
+
|
|
128
|
+
["complaint", data["complaintSubType"].presence || "named"]
|
|
129
|
+
end
|
|
110
130
|
end
|
|
111
131
|
end
|
|
112
132
|
end
|
|
@@ -7,13 +7,71 @@ module Bouncy
|
|
|
7
7
|
class SnsVerifier
|
|
8
8
|
BODY_LIMIT = 2 * 1024 * 1024
|
|
9
9
|
|
|
10
|
+
# Signing certificates, shared by every receiver in the process: each request builds a new
|
|
11
|
+
# adapter, and an app that subscribes to Send and Delivery events gets one SNS message per
|
|
12
|
+
# mail. At most 16 certificate URLs, each kept for an hour. Only a certificate that the
|
|
13
|
+
# allowlisted, bounded download returned is kept; a failed download keeps nothing.
|
|
14
|
+
class CertificateCache
|
|
15
|
+
Entry = Data.define(:pem, :fetched_at)
|
|
16
|
+
|
|
17
|
+
def initialize(size:, ttl:)
|
|
18
|
+
@size = size
|
|
19
|
+
@ttl = ttl
|
|
20
|
+
@entries = {}
|
|
21
|
+
@mutex = Mutex.new
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# The download runs outside the lock, so one slow fetch does not stop other requests.
|
|
25
|
+
def fetch(url)
|
|
26
|
+
entry = @mutex.synchronize { @entries[url] }
|
|
27
|
+
return entry.pem if entry && Time.now - entry.fetched_at < @ttl
|
|
28
|
+
|
|
29
|
+
pem = yield
|
|
30
|
+
@mutex.synchronize do
|
|
31
|
+
@entries.delete(url)
|
|
32
|
+
@entries[url] = Entry.new(pem: pem, fetched_at: Time.now)
|
|
33
|
+
@entries.shift while @entries.size > @size
|
|
34
|
+
end
|
|
35
|
+
pem
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Drops a certificate that failed a verification, so the next one fetches it again. A
|
|
39
|
+
# certificate fetched less than `after` seconds ago stays: a stream of forged messages
|
|
40
|
+
# must not turn into a stream of downloads.
|
|
41
|
+
def forget(url, after:)
|
|
42
|
+
@mutex.synchronize do
|
|
43
|
+
entry = @entries[url]
|
|
44
|
+
next false unless entry && Time.now - entry.fetched_at >= after
|
|
45
|
+
|
|
46
|
+
@entries.delete(url)
|
|
47
|
+
true
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def size = @mutex.synchronize { @entries.size }
|
|
52
|
+
def clear = @mutex.synchronize { @entries.clear }
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
CERTIFICATES = CertificateCache.new(size: 16, ttl: 3600)
|
|
56
|
+
# A cached certificate older than this is fetched again, once, when a signature fails.
|
|
57
|
+
REFETCH_AFTER = 60
|
|
58
|
+
|
|
10
59
|
class BoundedVerifier < Aws::SNS::MessageVerifier
|
|
11
60
|
private
|
|
12
61
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
62
|
+
# The SDK's download_pem checks the URL and the PEM framing, and calls the bounded
|
|
63
|
+
# https_get below. A certificate is cached only once it parses and carries the RSA key
|
|
64
|
+
# that SNS signs with; any other body is refused and never kept, so the next request
|
|
65
|
+
# fetches again at once.
|
|
66
|
+
def pem(uri) = CERTIFICATES.fetch(uri.to_s) { usable(download_pem(uri)) }
|
|
67
|
+
|
|
68
|
+
def usable(pem)
|
|
69
|
+
key = OpenSSL::X509::Certificate.new(pem).public_key
|
|
70
|
+
raise VerificationError, "The SNS signing certificate has no RSA key" unless key.is_a?(OpenSSL::PKey::RSA)
|
|
71
|
+
|
|
72
|
+
pem
|
|
73
|
+
rescue OpenSSL::X509::CertificateError, OpenSSL::PKey::PKeyError
|
|
74
|
+
raise VerificationError, "The SNS signing certificate does not parse"
|
|
17
75
|
end
|
|
18
76
|
|
|
19
77
|
def https_get(uri, _failed_attempts = 0)
|
|
@@ -65,7 +123,7 @@ module Bouncy
|
|
|
65
123
|
end
|
|
66
124
|
|
|
67
125
|
authorize!(envelope)
|
|
68
|
-
|
|
126
|
+
verify(raw_body, envelope.fetch("SigningCertURL"))
|
|
69
127
|
envelope
|
|
70
128
|
rescue JSON::ParserError, URI::InvalidURIError
|
|
71
129
|
raise MalformedMessage, "Malformed SNS envelope"
|
|
@@ -75,6 +133,16 @@ module Bouncy
|
|
|
75
133
|
|
|
76
134
|
private
|
|
77
135
|
|
|
136
|
+
# A certificate that SNS rotated at the same URL fails once from the cache; it is fetched
|
|
137
|
+
# again, once, and the message is checked with the new one.
|
|
138
|
+
def verify(raw_body, url)
|
|
139
|
+
@mutex.synchronize { @verifier.authenticate!(raw_body) }
|
|
140
|
+
rescue Aws::SNS::MessageVerifier::VerificationError, OpenSSL::OpenSSLError
|
|
141
|
+
raise unless @verifier.is_a?(BoundedVerifier) && CERTIFICATES.forget(url, after: REFETCH_AFTER)
|
|
142
|
+
|
|
143
|
+
@mutex.synchronize { @verifier.authenticate!(raw_body) }
|
|
144
|
+
end
|
|
145
|
+
|
|
78
146
|
def authorize!(envelope)
|
|
79
147
|
topic = envelope.fetch("TopicArn")
|
|
80
148
|
settings = @configuration.ses
|
|
@@ -84,7 +152,7 @@ module Bouncy
|
|
|
84
152
|
raise AuthenticationError, "SNS topic is not authorized"
|
|
85
153
|
end
|
|
86
154
|
|
|
87
|
-
suffix =
|
|
155
|
+
suffix = Ses::SNS_DOMAINS[parts[1]]
|
|
88
156
|
uri = URI.parse(envelope.fetch("SigningCertURL"))
|
|
89
157
|
unless suffix && uri.scheme == "https" && uri.host == "sns.#{settings.region}.#{suffix}" &&
|
|
90
158
|
uri.port == 443 && !uri.userinfo && !uri.query && !uri.fragment &&
|
data/lib/bouncy/prune_job.rb
CHANGED
|
@@ -4,7 +4,10 @@ module Bouncy
|
|
|
4
4
|
class PruneJob < ActiveJob::Base
|
|
5
5
|
queue_as :default
|
|
6
6
|
|
|
7
|
+
# Like SyncJob, a no-op until config.scope is set.
|
|
7
8
|
def perform
|
|
9
|
+
return Rails.logger&.info("[bouncy] Bouncy::PruneJob skipped: config.scope is not set") unless Bouncy.configured?
|
|
10
|
+
|
|
8
11
|
Bouncy.configuration.validate!
|
|
9
12
|
# Keep state rows, including inactive release fences, indefinitely.
|
|
10
13
|
Bouncy.events.where("created_at < ?", Bouncy.configuration.retention.ago).in_batches.delete_all
|
data/lib/bouncy/scope_lock.rb
CHANGED
|
@@ -13,7 +13,8 @@ module Bouncy
|
|
|
13
13
|
key = Digest::SHA256.hexdigest(Bouncy.scope)[0, 15].to_i(16)
|
|
14
14
|
case connection.adapter_name
|
|
15
15
|
when "PostgreSQL"
|
|
16
|
-
|
|
16
|
+
# Uncached: a cached answer would report the lock without taking it.
|
|
17
|
+
acquired = connection.uncached { connection.select_value("SELECT pg_try_advisory_lock(#{key})") }
|
|
17
18
|
raise ProviderError, "Another sync holds this scope lock" unless [true, "t"].include?(acquired)
|
|
18
19
|
|
|
19
20
|
begin
|
|
@@ -23,7 +24,7 @@ module Bouncy
|
|
|
23
24
|
end
|
|
24
25
|
when /Mysql|Trilogy/i
|
|
25
26
|
name = Digest::SHA256.hexdigest("bouncy:#{connection.pool.db_config.database}:#{Bouncy.scope}")
|
|
26
|
-
acquired = connection.select_value("SELECT GET_LOCK('#{name}', 0)")
|
|
27
|
+
acquired = connection.uncached { connection.select_value("SELECT GET_LOCK('#{name}', 0)") }
|
|
27
28
|
raise ProviderError, "Another sync holds this scope lock" unless acquired == 1
|
|
28
29
|
|
|
29
30
|
begin
|
data/lib/bouncy/status.rb
CHANGED
|
@@ -29,6 +29,9 @@ module Bouncy
|
|
|
29
29
|
def blocked? = @row ? @row.blocked? : false
|
|
30
30
|
def reasons = @row ? @row.reasons : []
|
|
31
31
|
def reason = (%i[complaint hard_bounce provider_list manual soft_bounces] & reasons).first
|
|
32
|
+
# The time the block in force began: when the address became blocked here, or the at: time
|
|
33
|
+
# of the Bouncy.import! that blocked it. nil when the address is not blocked.
|
|
34
|
+
def blocked_at = blocked? ? @row.blocked_at : nil
|
|
32
35
|
def observed_at = @row&.provider_checked_at || @row&.last_event_at
|
|
33
36
|
def provider_updated_at = @row&.provider_entries&.filter_map { |entry| Time.iso8601(entry.fetch("provider_updated_at")) }&.max
|
|
34
37
|
def last_event = @row && Bouncy.events.for(@row.email).recent.first
|
data/lib/bouncy/store.rb
CHANGED
|
@@ -5,15 +5,23 @@ module Bouncy
|
|
|
5
5
|
module_function
|
|
6
6
|
|
|
7
7
|
# Retry outside the transaction: PostgreSQL aborts a transaction on a unique violation.
|
|
8
|
+
# The address lock comes first, before any row lock: a sender inside Bouncy.hold sees this
|
|
9
|
+
# change either committed or not started.
|
|
8
10
|
def change(email)
|
|
9
11
|
key = Identity.normalize(email)
|
|
12
|
+
Record.connection_pool.with_connection { |connection| AddressLock.settle(connection) }
|
|
10
13
|
attempts = 0
|
|
11
14
|
begin
|
|
12
15
|
Suppression.transaction(requires_new: true) do
|
|
16
|
+
AddressLock.claim!(key)
|
|
13
17
|
row = Suppression.find_or_create_by!(scope: Bouncy.scope, email: key) do |record|
|
|
14
18
|
record.provider = Bouncy.configuration.provider.to_s
|
|
15
19
|
end
|
|
16
|
-
row.with_lock
|
|
20
|
+
row.with_lock do
|
|
21
|
+
# A soft hold that ran out leaves its start in blocked_at. The next block starts anew.
|
|
22
|
+
row.blocked_at = nil unless row.blocked?
|
|
23
|
+
yield row
|
|
24
|
+
end
|
|
17
25
|
end
|
|
18
26
|
rescue ActiveRecord::RecordNotUnique, ActiveRecord::StaleObjectError, ActiveRecord::Deadlocked
|
|
19
27
|
attempts += 1
|
data/lib/bouncy/sync_job.rb
CHANGED
|
@@ -6,7 +6,11 @@ module Bouncy
|
|
|
6
6
|
class SyncJob < ActiveJob::Base
|
|
7
7
|
queue_as :default
|
|
8
8
|
|
|
9
|
+
# A recurring job in an app that has not set config.scope does nothing, instead of failing
|
|
10
|
+
# every run. Bouncy.sync! itself still raises.
|
|
9
11
|
def perform
|
|
12
|
+
return Rails.logger&.info("[bouncy] Bouncy::SyncJob skipped: config.scope is not set") unless Bouncy.configured?
|
|
13
|
+
|
|
10
14
|
Bouncy.sync!
|
|
11
15
|
end
|
|
12
16
|
end
|
data/lib/bouncy/version.rb
CHANGED