bouncy 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +32 -0
- data/LICENSE.txt +21 -0
- data/README.md +213 -0
- data/config/routes.rb +5 -0
- data/guides/admin.md +71 -0
- data/guides/amazon-ses.md +63 -0
- data/guides/bounce-handling.md +27 -0
- data/guides/compatibility.md +42 -0
- data/guides/delivery.md +21 -0
- data/guides/migrating.md +15 -0
- data/guides/privacy.md +13 -0
- data/guides/recovery.md +28 -0
- data/guides/troubleshooting.md +46 -0
- data/lib/bouncy/configuration.rb +80 -0
- data/lib/bouncy/engine.rb +15 -0
- data/lib/bouncy/event.rb +22 -0
- data/lib/bouncy/identity.rb +18 -0
- data/lib/bouncy/ingestor.rb +94 -0
- data/lib/bouncy/interceptor.rb +57 -0
- data/lib/bouncy/model.rb +30 -0
- data/lib/bouncy/providers/base.rb +24 -0
- data/lib/bouncy/providers/ses.rb +197 -0
- data/lib/bouncy/providers/ses_parser.rb +112 -0
- data/lib/bouncy/providers/sns_verifier.rb +97 -0
- data/lib/bouncy/prune_job.rb +13 -0
- data/lib/bouncy/reconciler.rb +150 -0
- data/lib/bouncy/record.rb +16 -0
- data/lib/bouncy/recovery.rb +56 -0
- data/lib/bouncy/scope_lock.rb +52 -0
- data/lib/bouncy/status.rb +59 -0
- data/lib/bouncy/status_set.rb +31 -0
- data/lib/bouncy/store.rb +30 -0
- data/lib/bouncy/suppression.rb +33 -0
- data/lib/bouncy/sync_job.rb +13 -0
- data/lib/bouncy/version.rb +5 -0
- data/lib/bouncy/webhook.rb +40 -0
- data/lib/bouncy.rb +185 -0
- data/lib/generators/bouncy/install_generator.rb +37 -0
- data/lib/generators/bouncy/templates/create_bouncy_tables.rb.erb +97 -0
- data/lib/generators/bouncy/templates/initializer.rb.tt +23 -0
- data/lib/tasks/bouncy.rake +36 -0
- metadata +126 -0
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bouncy
|
|
4
|
+
module Providers
|
|
5
|
+
class SesParser
|
|
6
|
+
RECIPIENT_LIMIT = 1000
|
|
7
|
+
|
|
8
|
+
def initialize(configuration)
|
|
9
|
+
@configuration = configuration
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def call(envelope)
|
|
13
|
+
payload = JSON.parse(envelope.fetch("Message"), max_nesting: 20)
|
|
14
|
+
raise MalformedMessage, "Expected SES object" unless payload.is_a?(Hash)
|
|
15
|
+
|
|
16
|
+
type = payload["notificationType"] || payload["eventType"]
|
|
17
|
+
section = case type
|
|
18
|
+
when "Bounce" then "bounce"
|
|
19
|
+
when "Complaint" then "complaint"
|
|
20
|
+
when "Delivery" then "delivery"
|
|
21
|
+
when "DeliveryDelay" then "deliveryDelay"
|
|
22
|
+
else ""
|
|
23
|
+
end
|
|
24
|
+
data = payload.fetch(section, {})
|
|
25
|
+
raise MalformedMessage, "Invalid SES event details" unless data.is_a?(Hash)
|
|
26
|
+
|
|
27
|
+
recipients = case type
|
|
28
|
+
when "Bounce" then data.fetch("bouncedRecipients", [])
|
|
29
|
+
when "Complaint" then data.fetch("complainedRecipients", [])
|
|
30
|
+
when "Delivery" then @configuration.record_deliveries ? data.fetch("recipients", []) : []
|
|
31
|
+
when "DeliveryDelay" then data.fetch("delayedRecipients", [])
|
|
32
|
+
else []
|
|
33
|
+
end
|
|
34
|
+
raise MalformedMessage, "Invalid SES recipient count" unless recipients.is_a?(Array) && recipients.size <= RECIPIENT_LIMIT
|
|
35
|
+
|
|
36
|
+
mail = payload.fetch("mail", {})
|
|
37
|
+
raise MalformedMessage, "Invalid SES mail metadata" unless mail.is_a?(Hash)
|
|
38
|
+
|
|
39
|
+
time, fallback = timestamp(data["timestamp"], envelope.fetch("Timestamp"))
|
|
40
|
+
recipients = [nil] if recipients.empty?
|
|
41
|
+
recipients.map do |recipient|
|
|
42
|
+
raw_email = recipient.is_a?(Hash) ? recipient["emailAddress"] : recipient
|
|
43
|
+
email = normalize(raw_email)
|
|
44
|
+
kind, classification = classify(type, data)
|
|
45
|
+
details = { "exact_email" => raw_email.to_s.truncate(254), "timestamp_fallback" => fallback, "classification" => classification }
|
|
46
|
+
details["escalation_eligible"] = data["bounceSubType"] == "MailboxFull" if kind == "soft_bounce"
|
|
47
|
+
# SES lists every recipient of the message when the mailbox provider redacts the
|
|
48
|
+
# complainer; a single named recipient is the complainer. Only that case blocks locally.
|
|
49
|
+
details["certainty"] = recipients.size == 1 ? "confirmed" : "candidate" if kind == "complaint"
|
|
50
|
+
details["invalid_recipient"] = true if raw_email && !email
|
|
51
|
+
recipientless_kind = %w[reject rendering_failure ignored].include?(kind) && raw_email.nil?
|
|
52
|
+
Observation.new(email: email, kind: email || recipientless_kind ? kind : "ignored",
|
|
53
|
+
provider_event_id: bounded(data["feedbackId"] || envelope.fetch("MessageId"), 255),
|
|
54
|
+
message_id: bounded(mail["messageId"], 255), occurred_at: time, details: details,
|
|
55
|
+
provider_reason: bounded(data["bounceSubType"] || data["complaintSubType"] || data["complaintFeedbackType"], 255),
|
|
56
|
+
status_code: recipient.is_a?(Hash) ? bounded(recipient["status"], 255) : nil,
|
|
57
|
+
diagnostic: recipient.is_a?(Hash) ? bounded(recipient["diagnosticCode"], 1000) : nil)
|
|
58
|
+
end
|
|
59
|
+
rescue JSON::ParserError
|
|
60
|
+
raise MalformedMessage, "Malformed SES payload"
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
private
|
|
64
|
+
|
|
65
|
+
def normalize(value)
|
|
66
|
+
Identity.normalize(value)
|
|
67
|
+
rescue InvalidAddress
|
|
68
|
+
nil
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def bounded(value, length)
|
|
72
|
+
value&.to_s&.truncate(length)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def timestamp(value, fallback)
|
|
76
|
+
[Time.iso8601(value || fallback), value.nil?]
|
|
77
|
+
rescue ArgumentError, TypeError
|
|
78
|
+
raise MalformedMessage, "Invalid SES timestamp"
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def classify(type, data)
|
|
82
|
+
case type
|
|
83
|
+
when "Bounce"
|
|
84
|
+
case data["bounceSubType"]
|
|
85
|
+
when "OnAccountSuppressionList" then %w[provider_suppressed account]
|
|
86
|
+
when "Suppressed", "OnTenantSuppressionList", "EmailValidationSuppressed" then ["unknown", data["bounceSubType"]]
|
|
87
|
+
when "UnsubscribedRecipient" then %w[unsubscribe list]
|
|
88
|
+
else
|
|
89
|
+
if data["bounceType"] == "Permanent" && %w[General NoEmail].include?(data["bounceSubType"])
|
|
90
|
+
%w[hard_bounce address]
|
|
91
|
+
elsif %w[Transient Undetermined].include?(data["bounceType"])
|
|
92
|
+
["soft_bounce", data["bounceSubType"]]
|
|
93
|
+
else
|
|
94
|
+
%w[unknown unrecognized_bounce]
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
when "Complaint"
|
|
98
|
+
case data["complaintSubType"]
|
|
99
|
+
when "OnAccountSuppressionList" then %w[provider_suppressed account]
|
|
100
|
+
when nil then data["complaintFeedbackType"] == "not-spam" ? %w[ignored not_spam] : %w[complaint candidate]
|
|
101
|
+
else ["unknown", data["complaintSubType"]]
|
|
102
|
+
end
|
|
103
|
+
when "Delivery" then %w[delivery server_accepted]
|
|
104
|
+
when "DeliveryDelay" then %w[delay retrying]
|
|
105
|
+
when "Reject" then %w[reject message]
|
|
106
|
+
when "Rendering Failure", "RenderingFailure" then %w[rendering_failure message]
|
|
107
|
+
else %w[ignored unsupported_type]
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
end
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "aws-sdk-sns"
|
|
4
|
+
|
|
5
|
+
module Bouncy
|
|
6
|
+
module Providers
|
|
7
|
+
class SnsVerifier
|
|
8
|
+
BODY_LIMIT = 2 * 1024 * 1024
|
|
9
|
+
|
|
10
|
+
class BoundedVerifier < Aws::SNS::MessageVerifier
|
|
11
|
+
private
|
|
12
|
+
|
|
13
|
+
def pem(uri)
|
|
14
|
+
@cached_pems ||= {}
|
|
15
|
+
@cached_pems.shift if @cached_pems.size >= 32 && !@cached_pems.key?(uri.to_s)
|
|
16
|
+
super
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def https_get(uri, _failed_attempts = 0)
|
|
20
|
+
body = +""
|
|
21
|
+
Net::HTTP.start(uri.host, uri.port, use_ssl: true, open_timeout: 3, read_timeout: 5,
|
|
22
|
+
verify_mode: OpenSSL::SSL::VERIFY_PEER, max_retries: 0) do |http|
|
|
23
|
+
http.request(Net::HTTP::Get.new(uri.request_uri)) do |response|
|
|
24
|
+
raise ProviderError, "SNS certificate fetch failed" unless response.code == "200"
|
|
25
|
+
|
|
26
|
+
response.read_body do |chunk|
|
|
27
|
+
body << chunk
|
|
28
|
+
raise AuthenticationError, "SNS certificate exceeds limit" if body.bytesize > 64 * 1024
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
body
|
|
33
|
+
rescue Timeout::Error, SocketError, IOError, SystemCallError, OpenSSL::SSL::SSLError
|
|
34
|
+
raise ProviderError, "SNS certificate transport unavailable"
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# The bounded verifier overrides two private SDK methods. Refuse to run against an SDK that
|
|
39
|
+
# no longer has them rather than silently fetching certificates without limits.
|
|
40
|
+
HOOKS = %i[pem https_get].freeze
|
|
41
|
+
|
|
42
|
+
def initialize(configuration)
|
|
43
|
+
@configuration = configuration
|
|
44
|
+
unless HOOKS.all? { |hook| Aws::SNS::MessageVerifier.private_method_defined?(hook) }
|
|
45
|
+
raise ConfigurationError, "aws-sdk-sns #{Aws::SNS::GEM_VERSION} changed its certificate download internals; " \
|
|
46
|
+
"pin aws-sdk-sns to a release tested with this version of bouncy"
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
@verifier = configuration.ses.sns_message_verifier || BoundedVerifier.new
|
|
50
|
+
@mutex = Mutex.new
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def call(raw_body)
|
|
54
|
+
envelope = JSON.parse(raw_body, max_nesting: 30)
|
|
55
|
+
raise MalformedMessage, "Expected an SNS object" unless envelope.is_a?(Hash)
|
|
56
|
+
# The SDK supports Lambda aliases by rewriting them. This HTTP receiver
|
|
57
|
+
# must not allow an unvalidated alias to replace the checked certificate URL.
|
|
58
|
+
raise MalformedMessage, "Lambda SNS aliases are not supported" if envelope.key?("SigningCertUrl")
|
|
59
|
+
|
|
60
|
+
fields = %w[Type Message MessageId Timestamp TopicArn Signature SignatureVersion SigningCertURL]
|
|
61
|
+
fields += %w[Token SubscribeURL] if %w[SubscriptionConfirmation UnsubscribeConfirmation].include?(envelope["Type"])
|
|
62
|
+
raise MalformedMessage, "Missing SNS fields" unless fields.all? { |key| envelope[key].is_a?(String) && !envelope[key].empty? }
|
|
63
|
+
unless (Aws::SNS::MessageVerifier::SIGNABLE_KEYS & envelope.keys).all? { |key| envelope[key].is_a?(String) }
|
|
64
|
+
raise MalformedMessage, "SNS signable fields must be strings"
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
authorize!(envelope)
|
|
68
|
+
@mutex.synchronize { @verifier.authenticate!(raw_body) }
|
|
69
|
+
envelope
|
|
70
|
+
rescue JSON::ParserError, URI::InvalidURIError
|
|
71
|
+
raise MalformedMessage, "Malformed SNS envelope"
|
|
72
|
+
rescue Aws::SNS::MessageVerifier::VerificationError, OpenSSL::OpenSSLError
|
|
73
|
+
raise AuthenticationError, "SNS signature could not be verified"
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
private
|
|
77
|
+
|
|
78
|
+
def authorize!(envelope)
|
|
79
|
+
topic = envelope.fetch("TopicArn")
|
|
80
|
+
settings = @configuration.ses
|
|
81
|
+
parts = topic.split(":")
|
|
82
|
+
unless settings.topic_arns.is_a?(Array) && settings.topic_arns.include?(topic) && parts.size == 6 && parts[0] == "arn" && parts[2] == "sns" &&
|
|
83
|
+
parts[3] == settings.region && @configuration.scope == "ses:#{parts[4]}:#{parts[3]}:account"
|
|
84
|
+
raise AuthenticationError, "SNS topic is not authorized"
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
suffix = { "aws" => "amazonaws.com", "aws-us-gov" => "amazonaws.com", "aws-cn" => "amazonaws.com.cn" }[parts[1]]
|
|
88
|
+
uri = URI.parse(envelope.fetch("SigningCertURL"))
|
|
89
|
+
unless suffix && uri.scheme == "https" && uri.host == "sns.#{settings.region}.#{suffix}" &&
|
|
90
|
+
uri.port == 443 && !uri.userinfo && !uri.query && !uri.fragment &&
|
|
91
|
+
uri.path.match?(%r{\A/SimpleNotificationService-[A-Za-z0-9_-]+\.pem\z})
|
|
92
|
+
raise AuthenticationError, "SNS certificate URL is not authorized"
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bouncy
|
|
4
|
+
class PruneJob < ActiveJob::Base
|
|
5
|
+
queue_as :default
|
|
6
|
+
|
|
7
|
+
def perform
|
|
8
|
+
Bouncy.configuration.validate!
|
|
9
|
+
# Keep state rows, including inactive release fences, indefinitely.
|
|
10
|
+
Bouncy.events.where("created_at < ?", Bouncy.configuration.retention.ago).in_batches.delete_all
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
end
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bouncy
|
|
4
|
+
# Reconciles the provider's suppression list with local state under the scope lock.
|
|
5
|
+
#
|
|
6
|
+
# Idempotency contract: a snapshot that changes nothing about an address writes no
|
|
7
|
+
# event, fires no hook and does not bump the row's lock version. Only the observation
|
|
8
|
+
# time (provider_checked_at) is refreshed. Changed provider metadata advances the
|
|
9
|
+
# version to fence concurrent recovery, without a restriction event. Rows with neither
|
|
10
|
+
# provider evidence nor a webhook-derived block are never looked up at the provider.
|
|
11
|
+
class Reconciler
|
|
12
|
+
def initialize(adapter)
|
|
13
|
+
@adapter = adapter
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def call
|
|
17
|
+
ScopeLock.synchronize { reconcile }
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
private
|
|
21
|
+
|
|
22
|
+
def reconcile
|
|
23
|
+
started_at = Time.current
|
|
24
|
+
versions = Suppression.where(scope: Bouncy.scope).pluck(:email, :lock_version).to_h
|
|
25
|
+
snapshot = @adapter.snapshot
|
|
26
|
+
raise UnsafeSnapshot, "Provider returned a different scope" unless snapshot.scope == Bouncy.scope
|
|
27
|
+
|
|
28
|
+
counts = Hash.new(0)
|
|
29
|
+
grouped = snapshot.entries.group_by { |entry| Identity.normalize(entry.email) }
|
|
30
|
+
grouped.each { |email, entries| counts[apply_listed(email, entries, snapshot, versions, started_at)] += 1 }
|
|
31
|
+
if snapshot.complete && snapshot.policy_verified
|
|
32
|
+
Suppression.where(scope: Bouncy.scope).find_each do |row|
|
|
33
|
+
next if grouped.key?(row.email) || !unchanged?(row, versions, started_at)
|
|
34
|
+
|
|
35
|
+
counts[check_absence(row, snapshot)] += 1
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
Store.event!("sync", source: "sync", details: {
|
|
39
|
+
"complete" => snapshot.complete && snapshot.policy_verified, "enumerated" => snapshot.complete,
|
|
40
|
+
"policy_verified" => snapshot.policy_verified, "policy_reason" => snapshot.policy_reason,
|
|
41
|
+
"entries" => snapshot.entries.size, "added" => counts[:added], "updated" => counts[:updated],
|
|
42
|
+
"released" => counts[:released], "unchanged" => counts[:unchanged],
|
|
43
|
+
"started_at" => started_at.iso8601(6), "finished_at" => Time.current.iso8601(6)
|
|
44
|
+
})
|
|
45
|
+
rescue ProviderError => e
|
|
46
|
+
Store.event!("sync", source: "sync", details: { "complete" => false, "error_class" => e.class.name })
|
|
47
|
+
raise
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Applies the listed entries for one normalized address.
|
|
51
|
+
# Returns :added, :updated, :unchanged or :skipped.
|
|
52
|
+
def apply_listed(email, entries, snapshot, versions, started_at)
|
|
53
|
+
previous = Suppression.find_by(scope: Bouncy.scope, email: email)
|
|
54
|
+
retained = []
|
|
55
|
+
if snapshot.complete && previous
|
|
56
|
+
listed = entries.map(&:email)
|
|
57
|
+
retained = previous.provider_entries.reject { |entry| listed.include?(entry["email"]) }.reject do |entry|
|
|
58
|
+
@adapter.lookup(entry.fetch("email")) == :absent
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
Store.change(email) do |row|
|
|
62
|
+
next :skipped unless unchanged?(row, versions, started_at)
|
|
63
|
+
|
|
64
|
+
current = entries.reject { |entry| row.released_before && entry.updated_at <= row.released_before }
|
|
65
|
+
next :skipped if current.empty?
|
|
66
|
+
|
|
67
|
+
merged = (snapshot.complete ? retained + current.map(&:to_h) : row.provider_entries + current.map(&:to_h))
|
|
68
|
+
merged = merged.group_by { |entry| entry["email"] }.values.map do |variants|
|
|
69
|
+
variants.max_by { |entry| Time.iso8601(entry.fetch("provider_updated_at")) }
|
|
70
|
+
end
|
|
71
|
+
was_blocked = row.blocked?
|
|
72
|
+
before = restriction_of(row)
|
|
73
|
+
row.provider_entries = merged
|
|
74
|
+
row.provider_checked_at = snapshot.finished_at
|
|
75
|
+
if snapshot.policy_verified
|
|
76
|
+
row.provider_blocked_at ||= current.map(&:updated_at).min
|
|
77
|
+
row.provider_reason = merged.any? { |entry| entry["reason"] == "complaint" } ? "complaint" : "hard_bounce"
|
|
78
|
+
end
|
|
79
|
+
row.details = row.details.except("absence_count")
|
|
80
|
+
if before == restriction_of(row)
|
|
81
|
+
# New evidence must fence a concurrent recovery even when the restriction is unchanged.
|
|
82
|
+
if row.provider_entries_was.sort_by { |entry| entry["email"] } == merged.sort_by { |entry| entry["email"] }
|
|
83
|
+
row.update_columns(provider_checked_at: snapshot.finished_at)
|
|
84
|
+
else
|
|
85
|
+
row.save!
|
|
86
|
+
end
|
|
87
|
+
next :unchanged
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
row.summarize!
|
|
91
|
+
became_blocked = !was_blocked && row.blocked?
|
|
92
|
+
kind = before.first.empty? || became_blocked ? "sync_added" : "sync_updated"
|
|
93
|
+
Store.event!(kind, email: email, source: "sync", details: { "became_blocked" => became_blocked })
|
|
94
|
+
kind == "sync_added" ? :added : :updated
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def restriction_of(row)
|
|
99
|
+
[row.provider_entries.map { |entry| [entry["email"], entry["reason"]] }.sort,
|
|
100
|
+
row.provider_blocked_at.present?, row.provider_reason, row.details.key?("absence_count")]
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def unchanged?(row, versions, started_at)
|
|
104
|
+
versions.key?(row.email) ? row.lock_version == versions[row.email] : row.created_at >= started_at && row.lock_version.zero? && row.released_before.nil?
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# Handles a local row the complete, policy-verified snapshot did not list.
|
|
108
|
+
# Returns :released, :unchanged or :skipped.
|
|
109
|
+
def check_absence(row, snapshot)
|
|
110
|
+
return :skipped if row.provider_entries.empty? && row.event_blocked_at.nil?
|
|
111
|
+
|
|
112
|
+
version = row.lock_version
|
|
113
|
+
identifiers = row.provider_entries.map { |entry| entry.fetch("email") }
|
|
114
|
+
identifiers << row.details.fetch("exact_email", row.email) if identifiers.empty?
|
|
115
|
+
return :skipped unless identifiers.all? { |email| @adapter.lookup(email) == :absent }
|
|
116
|
+
|
|
117
|
+
Store.change(row.email) do |current|
|
|
118
|
+
next :skipped unless current.lock_version == version
|
|
119
|
+
|
|
120
|
+
was_blocked = current.blocked?
|
|
121
|
+
released = current.provider_entries.any?
|
|
122
|
+
counted = false
|
|
123
|
+
current.provider_entries = []
|
|
124
|
+
current.provider_blocked_at = current.provider_reason = nil
|
|
125
|
+
current.provider_checked_at = snapshot.finished_at
|
|
126
|
+
if current.event_blocked_at && current.event_blocked_at < 1.hour.ago
|
|
127
|
+
count = current.details.fetch("absence_count", 0) + 1
|
|
128
|
+
current.details = current.details.merge("absence_count" => count)
|
|
129
|
+
counted = true
|
|
130
|
+
if count >= 2
|
|
131
|
+
current.released_before = [current.released_before, current.event_blocked_at].compact.max
|
|
132
|
+
current.event_blocked_at = current.event_reason = nil
|
|
133
|
+
released = true
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
unless released || counted
|
|
137
|
+
current.update_columns(provider_checked_at: snapshot.finished_at)
|
|
138
|
+
next :unchanged
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
current.summarize!
|
|
142
|
+
next :unchanged unless released
|
|
143
|
+
|
|
144
|
+
Store.event!("sync_released", email: current.email, source: "sync",
|
|
145
|
+
details: { "became_unblocked" => was_blocked && !current.blocked? })
|
|
146
|
+
:released
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bouncy
|
|
4
|
+
class Record < ActiveRecord::Base
|
|
5
|
+
self.abstract_class = true
|
|
6
|
+
self.implicit_order_column = "created_at"
|
|
7
|
+
|
|
8
|
+
before_create :assign_string_primary_key
|
|
9
|
+
|
|
10
|
+
private
|
|
11
|
+
|
|
12
|
+
def assign_string_primary_key
|
|
13
|
+
self.id ||= SecureRandom.uuid if self.class.type_for_attribute(self.class.primary_key).type == :string
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bouncy
|
|
4
|
+
class Recovery
|
|
5
|
+
def initialize(adapter)
|
|
6
|
+
@adapter = adapter
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
def call(email, note:, actor:, at:)
|
|
10
|
+
raise ArgumentError, "at must be :local or :provider" unless %i[local provider].include?(at)
|
|
11
|
+
raise ArgumentError, "A recovery note is required" if at == :provider && note.to_s.strip.empty?
|
|
12
|
+
|
|
13
|
+
key = Identity.normalize(email)
|
|
14
|
+
row = Store.change(key) { |record| record }
|
|
15
|
+
version = row.lock_version
|
|
16
|
+
started_at = Time.current
|
|
17
|
+
if at == :provider
|
|
18
|
+
snapshot = @adapter.snapshot
|
|
19
|
+
unless snapshot.complete && snapshot.policy_verified && snapshot.scope == Bouncy.scope
|
|
20
|
+
raise UnsafeSnapshot, "Release requires a complete snapshot of the configured sending scope"
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
entries = snapshot.entries.select { |entry| Identity.normalize(entry.email) == key }
|
|
24
|
+
# Historical exact identifiers still need checking if a list page raced a write.
|
|
25
|
+
entries += row.provider_entries.map do |entry|
|
|
26
|
+
Providers::Entry.new(email: entry.fetch("email"), reason: entry.fetch("reason"), updated_at: Time.iso8601(entry.fetch("provider_updated_at")))
|
|
27
|
+
end
|
|
28
|
+
outcomes = @adapter.release(entries.uniq(&:email))
|
|
29
|
+
end
|
|
30
|
+
Store.change(key) do |current|
|
|
31
|
+
raise ReleaseConflict, "Address changed during recovery; review its latest state and retry" if current.lock_version != version
|
|
32
|
+
|
|
33
|
+
current.manual_blocked_at = current.manual_note = current.soft_blocked_until = nil
|
|
34
|
+
# Recovery clears the soft-bounce history too. Without this, one soft bounce after a
|
|
35
|
+
# release would meet the threshold again immediately and re-hold the address.
|
|
36
|
+
current.soft_bounce_count = 0
|
|
37
|
+
current.last_soft_bounce_at = nil
|
|
38
|
+
current.details = current.details.except("soft_bounces").merge("soft_released_before" => started_at.iso8601(6))
|
|
39
|
+
if at == :provider
|
|
40
|
+
current.provider_entries = []
|
|
41
|
+
current.provider_blocked_at = current.provider_reason = current.event_blocked_at = current.event_reason = nil
|
|
42
|
+
current.provider_checked_at = Time.current
|
|
43
|
+
current.released_before = started_at
|
|
44
|
+
end
|
|
45
|
+
current.summarize!
|
|
46
|
+
Store.event!("release", email: key, details: { "at" => at.to_s, "note" => note.to_s.truncate(1000),
|
|
47
|
+
"actor" => actor.to_s.truncate(200), "outcomes" => outcomes })
|
|
48
|
+
current
|
|
49
|
+
end
|
|
50
|
+
rescue ProviderError, ReleaseConflict => e
|
|
51
|
+
Store.event!("release_failed", email: key, details: { "error_class" => e.class.name,
|
|
52
|
+
"outcomes" => e.is_a?(ReleaseFailed) ? e.outcomes : [] })
|
|
53
|
+
raise
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "tmpdir"
|
|
4
|
+
|
|
5
|
+
module Bouncy
|
|
6
|
+
# One reconciliation per scope at a time. Every adapter fails fast: a second caller gets
|
|
7
|
+
# ProviderError immediately instead of queueing behind a run that may be waiting on AWS.
|
|
8
|
+
module ScopeLock
|
|
9
|
+
module_function
|
|
10
|
+
|
|
11
|
+
def synchronize
|
|
12
|
+
Record.connection_pool.with_connection do |connection|
|
|
13
|
+
key = Digest::SHA256.hexdigest(Bouncy.scope)[0, 15].to_i(16)
|
|
14
|
+
case connection.adapter_name
|
|
15
|
+
when "PostgreSQL"
|
|
16
|
+
acquired = connection.select_value("SELECT pg_try_advisory_lock(#{key})")
|
|
17
|
+
raise ProviderError, "Another sync holds this scope lock" unless [true, "t"].include?(acquired)
|
|
18
|
+
|
|
19
|
+
begin
|
|
20
|
+
yield
|
|
21
|
+
ensure
|
|
22
|
+
connection.execute("SELECT pg_advisory_unlock(#{key})")
|
|
23
|
+
end
|
|
24
|
+
when /Mysql|Trilogy/i
|
|
25
|
+
name = Digest::SHA256.hexdigest("bouncy:#{connection.pool.db_config.database}:#{Bouncy.scope}")
|
|
26
|
+
acquired = connection.select_value("SELECT GET_LOCK('#{name}', 0)")
|
|
27
|
+
raise ProviderError, "Another sync holds this scope lock" unless acquired == 1
|
|
28
|
+
|
|
29
|
+
begin
|
|
30
|
+
yield
|
|
31
|
+
ensure
|
|
32
|
+
connection.execute("SELECT RELEASE_LOCK('#{name}')")
|
|
33
|
+
end
|
|
34
|
+
when "SQLite"
|
|
35
|
+
database = connection.pool.db_config.database
|
|
36
|
+
path = database == ":memory:" ? File.join(Dir.tmpdir, "bouncy-#{Process.pid}-#{key}.lock") : "#{File.expand_path(database)}.bouncy-#{key}.lock"
|
|
37
|
+
File.open(path, File::RDWR | File::CREAT, 0o600) do |file|
|
|
38
|
+
raise ProviderError, "Another sync holds this scope lock" unless file.flock(File::LOCK_EX | File::LOCK_NB)
|
|
39
|
+
|
|
40
|
+
begin
|
|
41
|
+
yield
|
|
42
|
+
ensure
|
|
43
|
+
file.flock(File::LOCK_UN)
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
else
|
|
47
|
+
raise ConfigurationError, "Bouncy supports PostgreSQL, MySQL and SQLite"
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bouncy
|
|
4
|
+
class Status
|
|
5
|
+
attr_reader :scope, :provider, :knowledge
|
|
6
|
+
|
|
7
|
+
# The underlying Bouncy::Suppression row, or nil. Hosts use it to link an address to
|
|
8
|
+
# their own admin page for it; it is not needed for any policy decision.
|
|
9
|
+
attr_reader :record
|
|
10
|
+
|
|
11
|
+
# knowledge: :observed, :no_known_block, :unavailable (recognized database outage)
|
|
12
|
+
# or :unconfigured (config.scope is not set; Bouncy is inactive).
|
|
13
|
+
def initialize(row, knowledge: nil, unavailable: false)
|
|
14
|
+
@row = row
|
|
15
|
+
@record = row
|
|
16
|
+
@scope = Bouncy.configuration.scope
|
|
17
|
+
@provider = Bouncy.configuration.provider
|
|
18
|
+
@knowledge = knowledge
|
|
19
|
+
@knowledge ||= if unavailable
|
|
20
|
+
:unavailable
|
|
21
|
+
elsif row
|
|
22
|
+
:observed
|
|
23
|
+
else
|
|
24
|
+
:no_known_block
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def email = @row&.email
|
|
29
|
+
def blocked? = @row ? @row.blocked? : false
|
|
30
|
+
def reasons = @row ? @row.reasons : []
|
|
31
|
+
def reason = (%i[complaint hard_bounce provider_list manual soft_bounces] & reasons).first
|
|
32
|
+
def observed_at = @row&.provider_checked_at || @row&.last_event_at
|
|
33
|
+
def provider_updated_at = @row&.provider_entries&.filter_map { |entry| Time.iso8601(entry.fetch("provider_updated_at")) }&.max
|
|
34
|
+
def last_event = @row && Bouncy.events.for(@row.email).recent.first
|
|
35
|
+
def release_supported? = %i[unavailable unconfigured].exclude?(knowledge) && provider.to_sym == :ses
|
|
36
|
+
|
|
37
|
+
# The provider lists this address in the configured scope.
|
|
38
|
+
def provider_listed? = @row ? @row.provider_entries.any? : false
|
|
39
|
+
|
|
40
|
+
# Current scoped verification is separate from retained restriction evidence.
|
|
41
|
+
def policy_unverified? = provider_listed? && policy_details["policy_verified"] != true
|
|
42
|
+
def policy_reason = policy_unverified? ? (policy_details["policy_reason"] || "Sending policy could not be verified") : nil
|
|
43
|
+
|
|
44
|
+
def stale?
|
|
45
|
+
%i[unavailable unconfigured].include?(knowledge) || observed_at.nil? || observed_at < Bouncy.configuration.stale_after.ago
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
def policy_details
|
|
51
|
+
@policy_details ||= Event.where(scope: scope, kind: "sync").order(created_at: :desc, id: :desc).pick(:details) ||
|
|
52
|
+
{ "policy_reason" => "Sending policy has not been checked" }
|
|
53
|
+
rescue ActiveRecord::ActiveRecordError => e
|
|
54
|
+
raise unless Bouncy.database_unavailable?(e)
|
|
55
|
+
|
|
56
|
+
@policy_details = { "policy_reason" => "Sending policy is unavailable because the database could not be reached" }
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bouncy
|
|
4
|
+
# Statuses for many addresses, loaded with one query by Bouncy.statuses. Look an address
|
|
5
|
+
# up by any spelling; an address that was not asked for, or that is not a valid address,
|
|
6
|
+
# answers with a plain no-known-block status. When Bouncy is unconfigured or the database
|
|
7
|
+
# is unavailable every lookup answers with that knowledge instead.
|
|
8
|
+
class StatusSet
|
|
9
|
+
include Enumerable
|
|
10
|
+
|
|
11
|
+
def initialize(statuses, knowledge: nil)
|
|
12
|
+
@statuses = statuses
|
|
13
|
+
@knowledge = knowledge
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def [](email)
|
|
17
|
+
key = begin
|
|
18
|
+
Identity.normalize(email)
|
|
19
|
+
rescue InvalidAddress
|
|
20
|
+
nil
|
|
21
|
+
end
|
|
22
|
+
(key && @statuses[key]) || Status.new(nil, knowledge: @knowledge)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Yields [normalized_email, status] pairs for the addresses that were asked for.
|
|
26
|
+
def each(&) = @statuses.each(&)
|
|
27
|
+
def size = @statuses.size
|
|
28
|
+
def blocked = @statuses.select { |_email, status| status.blocked? }
|
|
29
|
+
def to_h = @statuses.dup
|
|
30
|
+
end
|
|
31
|
+
end
|
data/lib/bouncy/store.rb
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bouncy
|
|
4
|
+
module Store
|
|
5
|
+
module_function
|
|
6
|
+
|
|
7
|
+
# Retry outside the transaction: PostgreSQL aborts a transaction on a unique violation.
|
|
8
|
+
def change(email)
|
|
9
|
+
key = Identity.normalize(email)
|
|
10
|
+
attempts = 0
|
|
11
|
+
begin
|
|
12
|
+
Suppression.transaction(requires_new: true) do
|
|
13
|
+
row = Suppression.find_or_create_by!(scope: Bouncy.scope, email: key) do |record|
|
|
14
|
+
record.provider = Bouncy.configuration.provider.to_s
|
|
15
|
+
end
|
|
16
|
+
row.with_lock { yield row }
|
|
17
|
+
end
|
|
18
|
+
rescue ActiveRecord::RecordNotUnique, ActiveRecord::StaleObjectError, ActiveRecord::Deadlocked
|
|
19
|
+
attempts += 1
|
|
20
|
+
retry if attempts < 3
|
|
21
|
+
raise
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def event!(kind, email: nil, source: "admin", occurred_at: Time.current, **attributes)
|
|
26
|
+
Event.create!({ scope: Bouncy.scope, email: email, kind: kind, source: source,
|
|
27
|
+
received_at: Time.current, occurred_at: occurred_at }.merge(attributes))
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bouncy
|
|
4
|
+
class Suppression < Record
|
|
5
|
+
self.table_name = "bouncy_suppressions"
|
|
6
|
+
|
|
7
|
+
attribute :provider_entries, default: -> { [] }
|
|
8
|
+
attribute :details, default: -> { {} }
|
|
9
|
+
|
|
10
|
+
scope :blocked, lambda {
|
|
11
|
+
where("manual_blocked_at IS NOT NULL OR provider_blocked_at IS NOT NULL OR event_blocked_at IS NOT NULL OR soft_blocked_until > ?", Time.current)
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
def reasons
|
|
15
|
+
values = []
|
|
16
|
+
values << :manual if manual_blocked_at
|
|
17
|
+
values << provider_reason.to_sym if provider_blocked_at && provider_reason
|
|
18
|
+
values << event_reason.to_sym if event_blocked_at && event_reason
|
|
19
|
+
values << :soft_bounces if soft_blocked_until && soft_blocked_until > Time.current
|
|
20
|
+
values.uniq
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def blocked?
|
|
24
|
+
reasons.any?
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def summarize!
|
|
28
|
+
self.reason = (%i[complaint hard_bounce provider_list manual soft_bounces] & reasons).first
|
|
29
|
+
self.blocked_at = blocked? ? (blocked_at || Time.current) : nil
|
|
30
|
+
save!
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|