cloudflare-email 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +76 -0
- data/README.md +34 -9
- data/app/controllers/cloudflare/email/ingress_controller.rb +16 -60
- data/app/controllers/cloudflare/email/management/mailboxes_controller.rb +173 -0
- data/app/controllers/cloudflare/email/management/styles_controller.rb +13 -0
- data/app/views/cloudflare/email/management/mailboxes/index.html.erb +65 -0
- data/app/views/cloudflare/email/management/mailboxes/message.html.erb +34 -0
- data/app/views/cloudflare/email/management/mailboxes/show.html.erb +86 -0
- data/app/views/layouts/cloudflare/email/management.html.erb +29 -0
- data/docs/activerecord-tenanted.md +46 -0
- data/docs/architecture.md +13 -3
- data/docs/custom-ingress.md +207 -0
- data/docs/delivery-events.md +9 -0
- data/docs/features.md +36 -5
- data/docs/getting-started.md +18 -8
- data/docs/mailboxes.md +67 -3
- data/docs/management-engine.md +190 -0
- data/docs/routing-deliveries.md +94 -0
- data/docs/routing-diagnostics.md +68 -0
- data/docs/troubleshooting.md +4 -3
- data/docs/upgrading-0.2.md +5 -0
- data/docs/upgrading-0.3.md +55 -0
- data/docs/verification/2026-09-11-custom-ingress.md +49 -0
- data/docs/verification/2026-09-12-destination-context.md +46 -0
- data/docs/verification/2026-09-13-durable-inbound-and-routing.md +71 -0
- data/lib/cloudflare/email/active_record/base.rb +13 -1
- data/lib/cloudflare/email/active_record/routing_deliveries.rb +128 -0
- data/lib/cloudflare/email/check_route_task.rb +21 -0
- data/lib/cloudflare/email/client.rb +45 -10
- data/lib/cloudflare/email/delivery_method.rb +2 -0
- data/lib/cloudflare/email/deploy_worker_task.rb +3 -3
- data/lib/cloudflare/email/engine.rb +5 -0
- data/lib/cloudflare/email/envelope.rb +2 -2
- data/lib/cloudflare/email/event_consumer.rb +12 -3
- data/lib/cloudflare/email/ingress.rb +115 -0
- data/lib/cloudflare/email/mailboxes/models.rb +19 -2
- data/lib/cloudflare/email/mailboxes/service.rb +99 -15
- data/lib/cloudflare/email/management/adapter.rb +31 -0
- data/lib/cloudflare/email/management/configuration.rb +17 -0
- data/lib/cloudflare/email/management/engine.rb +15 -0
- data/lib/cloudflare/email/management/management.css +68 -0
- data/lib/cloudflare/email/management/routes.rb +14 -0
- data/lib/cloudflare/email/management.rb +7 -0
- data/lib/cloudflare/email/provider_metadata.rb +77 -0
- data/lib/cloudflare/email/routing_analytics.rb +196 -0
- data/lib/cloudflare/email/routing_diagnostics.rb +206 -0
- data/lib/cloudflare/email/verification.rb +17 -9
- data/lib/cloudflare/email/version.rb +1 -1
- data/lib/cloudflare/email/worker_deployer.rb +36 -5
- data/lib/generators/cloudflare/email/install_generator.rb +6 -5
- data/lib/generators/cloudflare/email/mailboxes/catch_all/catch_all_generator.rb +21 -0
- data/lib/generators/cloudflare/email/mailboxes/catch_all/templates/add_cloudflare_email_catch_all.rb +18 -0
- data/lib/generators/cloudflare/email/mailboxes/templates/create_cloudflare_email_mailboxes.rb +4 -0
- data/lib/generators/cloudflare/email/routing_tracking/routing_tracking_generator.rb +28 -0
- data/lib/generators/cloudflare/email/routing_tracking/templates/create_cloudflare_email_routing_delivery_receipts.rb +22 -0
- data/lib/tasks/cloudflare_email.rake +2 -1
- data/lib/tasks/cloudflare_email_diagnostics.rake +11 -0
- data/templates/deploy-to-cloudflare/.dev.vars.example +3 -0
- data/templates/deploy-to-cloudflare/.gitignore +6 -0
- data/templates/deploy-to-cloudflare/LICENSE.txt +21 -0
- data/templates/deploy-to-cloudflare/README.md +84 -0
- data/templates/deploy-to-cloudflare/package-lock.json +2627 -0
- data/templates/deploy-to-cloudflare/package.json +36 -0
- data/templates/deploy-to-cloudflare/src/index.js +445 -0
- data/templates/deploy-to-cloudflare/test/durable.test.ts +193 -0
- data/templates/deploy-to-cloudflare/test/index.test.ts +309 -0
- data/templates/deploy-to-cloudflare/test/relay.test.ts +226 -0
- data/templates/deploy-to-cloudflare/vitest.config.ts +8 -0
- data/templates/deploy-to-cloudflare/wrangler.toml +22 -0
- data/templates/worker/README.md +180 -15
- data/templates/worker/docs/durable-inbound.md +149 -0
- data/templates/worker/package-lock.json +2 -2
- data/templates/worker/package.json +1 -1
- data/templates/worker/src/index.js +331 -38
- data/templates/worker/test/durable.test.ts +193 -0
- data/templates/worker/test/index.test.ts +78 -20
- data/templates/worker/test/relay.test.ts +226 -0
- data/templates/worker/wrangler.toml +50 -0
- metadata +49 -3
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# Durable inbound and Routing delivery verification
|
|
2
|
+
|
|
3
|
+
This change brings the verified Routing-delivery correlation from Rebulk into an
|
|
4
|
+
optional gem API and adds inbound storage/recovery for Rails outages. All tests
|
|
5
|
+
below use synthetic messages and disposable local storage. This report does not
|
|
6
|
+
claim the new Worker configuration is deployed in Rebulk production.
|
|
7
|
+
|
|
8
|
+
## Boundaries verified
|
|
9
|
+
|
|
10
|
+
- Shared sending/queue HTTP requests stream bounded responses, including gzip
|
|
11
|
+
expansion, and enforce a total network deadline. Real socket tests cover
|
|
12
|
+
slow streams, socket cleanup, oversized replies and uncertain queue ACKs.
|
|
13
|
+
No ambiguous failure automatically resends an accepted email.
|
|
14
|
+
- Authenticated Routing analytics validates account/zone, provider ID, sender,
|
|
15
|
+
request window and a single saved envelope recipient. Dedicated encrypted
|
|
16
|
+
receipts preserve provider evidence and support idempotent projection.
|
|
17
|
+
- Inbound R2 persistence finishes before successful Email Worker return. Queue
|
|
18
|
+
messages contain only a storage pointer. Failed enqueue, Rails errors, lost
|
|
19
|
+
responses, failed deletion and exhausted retries retain the pending payload.
|
|
20
|
+
Scheduled pagination advances over poison messages and retries retained mail.
|
|
21
|
+
- A real local workerd instance uses R2 and Queues against Rails with SQLite and
|
|
22
|
+
disk-backed Active Storage. With queue retries disabled, Rails 503 leaves mail
|
|
23
|
+
in R2; invoking the scheduled handler after recovery persists one message and
|
|
24
|
+
removes the pending object. Returning 503 after Rails commits also replays to
|
|
25
|
+
one record. MIME and binary attachments remain byte-identical.
|
|
26
|
+
- Missing Message-ID fallback remains stable when Rails hostnames change. Valid
|
|
27
|
+
Message-IDs and raw MIME are preserved. Pre-upgrade fallback IDs require care
|
|
28
|
+
when replaying historical archive records.
|
|
29
|
+
- Local workerd capacity checks pass for one 25 MiB message and two concurrent
|
|
30
|
+
25 MiB receipts, followed by concurrent handoffs. The receiver verifies exact
|
|
31
|
+
SHA256 and HMAC values, and R2 pending objects clear after acknowledgement.
|
|
32
|
+
Local workerd does not certify production memory-limit enforcement.
|
|
33
|
+
|
|
34
|
+
## Reproduce
|
|
35
|
+
|
|
36
|
+
Use Ruby 3.4 and Node 22 or another supported version:
|
|
37
|
+
|
|
38
|
+
```sh
|
|
39
|
+
BUNDLE_GEMFILE=gemfiles/local_ingress.gemfile bundle install
|
|
40
|
+
npm ci --prefix templates/worker
|
|
41
|
+
BUNDLE_GEMFILE=gemfiles/local_ingress.gemfile bundle exec rake test
|
|
42
|
+
npm test --prefix templates/worker
|
|
43
|
+
npm run check --prefix templates/worker
|
|
44
|
+
node script/verify_durable_capacity.mjs
|
|
45
|
+
BUNDLE_GEMFILE=gemfiles/local_ingress.gemfile bundle exec ruby script/verify_local_ingress.rb
|
|
46
|
+
BUNDLE_GEMFILE=gemfiles/local_ingress.gemfile bundle exec ruby script/verify_package.rb
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
The local runtime harness invokes the real scheduled handler through a temporary
|
|
50
|
+
test-only endpoint. It does not test Cloudflare's production cron scheduler or
|
|
51
|
+
SMTP delivery behavior. No test endpoint is shipped in the Worker.
|
|
52
|
+
|
|
53
|
+
## Before production enablement
|
|
54
|
+
|
|
55
|
+
Provision a private R2 bucket, queue and schedule per environment, and keep
|
|
56
|
+
pending objects free of lifecycle expiration. Deploy the updated Rails ingress
|
|
57
|
+
first. Verify maximum message size and recovery throughput with expected traffic,
|
|
58
|
+
then perform a staging outage drill through the actual Cloudflare email route.
|
|
59
|
+
Monitor storage errors and pending object age as well as queue backlog. Keep
|
|
60
|
+
Rails deduplication records throughout the recovery horizon and monitor mailbox
|
|
61
|
+
routing jobs after durable ingestion.
|
|
62
|
+
|
|
63
|
+
Routing tracking is separately opt-in and needs encryption keys, its migration,
|
|
64
|
+
a restricted analytics read credential, and host scheduling. Cross-database
|
|
65
|
+
applications must retain their durable business projection boundary and enforce
|
|
66
|
+
provider-ID uniqueness across tenant databases. Missing delivery evidence stays
|
|
67
|
+
unresolved; it never authorizes another send.
|
|
68
|
+
|
|
69
|
+
R2 storage and retry protect mail after a successful storage commit. They cannot
|
|
70
|
+
promise absolute losslessness during upstream storage failure, deletion,
|
|
71
|
+
misconfiguration or account suspension. See the [operating guide](../../templates/worker/docs/durable-inbound.md).
|
|
@@ -4,13 +4,25 @@ require "cloudflare/email/tenancy"
|
|
|
4
4
|
module Cloudflare
|
|
5
5
|
module Email
|
|
6
6
|
module ActiveRecord
|
|
7
|
+
# Rails may inspect every model's pool while preloading schema metadata
|
|
8
|
+
# before any request has selected a tenant. Report an unavailable
|
|
9
|
+
# connection using Rails' error hierarchy so boot can recover, while
|
|
10
|
+
# retaining the same fail-closed guard for every caller.
|
|
11
|
+
class TenantConnectionUnavailable < ::ActiveRecord::ConnectionNotEstablished; end
|
|
12
|
+
|
|
7
13
|
# Uses the host's abstract tenant connection owner when explicitly configured.
|
|
8
14
|
class Base < Tenancy.model_base(::ActiveRecord::Base)
|
|
9
15
|
self.abstract_class = true
|
|
10
16
|
|
|
11
17
|
class << self
|
|
12
18
|
def connection_pool
|
|
13
|
-
|
|
19
|
+
if Tenancy.enabled?
|
|
20
|
+
begin
|
|
21
|
+
Tenancy.require_context!
|
|
22
|
+
rescue ConfigurationError => error
|
|
23
|
+
raise TenantConnectionUnavailable, error.message
|
|
24
|
+
end
|
|
25
|
+
end
|
|
14
26
|
super
|
|
15
27
|
end
|
|
16
28
|
end
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "cloudflare/email/routing_analytics"
|
|
4
|
+
require "cloudflare/email/active_record/outbox"
|
|
5
|
+
|
|
6
|
+
module Cloudflare
|
|
7
|
+
module Email
|
|
8
|
+
module ActiveRecord
|
|
9
|
+
class RoutingDeliveryReceipt < Base
|
|
10
|
+
self.table_name = "cloudflare_email_routing_delivery_receipts"
|
|
11
|
+
belongs_to :outbound_delivery, class_name: "Cloudflare::Email::ActiveRecord::OutboundDelivery"
|
|
12
|
+
encrypts :payload_json
|
|
13
|
+
attr_readonly :account_id, :zone_id, :event_key, :outbound_delivery_id, :payload_json, :received_at
|
|
14
|
+
|
|
15
|
+
def evidence
|
|
16
|
+
RoutingAnalytics::Evidence.new(JSON.parse(payload_json))
|
|
17
|
+
rescue JSON::ParserError, TypeError
|
|
18
|
+
raise RoutingAnalytics::Error, "Stored Routing analytics evidence is malformed", cause: nil
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# An opt-in delivery projection, separate from Sending webhook receipts and
|
|
23
|
+
# acceptance reconciliation. No method here can send or authorize a resend.
|
|
24
|
+
class RoutingDeliveries
|
|
25
|
+
class InvalidEvidence < RoutingAnalytics::Error; end
|
|
26
|
+
ReplayResult = Struct.new(:after_id, :processed, :errors, :finished, keyword_init: true)
|
|
27
|
+
|
|
28
|
+
class << self
|
|
29
|
+
def record(delivery:, evidence:)
|
|
30
|
+
unless evidence.is_a?(RoutingAnalytics::Evidence) && delivery.is_a?(OutboundDelivery) && delivery.persisted?
|
|
31
|
+
raise ArgumentError, "A persisted outbound delivery and RoutingAnalytics::Evidence are required"
|
|
32
|
+
end
|
|
33
|
+
if RoutingDeliveryReceipt.connection.transaction_open?
|
|
34
|
+
raise ArgumentError, "record must commit outside an existing transaction before projection"
|
|
35
|
+
end
|
|
36
|
+
validate!(delivery.reload, evidence)
|
|
37
|
+
receipt = RoutingDeliveryReceipt.create_or_find_by!(account_id: evidence.account_id, event_key: evidence.identity) do |row|
|
|
38
|
+
row.outbound_delivery = delivery
|
|
39
|
+
row.zone_id = evidence.zone_id
|
|
40
|
+
row.payload_json = JSON.generate(evidence.payload)
|
|
41
|
+
row.received_at = Time.now.utc
|
|
42
|
+
end
|
|
43
|
+
unless receipt.outbound_delivery_id == delivery.id && receipt.evidence.same_event?(evidence)
|
|
44
|
+
raise InvalidEvidence, "Routing analytics evidence identity conflict"
|
|
45
|
+
end
|
|
46
|
+
receipt
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Returns :applied (including idempotent replay). An existing terminal
|
|
50
|
+
# fact is preserved; inspect recipient.state instead of assuming delivery.
|
|
51
|
+
# Only new recipient changes invoke the same-database callback.
|
|
52
|
+
def apply(receipt)
|
|
53
|
+
unless receipt.is_a?(RoutingDeliveryReceipt) && receipt.persisted?
|
|
54
|
+
raise ArgumentError, "A persisted RoutingDeliveryReceipt is required"
|
|
55
|
+
end
|
|
56
|
+
completed = false
|
|
57
|
+
receipt.with_lock(requires_new: true) do
|
|
58
|
+
if receipt.state == "applied"
|
|
59
|
+
completed = true
|
|
60
|
+
next
|
|
61
|
+
end
|
|
62
|
+
evidence = receipt.evidence
|
|
63
|
+
unless receipt.account_id == evidence.account_id && receipt.zone_id == evidence.zone_id && receipt.event_key == evidence.identity
|
|
64
|
+
raise InvalidEvidence, "Routing analytics receipt identity mismatch"
|
|
65
|
+
end
|
|
66
|
+
delivery = receipt.outbound_delivery
|
|
67
|
+
delivery.with_lock do
|
|
68
|
+
recipient = validate!(delivery, evidence)
|
|
69
|
+
unless recipient.terminal?
|
|
70
|
+
recipient.update!(state: "delivered", occurred_at: evidence.occurred_at, terminal: true)
|
|
71
|
+
yield delivery, recipient if block_given?
|
|
72
|
+
end
|
|
73
|
+
receipt.update!(state: "applied", applied_at: Time.now.utc)
|
|
74
|
+
completed = true
|
|
75
|
+
end
|
|
76
|
+
# ActiveRecord::Rollback can be swallowed by the nested lock's
|
|
77
|
+
# transaction. Fail before this outer transaction can commit.
|
|
78
|
+
raise ArgumentError, "Projection rolled back instead of completing" unless completed
|
|
79
|
+
end
|
|
80
|
+
raise ArgumentError, "Projection rolled back instead of completing" unless completed
|
|
81
|
+
:applied
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Each failed receipt advances the cursor too. Persist after_id in the
|
|
85
|
+
# host job and reset it to zero after finished; discovery gets its own budget.
|
|
86
|
+
def replay(account_id:, limit: 100, after_id: 0, &on_change)
|
|
87
|
+
unless account_id.is_a?(String) && !account_id.empty? && limit.is_a?(Integer) && (1..1000).cover?(limit) &&
|
|
88
|
+
after_id.is_a?(Integer) && after_id >= 0
|
|
89
|
+
raise ArgumentError, "An account, limit of 1..1000 and nonnegative cursor are required"
|
|
90
|
+
end
|
|
91
|
+
scope = RoutingDeliveryReceipt.where(account_id: account_id, state: "pending").where("id > ?", after_id).order(:id)
|
|
92
|
+
rows = scope.limit(limit).to_a
|
|
93
|
+
errors = []
|
|
94
|
+
rows.each do |receipt|
|
|
95
|
+
apply(receipt, &on_change)
|
|
96
|
+
rescue StandardError => error
|
|
97
|
+
errors << {receipt_id: receipt.id, error: error}
|
|
98
|
+
end
|
|
99
|
+
cursor = rows.last&.id || after_id
|
|
100
|
+
ReplayResult.new(after_id: cursor, processed: rows.length, errors: errors,
|
|
101
|
+
finished: !RoutingDeliveryReceipt.where(account_id: account_id, state: "pending").where("id > ?", cursor).exists?)
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
private
|
|
105
|
+
|
|
106
|
+
def validate!(delivery, evidence)
|
|
107
|
+
ids = OutboundDelivery.where(account_id: evidence.account_id,
|
|
108
|
+
provider_message_id: [evidence.message_id, "<#{evidence.message_id}>"]).limit(2).pluck(:id)
|
|
109
|
+
unless delivery.state == "accepted" && delivery.account_id == evidence.account_id && ids == [delivery.id] &&
|
|
110
|
+
MessageId.normalize(delivery.provider_message_id) == evidence.message_id &&
|
|
111
|
+
delivery.from_address.casecmp?(evidence.event.fetch("from")) && delivery.request_started_at &&
|
|
112
|
+
evidence.occurred_at >= Time.at(delivery.request_started_at.to_i).utc
|
|
113
|
+
raise InvalidEvidence, "Routing analytics requires one matching accepted outbound operation"
|
|
114
|
+
end
|
|
115
|
+
recipients = delivery.outbound_recipients.to_a
|
|
116
|
+
envelope = delivery.recipients
|
|
117
|
+
unless envelope.is_a?(Array) && envelope.one? && recipients.one? &&
|
|
118
|
+
recipients.first.recipient == Outbox.normalize_recipient(envelope.first) &&
|
|
119
|
+
recipients.first.acceptance_state == "accepted"
|
|
120
|
+
raise InvalidEvidence, "Routing analytics requires one accepted saved envelope recipient"
|
|
121
|
+
end
|
|
122
|
+
recipients.first
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
require "cloudflare/email/credentials"
|
|
2
|
+
require "cloudflare/email/routing_diagnostics"
|
|
3
|
+
require "cloudflare/email/worker_deployer"
|
|
4
|
+
|
|
5
|
+
module Cloudflare
|
|
6
|
+
module Email
|
|
7
|
+
class CheckRouteTask
|
|
8
|
+
def self.call(address:, worker_name: nil, account_id: nil, io: $stdout)
|
|
9
|
+
report = RoutingDiagnostics.new(api_token: Credentials.management_token).check(
|
|
10
|
+
address: address, worker_name: worker_name || WorkerDeployer.default_script_name,
|
|
11
|
+
account_id: account_id || Credentials.account_id)
|
|
12
|
+
report[:checks].each { |check| io.puts "#{check[:status].upcase} #{check[:name]}: #{check[:message]}" }
|
|
13
|
+
io.puts report[:limitations]
|
|
14
|
+
report[:status] == "pass" ? 0 : 1
|
|
15
|
+
rescue StandardError
|
|
16
|
+
io.puts "UNKNOWN: Unable to inspect routing. Check ADDRESS, WORKER_NAME, and Cloudflare management credentials."
|
|
17
|
+
1
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -2,6 +2,8 @@ require "net/http"
|
|
|
2
2
|
require "json"
|
|
3
3
|
require "uri"
|
|
4
4
|
require "time"
|
|
5
|
+
require "timeout"
|
|
6
|
+
require "zlib"
|
|
5
7
|
require "cloudflare/email/endpoint"
|
|
6
8
|
|
|
7
9
|
module Cloudflare
|
|
@@ -11,33 +13,47 @@ module Cloudflare
|
|
|
11
13
|
DEFAULT_RETRIES = 3
|
|
12
14
|
DEFAULT_TIMEOUT = 30
|
|
13
15
|
DEFAULT_BACKOFF = 0.5
|
|
16
|
+
DEFAULT_MAX_RESPONSE_BYTES = 1_048_576
|
|
14
17
|
MAX_RETRY_AFTER = 60 # seconds; never sleep longer than this even if server says so
|
|
15
18
|
|
|
16
19
|
RETRYABLE_NETWORK = [
|
|
17
20
|
Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout, Errno::ECONNRESET,
|
|
18
21
|
Errno::ECONNREFUSED, Errno::EHOSTUNREACH, EOFError, SocketError,
|
|
19
|
-
IOError
|
|
22
|
+
IOError, Timeout::Error, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
|
|
23
|
+
OpenSSL::SSL::SSLError, Zlib::Error
|
|
20
24
|
].freeze
|
|
21
25
|
PRE_SEND_NETWORK = [Net::OpenTimeout, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, SocketError].freeze
|
|
22
26
|
|
|
23
|
-
attr_reader :account_id, :base_url, :retries, :timeout, :retry_ambiguous
|
|
27
|
+
attr_reader :account_id, :base_url, :retries, :timeout, :retry_ambiguous,
|
|
28
|
+
:total_timeout, :max_response_bytes
|
|
24
29
|
|
|
25
30
|
def initialize(account_id:, api_token:, base_url: DEFAULT_BASE_URL,
|
|
26
31
|
retries: DEFAULT_RETRIES, timeout: DEFAULT_TIMEOUT,
|
|
27
32
|
initial_backoff: DEFAULT_BACKOFF, max_retry_after: MAX_RETRY_AFTER,
|
|
28
|
-
retry_ambiguous: false, logger: nil
|
|
33
|
+
retry_ambiguous: false, logger: nil,
|
|
34
|
+
total_timeout: timeout, max_response_bytes: DEFAULT_MAX_RESPONSE_BYTES)
|
|
29
35
|
unless account_id.is_a?(String) && account_id.match?(/\A[a-zA-Z0-9_-]+\z/)
|
|
30
36
|
raise ConfigurationError, "account_id is required and must be a single account identifier"
|
|
31
37
|
end
|
|
32
38
|
unless api_token.is_a?(String) && !api_token.empty? && !api_token.match?(/\s/)
|
|
33
39
|
raise ConfigurationError, "api_token is required and must not contain whitespace"
|
|
34
40
|
end
|
|
41
|
+
[[:timeout, timeout], [:total_timeout, total_timeout]].each do |name, value|
|
|
42
|
+
unless value.is_a?(Numeric) && value.real? && value.finite? && value.positive?
|
|
43
|
+
raise ConfigurationError, "#{name} must be a finite positive number"
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
unless max_response_bytes.is_a?(Integer) && max_response_bytes.positive?
|
|
47
|
+
raise ConfigurationError, "max_response_bytes must be a positive integer"
|
|
48
|
+
end
|
|
35
49
|
|
|
36
50
|
@account_id = account_id
|
|
37
51
|
@api_token = api_token
|
|
38
52
|
@base_url = Endpoint.parse(base_url).to_s.delete_suffix("/")
|
|
39
53
|
@retries = retries
|
|
40
54
|
@timeout = timeout
|
|
55
|
+
@total_timeout = total_timeout
|
|
56
|
+
@max_response_bytes = max_response_bytes
|
|
41
57
|
@initial_backoff = initial_backoff
|
|
42
58
|
@max_retry_after = max_retry_after
|
|
43
59
|
@logger = logger
|
|
@@ -143,9 +159,9 @@ module Cloudflare
|
|
|
143
159
|
do_request(method, uri, body)
|
|
144
160
|
rescue *RETRYABLE_NETWORK => e
|
|
145
161
|
unless PRE_SEND_NETWORK.any? { |type| e.is_a?(type) } || @retry_ambiguous
|
|
146
|
-
raise NetworkError.new("
|
|
162
|
+
raise NetworkError.new("provider response could not be read; delivery outcome is unknown; automatic retry disabled"), cause: nil
|
|
147
163
|
end
|
|
148
|
-
raise NetworkError.new(
|
|
164
|
+
raise NetworkError.new("provider request could not be completed; request outcome is unknown"), cause: nil if attempts > @retries
|
|
149
165
|
log_retry(attempts, e)
|
|
150
166
|
sleep(backoff); backoff *= 2
|
|
151
167
|
retry
|
|
@@ -194,13 +210,32 @@ module Cloudflare
|
|
|
194
210
|
req["User-Agent"] = "cloudflare-email-ruby/#{Cloudflare::Email::VERSION}"
|
|
195
211
|
req.body = JSON.generate(body) if body
|
|
196
212
|
|
|
197
|
-
|
|
198
|
-
|
|
213
|
+
raw = +"".b
|
|
214
|
+
response = nil
|
|
215
|
+
byte_limit = response_byte_limit(uri)
|
|
216
|
+
# Bound each network attempt, including trickling headers/body. Parsing,
|
|
217
|
+
# queue handlers, and database transactions are outside this deadline.
|
|
218
|
+
Timeout.timeout(@total_timeout) do
|
|
219
|
+
http.request(req) do |incoming|
|
|
220
|
+
response = incoming
|
|
221
|
+
incoming.read_body do |chunk|
|
|
222
|
+
# Net::HTTP yields decompressed bytes, so compressed expansion is
|
|
223
|
+
# subject to the same budget. Check before extending the buffer.
|
|
224
|
+
raise IOError, "provider response exceeds byte limit" if raw.bytesize + chunk.bytesize > byte_limit
|
|
225
|
+
raw << chunk
|
|
226
|
+
end
|
|
227
|
+
end
|
|
228
|
+
end
|
|
229
|
+
handle_response(response, raw)
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
def response_byte_limit(_uri)
|
|
233
|
+
@max_response_bytes
|
|
199
234
|
end
|
|
200
235
|
|
|
201
|
-
def handle_response(response)
|
|
236
|
+
def handle_response(response, raw = response.body)
|
|
202
237
|
status = response.code.to_i
|
|
203
|
-
body = parse_body(
|
|
238
|
+
body = parse_body(raw)
|
|
204
239
|
retry_after = response["Retry-After"]
|
|
205
240
|
|
|
206
241
|
# Stash Retry-After on the error response so retry logic can use it.
|
|
@@ -236,7 +271,7 @@ module Cloudflare
|
|
|
236
271
|
return {} if raw.nil? || raw.empty?
|
|
237
272
|
JSON.parse(raw)
|
|
238
273
|
rescue JSON::ParserError
|
|
239
|
-
{ "errors" => [{ "message" =>
|
|
274
|
+
{ "errors" => [{ "message" => "invalid JSON API response; request outcome is unknown" }] }
|
|
240
275
|
end
|
|
241
276
|
|
|
242
277
|
def extract_message(body)
|
|
@@ -25,6 +25,8 @@ module Cloudflare
|
|
|
25
25
|
base_url: settings[:base_url] || Cloudflare::Email::Client::DEFAULT_BASE_URL,
|
|
26
26
|
retries: settings.fetch(:retries, Cloudflare::Email::Client::DEFAULT_RETRIES),
|
|
27
27
|
timeout: settings.fetch(:timeout, Cloudflare::Email::Client::DEFAULT_TIMEOUT),
|
|
28
|
+
total_timeout: settings.fetch(:total_timeout, settings.fetch(:timeout, Cloudflare::Email::Client::DEFAULT_TIMEOUT)),
|
|
29
|
+
max_response_bytes: settings.fetch(:max_response_bytes, Cloudflare::Email::Client::DEFAULT_MAX_RESPONSE_BYTES),
|
|
28
30
|
initial_backoff: settings.fetch(:initial_backoff, Cloudflare::Email::Client::DEFAULT_BACKOFF),
|
|
29
31
|
max_retry_after: settings.fetch(:max_retry_after, Cloudflare::Email::Client::MAX_RETRY_AFTER),
|
|
30
32
|
retry_ambiguous: settings.fetch(:retry_ambiguous, false),
|
|
@@ -6,8 +6,8 @@ module Cloudflare
|
|
|
6
6
|
# `bin/rails cloudflare:email:deploy_worker` — uploads the Worker
|
|
7
7
|
# script + both secrets via the Cloudflare API. No wrangler required.
|
|
8
8
|
class DeployWorkerTask < TaskBase
|
|
9
|
-
def self.call(script_path: nil, ingress_url: nil, io: $stdout)
|
|
10
|
-
new(io: io, script_path: script_path, ingress_url: ingress_url).call
|
|
9
|
+
def self.call(script_path: nil, ingress_url: nil, delivery_mode: "durable", io: $stdout)
|
|
10
|
+
new(io: io, script_path: script_path, ingress_url: ingress_url, delivery_mode: delivery_mode).call
|
|
11
11
|
end
|
|
12
12
|
|
|
13
13
|
protected
|
|
@@ -28,7 +28,7 @@ module Cloudflare
|
|
|
28
28
|
)
|
|
29
29
|
|
|
30
30
|
say " Deploying Worker '#{deployer.script_name}'..."
|
|
31
|
-
deployer.deploy(script_path: path)
|
|
31
|
+
deployer.deploy(script_path: path, delivery_mode: opts[:delivery_mode])
|
|
32
32
|
say " ✓ Worker script deployed"
|
|
33
33
|
|
|
34
34
|
deployer.put_secret("INGRESS_SECRET", ingress_secret)
|
|
@@ -32,6 +32,11 @@ module Cloudflare
|
|
|
32
32
|
end
|
|
33
33
|
|
|
34
34
|
config.before_initialize do
|
|
35
|
+
unless defined?(Cloudflare::Email::Management::Engine)
|
|
36
|
+
Rails.autoloaders.main.ignore(
|
|
37
|
+
File.expand_path("../../../app/controllers/cloudflare/email/management", __dir__),
|
|
38
|
+
)
|
|
39
|
+
end
|
|
35
40
|
unless defined?(::ActionMailbox::Engine)
|
|
36
41
|
Rails.autoloaders.main.ignore(
|
|
37
42
|
File.expand_path("../../../app/controllers/cloudflare/email/ingress_controller.rb", __dir__),
|
|
@@ -45,11 +45,11 @@ module Cloudflare
|
|
|
45
45
|
nil
|
|
46
46
|
end
|
|
47
47
|
|
|
48
|
-
# Only metadata written after v2 verification is trusted. MIME headers
|
|
48
|
+
# Only metadata written after v2/v3 verification is trusted. MIME headers
|
|
49
49
|
# (including X-CF-* headers) never participate in this lookup.
|
|
50
50
|
def self.for(inbound_email)
|
|
51
51
|
metadata = inbound_email.raw_email.blob&.metadata&.fetch(METADATA_KEY, nil)
|
|
52
|
-
return nil unless metadata.is_a?(Hash) && metadata["version"]
|
|
52
|
+
return nil unless metadata.is_a?(Hash) && [2, 3].include?(metadata["version"])
|
|
53
53
|
value = metadata.reject { |key, _| key == "version" }
|
|
54
54
|
valid?(value) ? value : nil
|
|
55
55
|
end
|
|
@@ -9,9 +9,12 @@ module Cloudflare
|
|
|
9
9
|
# Handler/parse/ack failures propagate; unacknowledged leases expire and
|
|
10
10
|
# are redelivered according to the queue's retry/dead-letter policy.
|
|
11
11
|
class EventConsumer < Client
|
|
12
|
-
|
|
12
|
+
# Up to 100 queue messages, including encoded payloads and metadata.
|
|
13
|
+
DEFAULT_MAX_RESPONSE_BYTES = 32 * 1_048_576
|
|
14
|
+
|
|
15
|
+
def initialize(queue_id:, domains: nil, max_response_bytes: DEFAULT_MAX_RESPONSE_BYTES, **options)
|
|
13
16
|
raise ConfigurationError, "queue_id is required" unless queue_id.to_s.match?(/\A[a-zA-Z0-9_-]+\z/)
|
|
14
|
-
super(**options.merge(retries: 0, retry_ambiguous: false))
|
|
17
|
+
super(**options.merge(retries: 0, retry_ambiguous: false, max_response_bytes: max_response_bytes))
|
|
15
18
|
@queue_id = queue_id
|
|
16
19
|
@domains = domains && Array(domains).map(&:downcase)
|
|
17
20
|
end
|
|
@@ -53,6 +56,12 @@ module Cloudflare
|
|
|
53
56
|
|
|
54
57
|
private
|
|
55
58
|
|
|
59
|
+
def response_byte_limit(uri)
|
|
60
|
+
# ACKs contain counts, never a queue batch. Keep their smaller budget
|
|
61
|
+
# even when a pull needs room for 100 encoded event payloads.
|
|
62
|
+
uri.path.end_with?("/messages/ack") ? [@max_response_bytes, Client::DEFAULT_MAX_RESPONSE_BYTES].min : @max_response_bytes
|
|
63
|
+
end
|
|
64
|
+
|
|
56
65
|
def queue_path(action)
|
|
57
66
|
"/accounts/#{account_id}/queues/#{@queue_id}/messages/#{action}"
|
|
58
67
|
end
|
|
@@ -79,7 +88,7 @@ module Cloudflare
|
|
|
79
88
|
end
|
|
80
89
|
DeliveryEvent.new(body)
|
|
81
90
|
rescue JSON::ParserError, ArgumentError, TypeError
|
|
82
|
-
raise ValidationError, "queue body must contain a valid JSON Email Sending event"
|
|
91
|
+
raise ValidationError, "queue body must contain a valid JSON Email Sending event", cause: nil
|
|
83
92
|
end
|
|
84
93
|
end
|
|
85
94
|
end
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
require "digest"
|
|
2
|
+
require "cloudflare/email/verification"
|
|
3
|
+
|
|
4
|
+
module Cloudflare
|
|
5
|
+
module Email
|
|
6
|
+
# Shared bounded verification for custom Rack/Rails ingestion endpoints.
|
|
7
|
+
# This API deliberately performs no domain lookup or business processing.
|
|
8
|
+
module Ingress
|
|
9
|
+
DEFAULT_MAX_EMAIL_BYTES = 25 * 1024 * 1024
|
|
10
|
+
Result = Struct.new(:status, :message, :bytes, keyword_init: true)
|
|
11
|
+
|
|
12
|
+
class VerifiedMessage
|
|
13
|
+
attr_reader :body, :envelope, :provider_metadata, :storage_metadata, :message_checksum
|
|
14
|
+
|
|
15
|
+
def initialize(body:, envelope:, provider_metadata:, version:)
|
|
16
|
+
@body = body.b.freeze
|
|
17
|
+
@envelope = Ingress.deep_freeze(envelope)
|
|
18
|
+
@provider_metadata = Ingress.deep_freeze(provider_metadata)
|
|
19
|
+
@storage_metadata = {Envelope::METADATA_KEY => envelope.merge("version" => version.to_i)}
|
|
20
|
+
if provider_metadata
|
|
21
|
+
@storage_metadata[ProviderMetadata::METADATA_KEY] = provider_metadata.merge("signature_version" => 3)
|
|
22
|
+
end
|
|
23
|
+
Ingress.deep_freeze(@storage_metadata)
|
|
24
|
+
identity = if version == "3"
|
|
25
|
+
"v3\0#{JSON.generate([envelope.fetch('from'), envelope.fetch('to'), ProviderMetadata.canonical(provider_metadata)])}\0".b
|
|
26
|
+
else
|
|
27
|
+
"v2\0#{envelope.fetch('to')}\0".b
|
|
28
|
+
end
|
|
29
|
+
@message_checksum = Digest::SHA256.hexdigest(identity + @body).freeze
|
|
30
|
+
freeze
|
|
31
|
+
end
|
|
32
|
+
private_class_method :new
|
|
33
|
+
|
|
34
|
+
# Call within Mailboxes.receive when using the optional registry so the
|
|
35
|
+
# framework records and memberships share the resolved tenant context.
|
|
36
|
+
# Custom endpoints may instead persist body/metadata in their own store.
|
|
37
|
+
def persist_action_mailbox!
|
|
38
|
+
::ActionMailbox::InboundEmail.transaction do
|
|
39
|
+
inbound = ::ActionMailbox::InboundEmail.create_and_extract_message_id!(body,
|
|
40
|
+
message_checksum: message_checksum, message_id: stable_message_id)
|
|
41
|
+
if inbound
|
|
42
|
+
blob = inbound.raw_email.blob
|
|
43
|
+
blob.update!(metadata: blob.metadata.merge(storage_metadata))
|
|
44
|
+
end
|
|
45
|
+
inbound
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
# ActionMailbox's fallback includes Socket.gethostname. A replay on a
|
|
52
|
+
# different Rails host must use the same database identity, while the
|
|
53
|
+
# original MIME stays untouched. Preserve parseable provider IDs.
|
|
54
|
+
def stable_message_id
|
|
55
|
+
parsed = begin
|
|
56
|
+
::Mail.from_source(body).message_id
|
|
57
|
+
rescue StandardError
|
|
58
|
+
nil
|
|
59
|
+
end
|
|
60
|
+
parsed || "#{message_checksum}@cloudflare-email.invalid"
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def self.verify(secret:, headers:, body:, content_length: nil,
|
|
65
|
+
max_email_bytes: DEFAULT_MAX_EMAIL_BYTES, now: Time.now.to_i, window: Verification::DEFAULT_WINDOW)
|
|
66
|
+
unless max_email_bytes.is_a?(Integer) && max_email_bytes.positive?
|
|
67
|
+
raise ArgumentError, "max_email_bytes must be a positive integer"
|
|
68
|
+
end
|
|
69
|
+
values = %w[Timestamp Signature Signature-Version Envelope Metadata].to_h do |name|
|
|
70
|
+
key = "X-CF-Email-#{name}"
|
|
71
|
+
value = headers[key] || headers["HTTP_#{key.upcase.tr('-', '_')}"]
|
|
72
|
+
[name, value.is_a?(String) ? value.dup.freeze : value]
|
|
73
|
+
end
|
|
74
|
+
options = {secret: secret, timestamp: values["Timestamp"], signature: values["Signature"],
|
|
75
|
+
version: values["Signature-Version"], envelope: values["Envelope"], metadata: values["Metadata"],
|
|
76
|
+
now: now, window: window}
|
|
77
|
+
status = Verification.verify_headers(**options)
|
|
78
|
+
return Result.new(status: status).freeze unless status == :ok
|
|
79
|
+
if content_length
|
|
80
|
+
length = Integer(content_length.to_s, exception: false)
|
|
81
|
+
return Result.new(status: :bad_signature).freeze unless length && length >= 0
|
|
82
|
+
return Result.new(status: :too_large).freeze if length > max_email_bytes
|
|
83
|
+
end
|
|
84
|
+
raw = if body.is_a?(String)
|
|
85
|
+
return Result.new(status: :too_large).freeze if body.bytesize > max_email_bytes
|
|
86
|
+
body.dup.b
|
|
87
|
+
elsif body.respond_to?(:read)
|
|
88
|
+
buffer = +"".b
|
|
89
|
+
while buffer.bytesize <= max_email_bytes
|
|
90
|
+
chunk = body.read(max_email_bytes + 1 - buffer.bytesize)
|
|
91
|
+
break if chunk.nil? || chunk.empty?
|
|
92
|
+
buffer << chunk.b
|
|
93
|
+
end
|
|
94
|
+
buffer
|
|
95
|
+
else
|
|
96
|
+
raise ArgumentError, "body must be a String or readable IO"
|
|
97
|
+
end
|
|
98
|
+
return Result.new(status: :too_large).freeze if raw.bytesize > max_email_bytes
|
|
99
|
+
status = Verification.verify(body: raw, **options)
|
|
100
|
+
return Result.new(status: status, bytes: raw.bytesize).freeze unless status == :ok
|
|
101
|
+
message = VerifiedMessage.send(:new, body: raw, version: values["Signature-Version"],
|
|
102
|
+
envelope: Envelope.decode(values["Envelope"]), provider_metadata: ProviderMetadata.decode(values["Metadata"]))
|
|
103
|
+
Result.new(status: :ok, message: message, bytes: raw.bytesize).freeze
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def self.deep_freeze(value)
|
|
107
|
+
case value
|
|
108
|
+
when Hash then value.each { |key, item| deep_freeze(key); deep_freeze(item) }
|
|
109
|
+
when Array then value.each { |item| deep_freeze(item) }
|
|
110
|
+
end
|
|
111
|
+
value.freeze
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
@@ -10,7 +10,7 @@ module Cloudflare
|
|
|
10
10
|
self.table_name = "cloudflare_email_receiving_domains"
|
|
11
11
|
STATES = %w[pending active suspended].freeze
|
|
12
12
|
attr_readonly :domain, :tenant_key, :account_id
|
|
13
|
-
before_validation { self.domain = domain.to_s.strip.downcase }
|
|
13
|
+
before_validation(on: :create) { self.domain = domain.to_s.strip.downcase }
|
|
14
14
|
validates :tenant_key, :account_id, presence: true
|
|
15
15
|
validates :domain, presence: true, uniqueness: true,
|
|
16
16
|
length: { maximum: 253 },
|
|
@@ -71,13 +71,14 @@ module Cloudflare
|
|
|
71
71
|
self.table_name = "cloudflare_email_addresses"
|
|
72
72
|
belongs_to :mailbox, class_name: "Cloudflare::Email::Mailboxes::Mailbox"
|
|
73
73
|
attr_readonly :mailbox_id, :receiving_domain_id, :local_part, :domain, :address
|
|
74
|
-
before_validation :normalize_address
|
|
74
|
+
before_validation :normalize_address, on: :create
|
|
75
75
|
validates :mailbox, :receiving_domain_id, presence: true
|
|
76
76
|
validates :local_part, length: { in: 1..64 },
|
|
77
77
|
format: { with: /\A[a-z0-9!\#$%&'*+\/=\?^_`{|}~-]+(?:\.[a-z0-9!\#$%&'*+\/=\?^_`{|}~-]+)*\z/ }
|
|
78
78
|
validates :address, uniqueness: true, length: { maximum: 254 }
|
|
79
79
|
validates :state, inclusion: { in: %w[pending active suspended] }
|
|
80
80
|
validate :validate_directory
|
|
81
|
+
validate :validate_catch_all
|
|
81
82
|
validate { validate_parent_tenant(mailbox, :mailbox) }
|
|
82
83
|
scope :active, -> { where(state: "active") }
|
|
83
84
|
|
|
@@ -103,6 +104,22 @@ module Cloudflare
|
|
|
103
104
|
errors.add(:receiving_domain_id, "must be active before creating addresses")
|
|
104
105
|
end
|
|
105
106
|
end
|
|
107
|
+
|
|
108
|
+
def validate_catch_all
|
|
109
|
+
# Existing exact-address installations need not migrate until they
|
|
110
|
+
# opt into catch-all receiving.
|
|
111
|
+
return unless has_attribute?(:catch_all) && self[:catch_all]
|
|
112
|
+
errors.add(:catch_all_evidence, "is required") if self[:catch_all_evidence].to_s.strip.empty?
|
|
113
|
+
if will_save_change_to_catch_all? && state != "active"
|
|
114
|
+
errors.add(:catch_all, "requires an active address")
|
|
115
|
+
end
|
|
116
|
+
return unless state == "active"
|
|
117
|
+
errors.add(:catch_all, "requires an active mailbox") unless mailbox&.state == "active"
|
|
118
|
+
errors.add(:catch_all, "requires an active receiving domain") unless receiving_domain&.state == "active"
|
|
119
|
+
if self.class.where(receiving_domain_id: receiving_domain_id, catch_all: true, state: "active").where.not(id: id).exists?
|
|
120
|
+
errors.add(:catch_all, "already exists for this receiving domain")
|
|
121
|
+
end
|
|
122
|
+
end
|
|
106
123
|
end
|
|
107
124
|
|
|
108
125
|
class Message < Cloudflare::Email::ActiveRecord::Base
|