mailcycle 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +202 -0
- data/README.md +208 -0
- data/lib/mailcycle/client.rb +803 -0
- data/lib/mailcycle/crypto/asymmetric.rb +126 -0
- data/lib/mailcycle/crypto/auth_proof.rb +58 -0
- data/lib/mailcycle/crypto/cipher.rb +136 -0
- data/lib/mailcycle/crypto/identity.rb +170 -0
- data/lib/mailcycle/crypto/mnemonic.rb +126 -0
- data/lib/mailcycle/crypto/primitives.rb +116 -0
- data/lib/mailcycle/crypto/sealed_box.rb +93 -0
- data/lib/mailcycle/crypto/wordlist.rb +171 -0
- data/lib/mailcycle/encoding.rb +93 -0
- data/lib/mailcycle/errors.rb +82 -0
- data/lib/mailcycle/events.rb +310 -0
- data/lib/mailcycle/http.rb +113 -0
- data/lib/mailcycle/keys.rb +81 -0
- data/lib/mailcycle/mail.rb +143 -0
- data/lib/mailcycle/manage.rb +202 -0
- data/lib/mailcycle/models.rb +243 -0
- data/lib/mailcycle/pairing.rb +338 -0
- data/lib/mailcycle/trackers.rb +312 -0
- data/lib/mailcycle/version.rb +5 -0
- data/lib/mailcycle/webhook_signature.rb +60 -0
- data/lib/mailcycle/websocket.rb +320 -0
- data/lib/mailcycle.rb +109 -0
- metadata +83 -0
|
@@ -0,0 +1,338 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module Mailcycle
|
|
6
|
+
# What a scanned pairing QR code, or a typed code, carries.
|
|
7
|
+
#
|
|
8
|
+
# `code` is the eight digits. `token` is a pairing token, from an old deep
|
|
9
|
+
# link; nothing can be looked up by it. `wrap` is the device's wrap secret,
|
|
10
|
+
# base64url; only a QR code carries it.
|
|
11
|
+
ScannedPairing = Struct.new(:code, :token, :wrap, keyword_init: true)
|
|
12
|
+
|
|
13
|
+
# One address's keys at one epoch above 0, base64url.
|
|
14
|
+
EpochKey = Struct.new(:key, :sealing_key, keyword_init: true) do
|
|
15
|
+
def inspect
|
|
16
|
+
"#<Mailcycle::EpochKey ..>"
|
|
17
|
+
end
|
|
18
|
+
alias_method :to_s, :inspect
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# The address keys for one device. The server stores and relays it blindly.
|
|
22
|
+
#
|
|
23
|
+
# - `keys`: address id to its symmetric key, base64url, for mail the
|
|
24
|
+
# account's own devices sealed.
|
|
25
|
+
# - `sealing_keys`: address id to its X25519 private key, base64url, for
|
|
26
|
+
# mail that arrived from outside.
|
|
27
|
+
# - `epoch_keys`: a rotated address's later epochs, address id to epoch (a
|
|
28
|
+
# string) to EpochKey. Epoch 0 stays in `keys` and `sealing_keys`, for
|
|
29
|
+
# older readers.
|
|
30
|
+
# - `sender_name`: an agent's sender name.
|
|
31
|
+
# - `sender_names`: address id to the name set on that address, which wins
|
|
32
|
+
# over `sender_name`.
|
|
33
|
+
#
|
|
34
|
+
# `inspect` shows counts, never the keys: they are live private key
|
|
35
|
+
# material, and would otherwise reach any log they were printed to.
|
|
36
|
+
KeyBundle = Struct.new(:keys, :sealing_keys, :epoch_keys, :sender_name, :sender_names, :issued_at,
|
|
37
|
+
keyword_init: true) do
|
|
38
|
+
# The JSON the device reads, field for field as the app writes it.
|
|
39
|
+
def to_wire
|
|
40
|
+
out = { "keys" => keys.sort.to_h, "sealingKeys" => sealing_keys.sort.to_h }
|
|
41
|
+
unless epoch_keys.empty?
|
|
42
|
+
out["epochKeys"] = epoch_keys.sort.to_h do |id, epochs|
|
|
43
|
+
[id, epochs.sort.to_h { |epoch, key| [epoch, { "key" => key.key, "sealingKey" => key.sealing_key }] }]
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
out["senderName"] = sender_name unless sender_name.nil?
|
|
47
|
+
out["senderNames"] = sender_names.sort.to_h unless sender_names.empty?
|
|
48
|
+
out["issuedAt"] = issued_at
|
|
49
|
+
out
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def to_json(*args)
|
|
53
|
+
to_wire.to_json(*args)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def self.from_wire(wire)
|
|
57
|
+
wire = Wire.object(wire)
|
|
58
|
+
strings = ->(value) { Wire.object(value).select { |_, v| v.is_a?(String) } }
|
|
59
|
+
epochs = Wire.object(wire["epochKeys"]).to_h do |id, by_epoch|
|
|
60
|
+
[id, Wire.object(by_epoch).to_h do |epoch, key|
|
|
61
|
+
key = Wire.object(key)
|
|
62
|
+
[epoch, EpochKey.new(key: Wire.text(key["key"]), sealing_key: Wire.text(key["sealingKey"]))]
|
|
63
|
+
end]
|
|
64
|
+
end
|
|
65
|
+
new(keys: strings.call(wire["keys"]), sealing_keys: strings.call(wire["sealingKeys"]), epoch_keys: epochs,
|
|
66
|
+
sender_name: Wire.maybe_text(wire["senderName"]), sender_names: strings.call(wire["senderNames"]),
|
|
67
|
+
issued_at: Wire.text(wire["issuedAt"]))
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def inspect
|
|
71
|
+
"#<Mailcycle::KeyBundle keys=#{keys.length} sealing_keys=#{sealing_keys.length} " \
|
|
72
|
+
"sender_name=#{sender_name.inspect} sender_names=#{sender_names.inspect} issued_at=#{issued_at.inspect}>"
|
|
73
|
+
end
|
|
74
|
+
alias_method :to_s, :inspect
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Pairing a device, and handing it the keys for its addresses.
|
|
78
|
+
#
|
|
79
|
+
# This is the account's side of pairing, as the app does it on the "Add
|
|
80
|
+
# device" screen. The device opens what is sealed here with code this
|
|
81
|
+
# library never runs, so every byte follows the app: `pairingCore.ts`,
|
|
82
|
+
# `api/pairingApi.ts` and `handKeysTo` in `api/fleetApi.ts`.
|
|
83
|
+
#
|
|
84
|
+
# Each device has a 64-byte wrap secret. Its address keys are sealed under
|
|
85
|
+
# it, and the account keeps a copy in the device's sealed profile so it can
|
|
86
|
+
# hand over keys again when an address is assigned later. The secret reaches
|
|
87
|
+
# the account one of two ways:
|
|
88
|
+
#
|
|
89
|
+
# - Optical. The device shows the secret in its QR code
|
|
90
|
+
# (`MC1:<code>:<wrap>`). It reaches the account without passing through
|
|
91
|
+
# Mailcycle.
|
|
92
|
+
# - Relayed. Only the eight digits are typed, which carry no secret. The
|
|
93
|
+
# device publishes a public key when it starts pairing, and the account
|
|
94
|
+
# mints a secret and seals it to that key. The server relays the key, so
|
|
95
|
+
# it could put its own in its place at pairing and read what follows. It
|
|
96
|
+
# cannot read anything passively.
|
|
97
|
+
module Pairing
|
|
98
|
+
QR_PREFIX = "MC1:"
|
|
99
|
+
WRAP_BYTES = 64
|
|
100
|
+
WORKER_RECORD = "worker"
|
|
101
|
+
|
|
102
|
+
module_function
|
|
103
|
+
|
|
104
|
+
# Reads a scanned QR value or a typed code, as `parseScannedValue` does.
|
|
105
|
+
#
|
|
106
|
+
# Accepts `MC1:<code>`, `MC1:<code>:<wrap>`, the older JSON payload, a
|
|
107
|
+
# deep link with a token, or eight digits written any way. nil if it is
|
|
108
|
+
# none of those.
|
|
109
|
+
def parse_scanned_value(raw)
|
|
110
|
+
value = raw.to_s.strip
|
|
111
|
+
return nil if value.empty?
|
|
112
|
+
|
|
113
|
+
if value.start_with?(QR_PREFIX)
|
|
114
|
+
code, wrap = value.delete_prefix(QR_PREFIX).split(":", 2)
|
|
115
|
+
code_ok = code.to_s.match?(/\A[0-9]{8}\z/)
|
|
116
|
+
wrap_ok = wrap.nil? || wrap.match?(/\A[A-Za-z0-9_-]+\z/)
|
|
117
|
+
# The app falls through to reading the digits out of a malformed one,
|
|
118
|
+
# which picks up the 1 in `MC1`. Refused here instead.
|
|
119
|
+
return nil unless code_ok && wrap_ok
|
|
120
|
+
|
|
121
|
+
return ScannedPairing.new(code: code, token: nil, wrap: wrap)
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# Older JSON payloads, from devices still showing a pre-update screen.
|
|
125
|
+
parsed = begin
|
|
126
|
+
JSON.parse(value)
|
|
127
|
+
rescue JSON::ParserError
|
|
128
|
+
nil
|
|
129
|
+
end
|
|
130
|
+
if parsed.is_a?(Hash) && parsed["type"] == "mailcycle.pairing" && parsed["token"].is_a?(String)
|
|
131
|
+
text = ->(field) { parsed[field].is_a?(String) && !parsed[field].empty? ? parsed[field] : nil }
|
|
132
|
+
return ScannedPairing.new(token: parsed["token"], code: text.call("code"), wrap: text.call("wrap"))
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
start = value.index("token=")
|
|
136
|
+
if start&.positive? && ["?", "&"].include?(value[start - 1])
|
|
137
|
+
token = value[(start + 6)..][/\A[A-Za-z0-9_]*/]
|
|
138
|
+
return ScannedPairing.new(code: nil, token: token, wrap: nil) unless token.empty?
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
digits = value.scan(/[0-9]/).first(8).join
|
|
142
|
+
digits.length == 8 ? ScannedPairing.new(code: digits, token: nil, wrap: nil) : nil
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# A new wrap secret: 64 random bytes, base64url.
|
|
146
|
+
def generate_wrap_secret
|
|
147
|
+
Encoding.to_b64u(Crypto::Primitives.random_bytes(WRAP_BYTES))
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# The key a device's key bundle is sealed under: the wrap secret itself.
|
|
151
|
+
#
|
|
152
|
+
# A secret of the wrong length is an error, never a fall back to a weaker
|
|
153
|
+
# key.
|
|
154
|
+
def wrapping_key(wrap_secret)
|
|
155
|
+
secret = begin
|
|
156
|
+
Encoding.from_b64u(wrap_secret)
|
|
157
|
+
rescue Base64UrlError
|
|
158
|
+
raise wrap_secret_error
|
|
159
|
+
end
|
|
160
|
+
raise wrap_secret_error unless secret.bytesize == WRAP_BYTES
|
|
161
|
+
|
|
162
|
+
secret
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def wrap_secret_error
|
|
166
|
+
UnsupportedError.new("That pairing secret is not usable. Pair the device again.")
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# What a relayed wrap secret is bound to.
|
|
170
|
+
def wrap_seal_aad(worker_id)
|
|
171
|
+
"mailcycle/v1/pairing-wrap:#{worker_id}"
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
# Seals a freshly minted wrap secret to the key the device published.
|
|
175
|
+
def seal_wrap_for_device(device_key, wrap_secret, worker_id)
|
|
176
|
+
public_key = Encoding.from_b64u(device_key)
|
|
177
|
+
Crypto::Asymmetric.seal_to_public_key(public_key, wrapping_key(wrap_secret), wrap_seal_aad(worker_id))
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# Now, as JavaScript's `toISOString` writes it: `2026-09-21T10:00:00.000Z`.
|
|
181
|
+
def iso_now
|
|
182
|
+
now = Time.now.utc
|
|
183
|
+
iso_at(now.to_i, now.usec / 1000)
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
def iso_at(seconds, millis)
|
|
187
|
+
"#{Time.at(seconds).utc.strftime('%Y-%m-%dT%H:%M:%S')}.#{format('%03d', millis)}Z"
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# Pairing, and handing a device the keys for its addresses.
|
|
192
|
+
class Devices
|
|
193
|
+
# Pairs a device with the account, as the app's "Add device" screen does.
|
|
194
|
+
#
|
|
195
|
+
# `code_or_qr` is the eight digits the device shows, or the text of its QR
|
|
196
|
+
# code. The QR code carries the device's wrap secret, so its keys reach it
|
|
197
|
+
# without passing through Mailcycle. With the digits alone the device must
|
|
198
|
+
# publish a pairing key; a secret is minted here and sealed to it, and the
|
|
199
|
+
# server relays that key, so it could put its own in its place at pairing.
|
|
200
|
+
# Prefer the QR code where there is one.
|
|
201
|
+
#
|
|
202
|
+
# `name` is sealed on this machine. An empty one takes the name the device
|
|
203
|
+
# gave. Needs the phrase and a signed-in session.
|
|
204
|
+
def pair(code_or_qr, name)
|
|
205
|
+
keys = @client.require_keys("pair a device")
|
|
206
|
+
scanned = Pairing.parse_scanned_value(code_or_qr) || ScannedPairing.new
|
|
207
|
+
if scanned.code.nil?
|
|
208
|
+
message = if scanned.token
|
|
209
|
+
"That QR code does not carry a pairing code. Enter the 8 digits shown on the device."
|
|
210
|
+
else
|
|
211
|
+
"A pairing code is eight digits."
|
|
212
|
+
end
|
|
213
|
+
raise ApiError.new(400, "invalid_code", message)
|
|
214
|
+
end
|
|
215
|
+
Pairing.wrapping_key(scanned.wrap) if scanned.wrap
|
|
216
|
+
|
|
217
|
+
preview = Wire.object(@client.http.request("POST", "/workers/pair/lookup", body: { "code" => scanned.code }))
|
|
218
|
+
session_id = Wire.text(preview["sessionId"])
|
|
219
|
+
device_key = Wire.maybe_text(preview["deviceKey"])
|
|
220
|
+
device_key = nil if device_key && device_key.empty?
|
|
221
|
+
worker_id = Vault.new_id("wkr", 10)
|
|
222
|
+
|
|
223
|
+
wrap_seal = nil
|
|
224
|
+
if scanned.wrap
|
|
225
|
+
wrap = scanned.wrap
|
|
226
|
+
transport = "optical"
|
|
227
|
+
elsif device_key
|
|
228
|
+
wrap = Pairing.generate_wrap_secret
|
|
229
|
+
wrap_seal = Pairing.seal_wrap_for_device(device_key, wrap, worker_id)
|
|
230
|
+
transport = "relayed"
|
|
231
|
+
else
|
|
232
|
+
raise UnsupportedError,
|
|
233
|
+
"This device publishes no pairing key, so its keys cannot be handed over from a typed code. " \
|
|
234
|
+
"Pass the text of its QR code instead, or pair it in the app."
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
given = name.to_s.strip
|
|
238
|
+
if given.empty?
|
|
239
|
+
offered = Wire.text(preview["deviceName"]).strip
|
|
240
|
+
given = offered.empty? ? "New device" : offered
|
|
241
|
+
end
|
|
242
|
+
# A device's sealed profile, as `encodeWorkerProfile` writes it.
|
|
243
|
+
profile = { "name" => given, "tags" => [], "notes" => "", "wrap" => wrap }
|
|
244
|
+
sealed = Vault.seal(keys, Pairing::WORKER_RECORD, worker_id, profile)
|
|
245
|
+
|
|
246
|
+
body = { "sessionId" => session_id, "workerId" => worker_id, "profile" => sealed, "keyTransport" => transport }
|
|
247
|
+
body["wrapSeal"] = wrap_seal.to_wire if wrap_seal
|
|
248
|
+
result = Wire.object(@client.http.request("POST", "/workers/pair", body: body))
|
|
249
|
+
|
|
250
|
+
wire = result["worker"].is_a?(Hash) ? result["worker"].dup : { "id" => worker_id }
|
|
251
|
+
wire["profile"] = sealed
|
|
252
|
+
to_device(wire)
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
# Hands a device the keys for every address assigned to it.
|
|
256
|
+
#
|
|
257
|
+
# Opens the device's sealed profile for its wrap secret, seals a fresh key
|
|
258
|
+
# bundle under it, and uploads that. `assign` and `unassign` call this
|
|
259
|
+
# themselves. Needs the phrase and a signed-in session.
|
|
260
|
+
def hand_over_keys(device_id)
|
|
261
|
+
keys = @client.require_keys("hand a device its keys")
|
|
262
|
+
worker = Wire.object(Wire.object(@client.http.request("GET", worker_path(device_id)))["worker"])
|
|
263
|
+
|
|
264
|
+
profile = Vault.open_or_empty(keys, Pairing::WORKER_RECORD, device_id, worker["profile"])
|
|
265
|
+
wrap = Wire.text(profile["wrap"])
|
|
266
|
+
if wrap.empty?
|
|
267
|
+
message = if worker["kind"] == "agent"
|
|
268
|
+
"This agent was connected without its key. Remove it and connect it again."
|
|
269
|
+
else
|
|
270
|
+
"This device was paired without keeping its key, so it cannot open the mail. Unpair it, " \
|
|
271
|
+
"pair it again, then assign the address."
|
|
272
|
+
end
|
|
273
|
+
raise ApiError.new(409, "no_wrap", message)
|
|
274
|
+
end
|
|
275
|
+
wrap_key = Pairing.wrapping_key(wrap)
|
|
276
|
+
|
|
277
|
+
sender_name = Wire.text(profile["senderName"])
|
|
278
|
+
bundle = KeyBundle.new(keys: {}, sealing_keys: {}, epoch_keys: {},
|
|
279
|
+
sender_name: sender_name.empty? ? nil : sender_name,
|
|
280
|
+
sender_names: {}, issued_at: Pairing.iso_now)
|
|
281
|
+
inboxes = Wire.array(Wire.object(@client.http.request("GET", "/inboxes"))["inboxes"])
|
|
282
|
+
inboxes.each { |inbox| add_to_bundle(bundle, keys, device_id, Wire.object(inbox)) }
|
|
283
|
+
|
|
284
|
+
sealed = Crypto::Cipher.seal_json(wrap_key, bundle.to_wire, device_id)
|
|
285
|
+
@client.http.request("POST", "#{worker_path(device_id)}/keys", body: { "keyBundle" => sealed.to_wire })
|
|
286
|
+
nil
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
# Assigns an address to a device, then hands it the keys to open it.
|
|
290
|
+
#
|
|
291
|
+
# The address is assigned even if the handover fails, and the error says
|
|
292
|
+
# why. Needs the phrase and a signed-in session.
|
|
293
|
+
def assign(device_id, address_id)
|
|
294
|
+
@client.http.request("POST", "#{worker_path(device_id)}/inboxes", body: { "inboxId" => address_id })
|
|
295
|
+
hand_over_keys(device_id)
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
# Takes an address off a device, then hands it a bundle without that
|
|
299
|
+
# address's keys.
|
|
300
|
+
#
|
|
301
|
+
# The second step is best effort, as in the app: the address is already
|
|
302
|
+
# gone from the device either way.
|
|
303
|
+
def unassign(device_id, address_id)
|
|
304
|
+
@client.http.request("DELETE", "#{worker_path(device_id)}/inboxes/#{Mailcycle.path_segment(address_id)}")
|
|
305
|
+
begin
|
|
306
|
+
hand_over_keys(device_id)
|
|
307
|
+
rescue Error
|
|
308
|
+
nil
|
|
309
|
+
end
|
|
310
|
+
nil
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
private
|
|
314
|
+
|
|
315
|
+
def add_to_bundle(bundle, keys, device_id, inbox)
|
|
316
|
+
return unless inbox["assignedWorkerId"] == device_id
|
|
317
|
+
|
|
318
|
+
id = inbox["id"]
|
|
319
|
+
return unless id.is_a?(String)
|
|
320
|
+
|
|
321
|
+
# The name set on the address, opened from its sealed details.
|
|
322
|
+
meta = Vault.open_or_empty(keys, "inbox", id, inbox["meta"])
|
|
323
|
+
sender_name = Wire.text(meta["senderName"])
|
|
324
|
+
bundle.sender_names[id] = sender_name unless sender_name.empty?
|
|
325
|
+
bundle.keys[id] = Encoding.to_b64u(keys.address_symmetric_key(id))
|
|
326
|
+
bundle.sealing_keys[id] = Encoding.to_b64u(keys.address_private_key(id))
|
|
327
|
+
# Every epoch a rotated address has reached, so the device reads mail
|
|
328
|
+
# from both sides of the rotation.
|
|
329
|
+
reached = Mail.key_epoch(inbox)
|
|
330
|
+
(1..reached).each do |epoch|
|
|
331
|
+
(bundle.epoch_keys[id] ||= {})[epoch.to_s] = EpochKey.new(
|
|
332
|
+
key: Encoding.to_b64u(keys.address_symmetric_key(id, epoch: epoch)),
|
|
333
|
+
sealing_key: Encoding.to_b64u(keys.address_private_key(id, epoch: epoch))
|
|
334
|
+
)
|
|
335
|
+
end
|
|
336
|
+
end
|
|
337
|
+
end
|
|
338
|
+
end
|
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Mailcycle
|
|
4
|
+
# One remote resource that was removed. `kind` is `pixel`, `image` or `link`.
|
|
5
|
+
BlockedTracker = Struct.new(:url, :host, :kind, keyword_init: true)
|
|
6
|
+
|
|
7
|
+
# What Trackers.strip_remote_content produced.
|
|
8
|
+
StripResult = Struct.new(:html, :blocked, keyword_init: true)
|
|
9
|
+
|
|
10
|
+
# Tracker stripping.
|
|
11
|
+
#
|
|
12
|
+
# Opening an email should not tell the sender that you opened it. Most HTML
|
|
13
|
+
# mail carries at least one remote resource whose only job is to report back:
|
|
14
|
+
# a 1x1 image on a per-recipient URL. Loading it leaks the open, the time,
|
|
15
|
+
# the approximate location and, across devices, how many devices you read on.
|
|
16
|
+
#
|
|
17
|
+
# Mailcycle's rule is absolute and needs no per-message decision from the
|
|
18
|
+
# reader: remote content in mail is never fetched. `Message#safe_html` is the
|
|
19
|
+
# body with everything remote removed, and `Message#blocked` is what was
|
|
20
|
+
# taken out.
|
|
21
|
+
#
|
|
22
|
+
# A remote resource hides in more than `<img src>`: `srcset`, CSS
|
|
23
|
+
# `background-image: url(...)` in a style attribute or a `<style>` block, SVG
|
|
24
|
+
# `<image href>`/`<use href>`, `<video poster>`,
|
|
25
|
+
# `<iframe>`/`<object>`/`<embed>`, and a scheme obfuscated with HTML entities
|
|
26
|
+
# (`https://...`) all fetch. Every one of them is covered. An `<a href>`
|
|
27
|
+
# is left alone, because it loads only when the reader clicks it.
|
|
28
|
+
#
|
|
29
|
+
# This is a port of `src/privacy/trackers.ts`, kept behaviour-for-behaviour
|
|
30
|
+
# with it so the same message reads the same way here as in the app. The
|
|
31
|
+
# JavaScript patterns run under JavaScript's rules, which are not Ruby's:
|
|
32
|
+
# `^` is the start of the text rather than of a line, `\s` includes the
|
|
33
|
+
# Unicode spaces, `\b` and `\w` are ASCII, and a case-insensitive letter
|
|
34
|
+
# never matches a non-ASCII one (Ruby's would match `ſ` for `s`). So each
|
|
35
|
+
# pattern below spells those out rather than borrowing Ruby's meaning, and
|
|
36
|
+
# the differential test runs both implementations over the same markup and
|
|
37
|
+
# requires identical output.
|
|
38
|
+
module Trackers
|
|
39
|
+
# What JavaScript's `\s` matches.
|
|
40
|
+
WS = "\t\n\v\f\r -
"
|
|
41
|
+
# JavaScript's `\w`, which is ASCII.
|
|
42
|
+
WORD = "A-Za-z0-9_"
|
|
43
|
+
|
|
44
|
+
# An ASCII word, matched without regard to case the way JavaScript's `i`
|
|
45
|
+
# flag does it.
|
|
46
|
+
def self.ci(word)
|
|
47
|
+
word.each_char.map { |c| c.match?(/[A-Za-z]/) ? "[#{c.upcase}#{c.downcase}]" : Regexp.escape(c) }.join
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Which attributes fetch a remote resource, per element. `href` is listed
|
|
51
|
+
# only for SVG `<image>`/`<use>`: an `<a href>` is a click, not an
|
|
52
|
+
# automatic load.
|
|
53
|
+
RESOURCE_ATTRS = {
|
|
54
|
+
"img" => %w[src srcset lowsrc],
|
|
55
|
+
"source" => %w[src srcset],
|
|
56
|
+
"image" => %w[xlink:href href],
|
|
57
|
+
"use" => %w[xlink:href href],
|
|
58
|
+
"video" => %w[src poster],
|
|
59
|
+
"audio" => %w[src],
|
|
60
|
+
"track" => %w[src],
|
|
61
|
+
"iframe" => %w[src],
|
|
62
|
+
"embed" => %w[src],
|
|
63
|
+
"object" => %w[data],
|
|
64
|
+
"input" => %w[src],
|
|
65
|
+
"body" => %w[background],
|
|
66
|
+
"table" => %w[background],
|
|
67
|
+
"td" => %w[background],
|
|
68
|
+
"th" => %w[background],
|
|
69
|
+
"tr" => %w[background]
|
|
70
|
+
}.freeze
|
|
71
|
+
|
|
72
|
+
# Elements that exist only to load their remote resource: drop them whole.
|
|
73
|
+
DROP_WHOLE = %w[img source].freeze
|
|
74
|
+
|
|
75
|
+
# Hosts whose presence in mail is tracking and nothing else.
|
|
76
|
+
TRACKER_HINTS = %w[
|
|
77
|
+
open. track. trk. click. email. mailstat pixel beacon sendgrid.net mailchimp list-manage hubspot
|
|
78
|
+
mktoresp omtrdc doubleclick
|
|
79
|
+
].freeze
|
|
80
|
+
|
|
81
|
+
DIMENSION = /(?<![#{WORD}])(#{ci('width')}|#{ci('height')})[#{WS}]*=[#{WS}]*["']?[#{WS}]*([0-9]+)/
|
|
82
|
+
REMOTE = %r{\A(#{ci('http')}[sS]?:)?//}
|
|
83
|
+
# The JavaScript is `url\(\s*(['"]?)([^'")]+)\1\s*\)`, backreference and all.
|
|
84
|
+
CSS_URL = /#{ci('url')}\([#{WS}]*(['"]?)([^'")]+)\1[#{WS}]*\)/
|
|
85
|
+
STYLE_BLOCK = %r{<#{ci('style')}(?![#{WORD}])[^>]*>(.*?)</#{ci('style')}>}m
|
|
86
|
+
# The attributes group tolerates quoted values that contain `>`.
|
|
87
|
+
TAG = /<([a-zA-Z][#{WORD}:-]*)((?:"[^"]*"|'[^']*'|[^>"'])*)>/
|
|
88
|
+
HOST = %r{\A(?:#{ci('http')}[sS]?:)?//([^/?#]+)}
|
|
89
|
+
WWW = /\A#{ci('www')}\./
|
|
90
|
+
ENTITY_HEX = /&#[xX]([0-9a-fA-F]+);?/
|
|
91
|
+
ENTITY_DEC = /&#([0-9]+);?/
|
|
92
|
+
ENTITY_NAMED = /&(#{ci('amp')}|#{ci('colon')}|#{ci('sol')}|#{ci('NewLine')}|#{ci('Tab')});/
|
|
93
|
+
# `[\s\x00-\x1f]`: the control characters cover JavaScript's ASCII spaces.
|
|
94
|
+
IGNORABLE = /[\u0000-\u001f \u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+/
|
|
95
|
+
LINK = %r{#{ci('http')}[sS]?://[^#{WS}<>"')]+}
|
|
96
|
+
SPACES = /[#{WS}]+/
|
|
97
|
+
TRIM = /\A[#{WS}]+|[#{WS}]+\z/
|
|
98
|
+
|
|
99
|
+
# A leading boundary (start, whitespace or quote) keeps `href` from
|
|
100
|
+
# matching inside `xlink:href`. Groups: 1 boundary, 2 name, 3 whole value,
|
|
101
|
+
# 4/5/6 the double-quoted / single-quoted / unquoted value.
|
|
102
|
+
ATTR_PATTERNS = (RESOURCE_ATTRS.values.flatten.uniq + ["style"]).to_h do |attr|
|
|
103
|
+
[attr, /(\A|[#{WS}"'])(#{ci(attr)})[#{WS}]*=[#{WS}]*("([^"]*)"|'([^']*)'|([^#{WS}>]+))/]
|
|
104
|
+
end.freeze
|
|
105
|
+
|
|
106
|
+
module_function
|
|
107
|
+
|
|
108
|
+
def code_point(value)
|
|
109
|
+
return "" if value > 0x10ffff
|
|
110
|
+
# A lone surrogate has no UTF-8 spelling. JavaScript keeps it as it is;
|
|
111
|
+
# here it becomes the replacement character, which is not whitespace and
|
|
112
|
+
# not part of any scheme, so it decides nothing differently.
|
|
113
|
+
return "�" if value.between?(0xd800, 0xdfff)
|
|
114
|
+
|
|
115
|
+
[value].pack("U")
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Collapses HTML entities and ignorable characters.
|
|
119
|
+
#
|
|
120
|
+
# Used only to decide remoteness, so a scheme a renderer would resolve is
|
|
121
|
+
# judged by what it resolves to rather than by how it was spelled. The
|
|
122
|
+
# original text is what gets removed.
|
|
123
|
+
def canonical(value)
|
|
124
|
+
current = value
|
|
125
|
+
3.times do
|
|
126
|
+
step = current.gsub(ENTITY_HEX) { code_point(Regexp.last_match(1).to_i(16)) }
|
|
127
|
+
step = step.gsub(ENTITY_DEC) { code_point(Regexp.last_match(1).to_i) }
|
|
128
|
+
step = step.gsub(ENTITY_NAMED) do
|
|
129
|
+
case Regexp.last_match(1).downcase
|
|
130
|
+
when "amp" then "&"
|
|
131
|
+
when "colon" then ":"
|
|
132
|
+
when "sol" then "/"
|
|
133
|
+
else ""
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
break if step == current
|
|
137
|
+
|
|
138
|
+
current = step
|
|
139
|
+
end
|
|
140
|
+
current.gsub(IGNORABLE, "")
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# The host of a URL, lowercased and without a leading `www.`.
|
|
144
|
+
def host_of(url)
|
|
145
|
+
match = HOST.match(canonical(utf8(url)).gsub(TRIM, ""))
|
|
146
|
+
return "" if match.nil?
|
|
147
|
+
|
|
148
|
+
match[1].sub(WWW, "").downcase
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def remote?(url)
|
|
152
|
+
REMOTE.match?(canonical(url))
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# The URLs inside a `srcset`: each candidate is `url [descriptor]`.
|
|
156
|
+
def srcset_urls(value)
|
|
157
|
+
value.split(",", -1).map { |candidate| candidate.gsub(TRIM, "").split(SPACES).first || "" }
|
|
158
|
+
.reject(&:empty?)
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def attr_value(attrs, attr)
|
|
162
|
+
match = ATTR_PATTERNS[attr].match(attrs)
|
|
163
|
+
return nil if match.nil?
|
|
164
|
+
|
|
165
|
+
match[4] || match[5] || match[6] || ""
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def remove_attr(attrs, attr)
|
|
169
|
+
attrs.sub(ATTR_PATTERNS[attr]) { Regexp.last_match(1) }
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# The new value goes in as it is, never read as a replacement pattern:
|
|
173
|
+
# it is the sender's, and a `` $` `` or `$'` in it would otherwise copy the
|
|
174
|
+
# rest of the tag back in.
|
|
175
|
+
def set_attr(attrs, attr, value)
|
|
176
|
+
match = ATTR_PATTERNS[attr].match(attrs)
|
|
177
|
+
return attrs if match.nil?
|
|
178
|
+
|
|
179
|
+
"#{match.pre_match}#{match[1]}#{attr}=\"#{value.gsub('"', '"')}\"#{match.post_match}"
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# Pixel or ordinary blocked imagery.
|
|
183
|
+
#
|
|
184
|
+
# The distinction only affects wording; both are blocked identically.
|
|
185
|
+
def classify(tag, url)
|
|
186
|
+
smallest = tag.scan(DIMENSION).map { |_, digits| digits.to_i }.min
|
|
187
|
+
return "pixel" if smallest && smallest <= 3
|
|
188
|
+
|
|
189
|
+
host = host_of(url)
|
|
190
|
+
TRACKER_HINTS.any? { |hint| host.include?(hint) } ? "pixel" : "image"
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def utf8(text)
|
|
194
|
+
value = text.to_s
|
|
195
|
+
value = value.dup.force_encoding(::Encoding::UTF_8) unless value.encoding == ::Encoding::UTF_8
|
|
196
|
+
value.valid_encoding? ? value : value.scrub
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
# Removes every remote resource from an HTML body.
|
|
200
|
+
#
|
|
201
|
+
# Inline `cid:` and `data:` references are left alone; they travel inside
|
|
202
|
+
# the message and fetch nothing. Everything remote goes, tracker or not,
|
|
203
|
+
# because "block only the ones that look like trackers" is a game the
|
|
204
|
+
# sender wins.
|
|
205
|
+
def strip_remote_content(html)
|
|
206
|
+
html = utf8(html)
|
|
207
|
+
blocked = []
|
|
208
|
+
seen = {}
|
|
209
|
+
record = lambda do |url, tag|
|
|
210
|
+
next if seen.key?(url)
|
|
211
|
+
|
|
212
|
+
seen[url] = true
|
|
213
|
+
blocked << BlockedTracker.new(url: url, host: host_of(url), kind: classify(tag, url))
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
strip_css_urls = lambda do |css|
|
|
217
|
+
css.gsub(CSS_URL) do
|
|
218
|
+
whole = Regexp.last_match(0)
|
|
219
|
+
url = Regexp.last_match(2)
|
|
220
|
+
if remote?(url)
|
|
221
|
+
record.call(url, "")
|
|
222
|
+
"url()"
|
|
223
|
+
else
|
|
224
|
+
whole
|
|
225
|
+
end
|
|
226
|
+
end
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# 1) CSS url(...) inside <style> blocks.
|
|
230
|
+
# The CSS is the text just before `</style>`, so it is replaced there
|
|
231
|
+
# by position: searching the block for it would find a copy in the
|
|
232
|
+
# tag's own attributes first and leave the real CSS fetching.
|
|
233
|
+
out = html.gsub(STYLE_BLOCK) do
|
|
234
|
+
block = Regexp.last_match(0)
|
|
235
|
+
css = Regexp.last_match(1)
|
|
236
|
+
finish = block.length - "</style>".length
|
|
237
|
+
block[0, finish - css.length] + strip_css_urls.call(css) + block[finish..]
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
# 2) Every tag: drop remote <img>/<source>, strip remote URL attributes
|
|
241
|
+
# on other elements, and neutralize remote url() in any style
|
|
242
|
+
# attribute.
|
|
243
|
+
out = out.gsub(TAG) do
|
|
244
|
+
tag = Regexp.last_match(0)
|
|
245
|
+
name = Regexp.last_match(1)
|
|
246
|
+
attrs = Regexp.last_match(2)
|
|
247
|
+
rewrite_tag(tag, name, attrs, record, strip_css_urls)
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
StripResult.new(html: out, blocked: blocked)
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
def rewrite_tag(tag, name, attrs, record, strip_css_urls)
|
|
254
|
+
lower = name.downcase
|
|
255
|
+
next_attrs = attrs
|
|
256
|
+
|
|
257
|
+
RESOURCE_ATTRS.fetch(lower, []).each do |attr|
|
|
258
|
+
value = attr_value(next_attrs, attr)
|
|
259
|
+
next if value.nil?
|
|
260
|
+
|
|
261
|
+
urls = attr == "srcset" ? srcset_urls(value) : [value]
|
|
262
|
+
remote = urls.find { |candidate| remote?(candidate) }
|
|
263
|
+
next if remote.nil?
|
|
264
|
+
|
|
265
|
+
record.call(remote, tag)
|
|
266
|
+
# A src-less <img> is pointless, and a permissive renderer cannot
|
|
267
|
+
# re-add a URL that is not there.
|
|
268
|
+
return "" if DROP_WHOLE.include?(lower)
|
|
269
|
+
|
|
270
|
+
next_attrs = remove_attr(next_attrs, attr)
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
style = attr_value(next_attrs, "style")
|
|
274
|
+
if style && !style.empty?
|
|
275
|
+
cleaned = strip_css_urls.call(style)
|
|
276
|
+
next_attrs = set_attr(next_attrs, "style", cleaned) if cleaned != style
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
next_attrs == attrs ? tag : "<#{name}#{next_attrs}>"
|
|
280
|
+
end
|
|
281
|
+
private_class_method :rewrite_tag
|
|
282
|
+
|
|
283
|
+
# Plain-text bodies carry trackers too, as one-pixel-equivalent link opens.
|
|
284
|
+
def find_tracking_links(text)
|
|
285
|
+
out = []
|
|
286
|
+
seen = {}
|
|
287
|
+
utf8(text).scan(LINK) do
|
|
288
|
+
url = Regexp.last_match(0)
|
|
289
|
+
host = host_of(url)
|
|
290
|
+
next if seen.key?(url)
|
|
291
|
+
next unless TRACKER_HINTS.any? { |hint| host.include?(hint) }
|
|
292
|
+
|
|
293
|
+
seen[url] = true
|
|
294
|
+
out << BlockedTracker.new(url: url, host: host, kind: "link")
|
|
295
|
+
end
|
|
296
|
+
out
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
# A one-line summary, as the app shows above a message.
|
|
300
|
+
def summarize_blocked(blocked)
|
|
301
|
+
return "No trackers in this message." if blocked.empty?
|
|
302
|
+
|
|
303
|
+
hosts = blocked.map(&:host).reject(&:empty?).uniq
|
|
304
|
+
noun = blocked.length == 1 ? "tracker" : "trackers"
|
|
305
|
+
case hosts.length
|
|
306
|
+
when 0 then "#{blocked.length} #{noun} blocked."
|
|
307
|
+
when 1 then "#{blocked.length} #{noun} blocked from #{hosts[0]}."
|
|
308
|
+
else "#{blocked.length} #{noun} blocked from #{hosts.length} hosts."
|
|
309
|
+
end
|
|
310
|
+
end
|
|
311
|
+
end
|
|
312
|
+
end
|