mailcycle 0.1.0

data/lib/mailcycle/events.rb

@@ -0,0 +1,310 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Mailcycle
+  # The account event stream, `GET /accounts/live`.
+  #
+  # Receive only. A background thread holds the socket and reconnects with
+  # backoff after a drop, saying so through `on_reconnect`, because events that
+  # happen while it is down are NOT replayed. It stops for good when the server
+  # ends it for a reason reconnecting cannot fix: the sign-in is gone (4001),
+  # the plan does not include the stream (4003), or the upgrade itself was
+  # refused.
+  #
+  #   stream = mc.events.stream
+  #   stream.each { |event| puts event.event_type }
+  #
+  # A refused stream is accepted and then sent an error frame at once, so the
+  # socket coming up proves nothing on its own. The stream counts as open once
+  # an event arrives or a second passes without a refusal, which is what
+  # `wait_until_open` waits for.
+  class EventStream
+    include Enumerable
+
+    PROTOCOL = "mailcycle.v1"
+    PING_SECONDS = 4 * 60
+    OPEN_GRACE_SECONDS = 1.0
+    MAX_RETRY_SECONDS = 60
+    # Closing codes reconnecting cannot fix.
+    FINAL_CODES = [4001, 4003].freeze
+    # The same, for an upgrade the server refused outright.
+    FINAL_STATUSES = [400, 401, 403, 404, 426].freeze
+    # How often the reading loop looks up to notice `close`, the grace and the
+    # ping.
+    TICK = 0.2
+
+    # Opens the stream in the background and returns at once.
+    #
+    # `on_reconnect` is called on the stream's thread each time it comes back
+    # after a drop; events that happen while it is down are not replayed, so
+    # it is where to refetch whatever the program is showing. `on_error` is
+    # called with the API's code and message when the stream is refused.
+    def initialize(base_url, token, on_reconnect: nil, on_error: nil)
+      @url = "#{base_url.to_s.sub('https://', 'wss://').sub('http://', 'ws://').sub(%r{/+\z}, '')}/accounts/live"
+      @token = token
+      @on_reconnect = on_reconnect
+      @on_error = on_error
+
+      @lock = Mutex.new
+      @changed = ConditionVariable.new
+      @events = []
+      @opened = false
+      @closed_reason = nil
+      @stopping = false
+      @socket = nil
+      @thread = Thread.new { run }
+      @thread.name = "mailcycle-events" if @thread.respond_to?(:name=)
+      @thread.report_on_exception = false
+    end
+
+    # The next event, or nil if the stream ends or `timeout` seconds run out.
+    #
+    # Blocking forever (no timeout) is right for a long-lived listener and
+    # wrong for anything with a deadline: a stream that is healthy and silent
+    # would never return. Pass a timeout for the second case.
+    def next_event(timeout: nil)
+      deadline = timeout && (monotonic + timeout)
+      @lock.synchronize do
+        loop do
+          # Anything already queued is handed out before saying it is over.
+          return @events.shift unless @events.empty?
+          return nil if @closed_reason
+
+          remaining = deadline && (deadline - monotonic)
+          return nil if remaining && remaining <= 0
+
+          @changed.wait(@lock, remaining)
+        end
+      end
+    end
+
+    # Every event, as it arrives, until the stream ends.
+    def each
+      return enum_for(:each) unless block_given?
+
+      while (event = next_event)
+        yield event
+      end
+    end
+
+    # Blocks until the stream is open, or it ends, or `timeout` seconds pass.
+    # True only for a stream that is open and was not refused.
+    #
+    # Connecting is retried in the background forever, which is right for a
+    # long-lived listener and wrong for a caller with something else it could
+    # be doing: without this, a stream that never connects looks exactly like
+    # a stream with nothing to say.
+    def wait_until_open(timeout)
+      deadline = monotonic + timeout
+      @lock.synchronize do
+        until @opened
+          remaining = deadline - monotonic
+          return false if remaining <= 0
+
+          @changed.wait(@lock, remaining)
+        end
+        @closed_reason.nil?
+      end
+    end
+
+    # True once the stream has stopped for good and will not reconnect.
+    def ended?
+      @lock.synchronize { !@closed_reason.nil? }
+    end
+
+    # Why it ended, once it has: `[code, reason]`, or nil while it runs.
+    def closed_reason
+      @lock.synchronize { @closed_reason&.dup }
+    end
+
+    # Stops listening and closes the socket. Safe to call twice.
+    def close
+      socket = @lock.synchronize do
+        @stopping = true
+        @changed.broadcast
+        @socket
+      end
+      socket&.close
+      # A connection attempt in flight can take a while to give up; the
+      # thread stops on its own once it does, so this does not wait for it.
+      @thread.join(5) unless Thread.current == @thread
+      finish(1000, "closed")
+      nil
+    end
+
+    private
+
+    def monotonic
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    def stopping?
+      @lock.synchronize { @stopping }
+    end
+
+    def finish(code, reason)
+      @lock.synchronize do
+        @closed_reason ||= [code, reason]
+        # Anything waiting for the socket to come up is waiting for something
+        # that is not going to happen now.
+        @opened = true
+        @changed.broadcast
+      end
+    end
+
+    def mark_open
+      @lock.synchronize do
+        @opened = true
+        @changed.broadcast
+      end
+    end
+
+    def push(event)
+      @lock.synchronize do
+        @events << event
+        @changed.broadcast
+      end
+    end
+
+    def run
+      retry_count = 0
+      ever_opened = false
+
+      until stopping?
+        begin
+          # The sign-in travels as the second subprotocol, which is what the
+          # server reads it from; both are plain HTTP tokens.
+          socket = WebSocket.connect(@url, protocols: [PROTOCOL, @token],
+                                           headers: { "User-Agent" => Http::USER_AGENT })
+        rescue WebSocket::Refused => e
+          # The upgrade itself was refused, which is how the API says the
+          # sign-in is wrong or the plan does not include the stream.
+          # Retrying that is pointless.
+          return finish(1006, e.message) if FINAL_STATUSES.include?(e.status)
+
+          retry_count = backoff(retry_count)
+          next
+        rescue StandardError
+          # A refused connection, a DNS failure, a TLS failure. Worth
+          # retrying, because all three are often temporary.
+          retry_count = backoff(retry_count)
+          next
+        end
+
+        @lock.synchronize { @socket = socket }
+        up = lambda do
+          @on_reconnect&.call if ever_opened
+          ever_opened = true
+          retry_count = 0
+          mark_open
+        end
+        code, reason, refused = pump(socket, up)
+        @lock.synchronize { @socket = nil }
+        socket.close
+
+        return finish(1000, "closed") if stopping?
+        return finish(code, refused) if refused
+        return finish(code, reason.to_s.empty? ? "closed" : reason) if FINAL_CODES.include?(code)
+
+        retry_count = backoff(retry_count)
+      end
+      finish(1000, "closed")
+    ensure
+      # A hook that raised ends the thread; whoever is waiting must hear so.
+      finish(1006, "the stream stopped unexpectedly")
+    end
+
+    # Reads one connection until it ends: `[code, reason, refused]`, where
+    # `refused` is the API's code when it sent an error frame, which
+    # reconnecting cannot fix. `opened` is called once the connection has proved
+    # it was not refused.
+    def pump(socket, opened)
+      grace = monotonic + OPEN_GRACE_SECONDS
+      next_ping = monotonic + PING_SECONDS
+      confirmed = false
+      confirm = lambda do
+        next if confirmed
+
+        confirmed = true
+        opened.call
+      end
+
+      loop do
+        return [1000, "closed", nil] if stopping?
+
+        confirm.call if !confirmed && monotonic >= grace
+        if monotonic >= next_ping
+          socket.send_text("ping")
+          next_ping = monotonic + PING_SECONDS
+        end
+
+        frame = socket.read(timeout: TICK)
+        next if frame.nil?
+        return [frame.code, frame.reason, nil] if frame.type == :close
+        next unless frame.type == :text
+
+        refused = handle_frame(frame.data)
+        if refused
+          # Wait for the close that follows, so `closed_reason` reports the
+          # server's 4001 or 4003 rather than a lost connection.
+          return [closing_code(socket), refused, refused]
+        end
+
+        confirm.call
+      end
+    rescue WebSocket::Broken, IOError, SystemCallError
+      [1006, "connection lost", nil]
+    end
+
+    def closing_code(socket)
+      deadline = monotonic + 2
+      while monotonic < deadline && !stopping?
+        frame = socket.read(timeout: TICK)
+        return frame.code if frame&.type == :close
+      end
+      1006
+    rescue WebSocket::Broken, IOError, SystemCallError
+      1006
+    end
+
+    # Queues an event, or returns the API's code for a refusal.
+    def handle_frame(text)
+      return nil if text == "pong"
+
+      begin
+        frame = JSON.parse(text)
+      rescue JSON::ParserError
+        return nil
+      end
+      return nil unless frame.is_a?(Hash)
+
+      if frame["type"] == "error"
+        code = frame["code"].is_a?(String) ? frame["code"] : "error"
+        message = frame["message"].is_a?(String) ? frame["message"] : "The event stream was refused."
+        @on_error&.call(code, message)
+        return code
+      end
+
+      event = AccountEvent.from_wire(frame)
+      push(event) if event
+      nil
+    end
+
+    # Waits before the next attempt, waking early for `close`. Returns the
+    # next retry count.
+    def backoff(retry_count)
+      seconds = [MAX_RETRY_SECONDS, 2**[retry_count, 6].min].min
+      deadline = monotonic + seconds
+      @lock.synchronize do
+        until @stopping
+          remaining = deadline - monotonic
+          break if remaining <= 0
+
+          @changed.wait(@lock, remaining)
+        end
+      end
+      [retry_count + 1, 8].min
+    end
+  end
+end

data/lib/mailcycle/http.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "openssl"
+require "uri"
+
+module Mailcycle
+  # One HTTP wrapper. Every failure comes out as an ApiError with the API's own
+  # code, so a caller matches on the code rather than on a message.
+  class Http
+    DEFAULT_API_URL = "https://api.mailcycle.email"
+    DEFAULT_TIMEOUT_SECONDS = 30
+    USER_AGENT = "mailcycle-ruby/#{VERSION}".freeze
+
+    # What failing to reach the API looks like from Net::HTTP.
+    NETWORK_ERRORS = [
+      SystemCallError, SocketError, IOError, EOFError, Timeout::Error,
+      OpenSSL::SSL::SSLError, Net::HTTPBadResponse, Net::ProtocolError
+    ].freeze
+
+    attr_reader :base_url, :timeout
+    attr_accessor :token
+
+    def initialize(base_url, token, timeout)
+      @base_url = base_url.to_s.sub(%r{/+\z}, "")
+      @token = token
+      @timeout = timeout
+    end
+
+    # JSON in and out. Returns the parsed answer, or nil for an empty one.
+    def request(method, path, body: nil, query: nil)
+      response = perform(method, path, body, query)
+      text = response.body.to_s
+      return nil if text.strip.empty?
+
+      JSON.parse(text.dup.force_encoding(::Encoding::UTF_8))
+    rescue JSON::ParserError, ::EncodingError
+      raise ApiError.new(0, "bad_response", "The API returned something unexpected.")
+    end
+
+    # Raw bytes back instead of JSON, for an attachment body.
+    def request_bytes(method, path)
+      perform(method, path, nil, nil).body.to_s.b
+    end
+
+    private
+
+    def perform(method, path, body, query)
+      uri = URI.parse("#{@base_url}#{path}")
+      uri.query = URI.encode_www_form(query) if query && !query.empty?
+      request = build(method, uri, body)
+
+      response = begin
+        Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: @timeout,
+                                            read_timeout: @timeout, write_timeout: @timeout) do |http|
+          http.request(request)
+        end
+      rescue *NETWORK_ERRORS => e
+        raise ApiError.new(0, "network_error", "Could not reach #{@base_url}: #{e.message}")
+      end
+      finish(response)
+    end
+
+    def build(method, uri, body)
+      request = Net::HTTPGenericRequest.new(method, method != "GET" && !body.nil?, true, uri)
+      request["Accept"] = "application/json"
+      request["User-Agent"] = USER_AGENT
+      request["Authorization"] = "Bearer #{@token}" if @token
+      if method != "GET" && !body.nil?
+        request["Content-Type"] = "application/json"
+        request.body = JSON.generate(body)
+      end
+      request
+    end
+
+    def finish(response)
+      status = response.code.to_i
+      return response if status.between?(200, 299)
+
+      # Redirects are not followed, so the bearer can never be replayed to
+      # another host. One that arrives anyway is a failure, not an empty body.
+      if status.between?(300, 399)
+        raise ApiError.new(status, "unexpected_redirect", "#{@base_url} redirected the request, which is not followed.")
+      end
+
+      code = "http_error"
+      message = "HTTP #{status}"
+      begin
+        found = JSON.parse(response.body.to_s.dup.force_encoding(::Encoding::UTF_8))
+        if found.is_a?(Hash)
+          code = found["code"] if found["code"].is_a?(String)
+          message = found["message"] if found["message"].is_a?(String)
+        end
+      rescue JSON::ParserError, ::EncodingError
+        # The status alone, then.
+      end
+      raise ApiError.new(status, code, message)
+    end
+  end
+
+  # One path component, escaped.
+  #
+  # Ids come back from the server, and the server is the party this product
+  # distrusts: an id carrying a slash or a query string would otherwise change
+  # which endpoint a call reaches.
+  def self.path_segment(value)
+    value.to_s.b.each_byte.map do |byte|
+      char = byte.chr
+      char.match?(/[A-Za-z0-9\-_.~]/) ? char : format("%%%02X", byte)
+    end.join
+  end
+end

data/lib/mailcycle/keys.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Mailcycle
+  # The account's keys, derived from its recovery phrase on this machine.
+  #
+  # Nothing here leaves the machine. Deriving is 120,000 PBKDF2 iterations for
+  # a 64-byte seed, which OpenSSL does in a few tens of milliseconds. The app
+  # computes the same bytes in its own JavaScript and takes far longer over it;
+  # the iteration count is the format either way, and changing it would change
+  # every account id ever derived.
+  class Keys
+    def self.from_phrase(phrase, passphrase = "")
+      problem = Crypto::Mnemonic.validate(phrase)
+      raise PhraseError, problem unless problem.nil?
+
+      new(Crypto::Identity.identity_from_seed(Crypto::Mnemonic.phrase_to_seed(phrase, passphrase)))
+    end
+
+    def initialize(identity)
+      @identity = identity
+    end
+
+    # Public, safe to log, and meaningless on its own.
+    def account_id
+      @identity.account_id
+    end
+
+    def auth_key
+      @identity.auth_key
+    end
+
+    def auth_public_key
+      @identity.auth_public_key
+    end
+
+    def auth_secret
+      @identity.auth_secret
+    end
+
+    def auth_verifier
+      Crypto::Identity.auth_verifier(@identity.auth_secret)
+    end
+
+    def vault_key
+      @identity.vault_key
+    end
+
+    # The address's public key, base64url, which lets the mail server seal to
+    # it. `epoch` is 0 unless the address's keys were rotated.
+    def address_public_key(address_id, epoch: 0)
+      Encoding.to_b64u(pair_at(address_id, epoch)[1])
+    end
+
+    def address_private_key(address_id, epoch: 0)
+      pair_at(address_id, epoch)[0]
+    end
+
+    # The key the account's own devices seal mail under.
+    def address_symmetric_key(address_id, epoch: 0)
+      key = Crypto::Identity.address_key_at(vault_key, address_id, epoch)
+      raise DecryptionError if key.nil?
+
+      key
+    end
+
+    def inspect
+      # Never the key material, however convenient that would be in a log.
+      "#<Mailcycle::Keys account_id=#{account_id.inspect}>"
+    end
+    alias to_s inspect
+
+    private
+
+    def pair_at(address_id, epoch)
+      pair = Crypto::Identity.address_key_pair_at(vault_key, address_id, epoch)
+      raise DecryptionError if pair.nil?
+
+      pair
+    end
+  end
+end

data/lib/mailcycle/mail.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module Mailcycle
+  # Opening a stored message.
+  #
+  # Two seals reach a client. Mail from outside, and copies of what was sent,
+  # are sealed by the mail server to the address's public key; mail the
+  # account's own devices delivered is sealed under the address's symmetric
+  # key. Either opens here, with keys derived from the phrase.
+  #
+  # Without the keys, or with the wrong ones, a message comes back with
+  # `opened: false` and empty content rather than an error, so one unreadable
+  # message does not hide the rest of a page.
+  module Mail
+    module_function
+
+    def address_of(value)
+      case value
+      when String then value
+      when Hash then Wire.text(value["address"])
+      else ""
+      end
+    end
+
+    def name_of(value)
+      value.is_a?(Hash) ? Wire.text(value["name"]) : ""
+    end
+
+    def attachments_of(value)
+      Wire.array(value).each_with_index.map do |item, index|
+        item = Wire.object(item)
+        size = [item["size"], item["sizeBytes"]].find { |n| n.is_a?(Integer) && n >= 0 } || 0
+        filename = Wire.text(item["filename"])
+        content_id = Wire.text(item["contentId"])
+        Attachment.new(
+          index: index,
+          filename: filename.empty? ? "attachment" : filename,
+          mime_type: Wire.text(item["mimeType"]),
+          size: size,
+          content_id: content_id.empty? ? nil : content_id,
+          # Only an explicit false: mail sealed before the field existed
+          # always had a body.
+          stored: Wire.flag(item["stored"], true)
+        )
+      end
+    end
+
+    # A message's key epoch: absent from servers before epochs, meaning 0.
+    def key_epoch(wire)
+      epoch = wire["keyEpoch"]
+      epoch.is_a?(Integer) && epoch.between?(0, 0xffff_ffff) ? epoch : 0
+    end
+
+    # The content, opened, or nil.
+    def open_content(keys, wire)
+      return nil if keys.nil?
+
+      content = wire["content"]
+      return nil if content.nil?
+
+      address_id = Wire.text(wire["inboxId"])
+      message_id = Wire.text(wire["id"])
+      # The key for the epoch it was sealed under: 0 unless the address's keys
+      # were rotated.
+      epoch = key_epoch(wire)
+
+      # An asymmetric box is the one with an ephemeral public key in it.
+      if content.is_a?(Hash) && content["epk"].is_a?(String)
+        box = Crypto::AsymmetricBox.from_wire(content)
+        return nil if box.nil?
+
+        private_key = keys.address_private_key(address_id, epoch: epoch)
+        # The additional data binds the ciphertext to this address and this
+        # message, exactly as the mail server sealed it.
+        Crypto::Asymmetric.open_json_with_private_key(private_key, box, "#{address_id}:#{message_id}")
+      else
+        box = Crypto::SealedBox.from_wire(content)
+        return nil if box.nil?
+
+        key = keys.address_symmetric_key(address_id, epoch: epoch)
+        Crypto::Cipher.open_json(key, box, message_id)
+      end
+    rescue Error
+      nil
+    end
+
+    def first_text(body, *fields)
+      fields.each do |field|
+        value = Wire.text(body[field])
+        return value unless value.empty?
+      end
+      ""
+    end
+
+    def to_message(keys, wire)
+      wire = Wire.object(wire)
+      content = open_content(keys, wire)
+      body = Wire.object(content)
+
+      html = first_text(body, "html", "bodyHtml")
+      from_address = first_text(body, "fromAddress")
+      from_address = address_of(body["from"]) if from_address.empty?
+      to = body["to"].is_a?(Array) ? address_of(body["to"].first) : address_of(body["to"])
+      stripped = html.empty? ? nil : Trackers.strip_remote_content(html)
+      from_name = first_text(body, "fromName")
+      from_name = name_of(body["from"]) if from_name.empty?
+      reply_to = first_text(body, "replyTo")
+      folder = Wire.text(wire["folder"])
+
+      Message.new(
+        id: Wire.text(wire["id"]),
+        address_id: Wire.text(wire["inboxId"]),
+        received_at: Wire.text(wire["receivedAt"]),
+        read: Wire.flag(wire["read"], false),
+        starred: Wire.flag(wire["starred"], false),
+        folder: folder.empty? ? "inbox" : folder,
+        size_bytes: Wire.count(wire["sizeBytes"]),
+        sent_via: Wire.maybe_text(wire["sentVia"]),
+        key_epoch: key_epoch(wire),
+        opened: !content.nil?,
+        from_address: from_address,
+        from_name: from_name,
+        reply_to: reply_to.empty? ? from_address : reply_to,
+        to: to,
+        subject: Wire.text(body["subject"]),
+        text: first_text(body, "text", "bodyText"),
+        html: html,
+        safe_html: stripped ? stripped.html : "",
+        blocked: stripped ? stripped.blocked : [],
+        message_id: Wire.text(body["messageId"]),
+        attachments: attachments_of(body["attachments"])
+      )
+    end
+
+    # An attachment body, opened with the key for the epoch its message was
+    # sealed under.
+    def open_attachment_bytes(keys, address_id, message_id, index, data, epoch: 0)
+      private_key = keys.address_private_key(address_id, epoch: epoch)
+      Crypto::Asymmetric.open_attachment_with_key(private_key, address_id, "#{address_id}/#{message_id}/#{index}",
+                                                  data)
+    end
+  end
+end