mail_room 0.10.0 → 0.11.0

data/lib/mail_room/delivery/que.rb

@@ -1,5 +1,6 @@
 require 'pg'
 require 'json'
+require 'charlock_holmes'
 
 module MailRoom
   module Delivery
@@ -34,7 +35,7 @@ module MailRoom
       # deliver the message by pushing it onto the configured Sidekiq queue
       # @param message [String] the email message as a string, RFC822 format
       def deliver(message)
-        queue_job(message)
+        queue_job(utf8_encode_message(message))
         @options.logger.info({ delivery_method: 'Que', action: 'message pushed' })
       end
 
@@ -58,6 +59,19 @@ module MailRoom
 
         connection.exec(sql, [options.priority, options.job_class, options.queue, JSON.dump(args)])
       end
+
+      def utf8_encode_message(message)
+        message = message.dup
+
+        message.force_encoding("UTF-8")
+        return message if message.valid_encoding?
+
+        detection = CharlockHolmes::EncodingDetector.detect(message)
+        return message unless detection && detection[:encoding]
+
+        # Convert non-UTF-8 body UTF-8 so it can be dumped as JSON.
+        CharlockHolmes::Converter.convert(message, detection[:encoding], 'UTF-8')
+      end
     end
   end
 end

data/lib/mail_room/delivery/sidekiq.rb

@@ -8,16 +8,24 @@ module MailRoom
     # Sidekiq Delivery method
     # @author Douwe Maan
     class Sidekiq
-      Options = Struct.new(:redis_url, :namespace, :sentinels, :queue, :worker, :logger) do
+      Options = Struct.new(:redis_url, :namespace, :sentinels, :queue, :worker, :logger, :redis_db) do
         def initialize(mailbox)
           redis_url = mailbox.delivery_options[:redis_url] || "redis://localhost:6379"
+          redis_db  = mailbox.delivery_options[:redis_db] || 0
           namespace = mailbox.delivery_options[:namespace]
           sentinels = mailbox.delivery_options[:sentinels]
           queue     = mailbox.delivery_options[:queue] || "default"
           worker    = mailbox.delivery_options[:worker]
           logger = mailbox.logger
 
-          super(redis_url, namespace, sentinels, queue, worker, logger)
+          if namespace
+            warn <<~MSG
+              Redis namespaces are deprecated. This option will be ignored in future versions.
+              See https://github.com/sidekiq/sidekiq/issues/2586 for more details."
+            MSG
+          end
+
+          super(redis_url, namespace, sentinels, queue, worker, logger, redis_db)
         end
       end
 
@@ -45,7 +53,7 @@ module MailRoom
       def client
         @client ||= begin
           sentinels = options.sentinels
-          redis_options = { url: options.redis_url }
+          redis_options = { url: options.redis_url, db: options.redis_db }
           redis_options[:sentinels] = sentinels if sentinels
 
           redis = ::Redis.new(redis_options)

data/lib/mail_room/imap/connection.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+module MailRoom
+  module IMAP
+    class Connection < MailRoom::Connection
+      def initialize(mailbox)
+        super
+
+        # log in and set the mailbox
+        reset
+        setup
+      end
+
+      # is the connection logged in?
+      # @return [Boolean]
+      def logged_in?
+        @logged_in
+      end
+
+      # is the connection blocked idling?
+      # @return [Boolean]
+      def idling?
+        @idling
+      end
+
+      # is the imap connection closed?
+      # @return [Boolean]
+      def disconnected?
+        imap.disconnected?
+      end
+
+      # is the connection ready to idle?
+      # @return [Boolean]
+      def ready_to_idle?
+        logged_in? && !idling?
+      end
+
+      def quit
+        stop_idling
+        reset
+      end
+
+      def wait
+        # in case we missed any between idles
+        process_mailbox
+
+        idle
+
+        process_mailbox
+      rescue Net::IMAP::Error, IOError => e
+        @mailbox.logger.warn({ context: @mailbox.context, action: 'Disconnected. Resetting...', error: e.message })
+        reset
+        setup
+      end
+
+      private
+
+      def reset
+        @imap = nil
+        @logged_in = false
+        @idling = false
+      end
+
+      def setup
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Starting TLS session' })
+        start_tls
+
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Logging into mailbox' })
+        log_in
+
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Setting mailbox' })
+        set_mailbox
+      end
+
+      # build a net/imap connection to google imap
+      def imap
+        @imap ||= Net::IMAP.new(@mailbox.host, port: @mailbox.port, ssl: @mailbox.ssl_options)
+      end
+
+      # start a TLS session
+      def start_tls
+        imap.starttls if @mailbox.start_tls
+      end
+
+      # send the imap login command to google
+      def log_in
+        imap.login(@mailbox.email, @mailbox.password)
+        @logged_in = true
+      end
+
+      # select the mailbox name we want to use
+      def set_mailbox
+        imap.select(@mailbox.name) if logged_in?
+      end
+
+      # is the response for a new message?
+      # @param response [Net::IMAP::TaggedResponse] the imap response from idle
+      # @return [Boolean]
+      def message_exists?(response)
+        response.respond_to?(:name) && response.name == 'EXISTS'
+      end
+
+      # @private
+      def idle_handler
+        ->(response) { imap.idle_done if message_exists?(response) }
+      end
+
+      # maintain an imap idle connection
+      def idle
+        return unless ready_to_idle?
+
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Idling' })
+        @idling = true
+
+        imap.idle(@mailbox.idle_timeout, &idle_handler)
+      ensure
+        @idling = false
+      end
+
+      # trigger the idle to finish and wait for the thread to finish
+      def stop_idling
+        return unless idling?
+
+        imap.idle_done
+
+        # idling_thread.join
+        # self.idling_thread = nil
+      end
+
+      def process_mailbox
+        return unless @new_message_handler
+
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Processing started' })
+
+        msgs = new_messages
+        any_deletions = msgs.
+                        # deliver each new message, collect success
+                        map(&@new_message_handler).
+                        # include messages with success
+                        zip(msgs).
+                        # filter failed deliveries, collect message
+                        select(&:first).map(&:last).
+                        # scrub delivered messages
+                        map { |message| scrub(message) }
+                            .any?
+
+        imap.expunge if @mailbox.expunge_deleted && any_deletions
+      end
+
+      def scrub(message)
+        if @mailbox.delete_after_delivery
+          imap.store(message.seqno, '+FLAGS', [Net::IMAP::DELETED])
+          true
+        end
+      end
+
+      # @private
+      # fetch all messages for the new message ids
+      def new_messages
+        # Both of these calls may results in
+        #   imap raising an EOFError, we handle
+        #   this exception in the watcher
+        messages_for_ids(new_message_ids)
+      end
+
+      # TODO: label messages?
+      #   @imap.store(id, "+X-GM-LABELS", [label])
+
+      # @private
+      # search for all new (unseen) message ids
+      # @return [Array<Integer>] message ids
+      def new_message_ids
+        # uid_search still leaves messages UNSEEN
+        all_unread = imap.uid_search(@mailbox.search_command)
+
+        all_unread = all_unread.slice(0, @mailbox.limit_max_unread) if @mailbox.limit_max_unread.to_i > 0
+
+        to_deliver = all_unread.select { |uid| @mailbox.deliver?(uid) }
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Getting new messages',
+                               unread: { count: all_unread.count, ids: all_unread }, to_be_delivered: { count: to_deliver.count, ids: to_deliver } })
+        to_deliver
+      end
+
+      # @private
+      # fetch the email for all given ids in RFC822 format
+      # @param ids [Array<Integer>] list of message ids
+      # @return [Array<MailRoom::IMAP::Message>] the net/imap messages for the given ids
+      def messages_for_ids(uids)
+        return [] if uids.empty?
+
+        # uid_fetch marks as SEEN, will not be re-fetched for UNSEEN
+        imap_messages = imap.uid_fetch(uids, 'RFC822')
+
+        imap_messages.each_with_object([]) do |msg, messages|
+          messages << ::MailRoom::IMAP::Message.new(uid: msg.attr['UID'], body: msg.attr['RFC822'], seqno: msg.seqno)
+        end
+      end
+    end
+  end
+end

data/lib/mail_room/imap/message.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal:true
+
+module MailRoom
+  module IMAP
+    class Message < MailRoom::Message
+      attr_reader :seqno
+
+      def initialize(uid:, body:, seqno:)
+        super(uid: uid, body: body)
+
+        @seqno = seqno
+      end
+
+      def ==(other)
+        super && seqno == other.seqno
+      end
+    end
+  end
+end

data/lib/mail_room/imap.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+ 
+module MailRoom
+  module IMAP
+    autoload :Connection, 'mail_room/imap/connection'
+    autoload :Message, 'mail_room/imap/message'
+  end
+end

data/lib/mail_room/jwt.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'faraday'
+require 'securerandom'
+require 'jwt'
+require 'base64'
+
+module MailRoom
+  # Responsible for validating and generating JWT token
+  class JWT
+    DEFAULT_ISSUER = 'mailroom'
+    DEFAULT_ALGORITHM = 'HS256'
+
+    attr_reader :header, :secret_path, :issuer, :algorithm
+
+    def initialize(header:, secret_path:, issuer:, algorithm:)
+      @header = header
+      @secret_path = secret_path
+      @issuer = issuer || DEFAULT_ISSUER
+      @algorithm = algorithm || DEFAULT_ALGORITHM
+    end
+
+    def valid?
+      [@header, @secret_path, @issuer, @algorithm].none?(&:nil?)
+    end
+
+    def token
+      return nil unless valid?
+
+      secret = Base64.strict_decode64(File.read(@secret_path).chomp)
+      payload = {
+        nonce: SecureRandom.hex(12),
+        iat: Time.now.to_i, # https://github.com/jwt/ruby-jwt#issued-at-claim
+        iss: @issuer
+      }
+      ::JWT.encode payload, secret, @algorithm
+    end
+  end
+end

data/lib/mail_room/logger/structured.rb

@@ -1,3 +1,4 @@
+require 'date'
 require 'logger'
 require 'json'
 
@@ -10,12 +11,25 @@ module MailRoom
 
         data = {}
         data[:severity] = severity
-        data[:time] = timestamp || Time.now.to_s
+        data[:time] = format_timestamp(timestamp || Time.now)
         # only accept a Hash
         data.merge!(message)
 
         data.to_json + "\n"
       end
+
+      private
+
+      def format_timestamp(timestamp)
+        case timestamp
+        when Time
+          timestamp.to_datetime.iso8601(3).to_s
+        when DateTime
+          timestamp.iso8601(3).to_s
+        else
+          timestamp
+        end
+      end
     end
   end
 end

data/lib/mail_room/mailbox.rb

@@ -1,15 +1,20 @@
 require "mail_room/delivery"
 require "mail_room/arbitration"
+require "mail_room/imap"
+require "mail_room/microsoft_graph"
 
 module MailRoom
   # Mailbox Configuration fields
   MAILBOX_FIELDS = [
     :email,
+    :inbox_method,
+    :inbox_options,
     :password,
     :host,
     :port,
     :ssl,
     :start_tls,
+    :limit_max_unread, #to avoid 'Error in IMAP command UID FETCH: Too long argument'
     :idle_timeout,
     :search_command,
     :name,
@@ -20,6 +25,7 @@ module MailRoom
     :log_path, # for logger
     :delivery_url, # for postback
     :delivery_token, # for postback
+    :content_type, # for postback
     :location, # for letter_opener
     :delivery_options,
     :arbitration_method,
@@ -39,23 +45,26 @@ module MailRoom
     # 29 minutes, as suggested by the spec: https://tools.ietf.org/html/rfc2177
     IMAP_IDLE_TIMEOUT = 29 * 60 # 29 minutes in in seconds
 
-    REQUIRED_CONFIGURATION = [:name, :email, :password, :host, :port]
+    IMAP_CONFIGURATION = [:name, :email, :password, :host, :port].freeze
+    MICROSOFT_GRAPH_CONFIGURATION = [:name, :email].freeze
+    MICROSOFT_GRAPH_INBOX_OPTIONS = [:tenant_id, :client_id, :client_secret].freeze
 
     # Default attributes for the mailbox configuration
     DEFAULTS = {
-      :search_command => 'UNSEEN',
-      :delivery_method => 'postback',
-      :host => 'imap.gmail.com',
-      :port => 993,
-      :ssl => true,
-      :start_tls => false,
-      :idle_timeout => IMAP_IDLE_TIMEOUT,
-      :delete_after_delivery => false,
-      :expunge_deleted => false,
-      :delivery_options => {},
-      :arbitration_method => 'noop',
-      :arbitration_options => {},
-      :logger => {}
+      search_command: 'UNSEEN',
+      delivery_method: 'postback',
+      host: 'imap.gmail.com',
+      port: 993,
+      ssl: true,
+      start_tls: false,
+      limit_max_unread: 0,
+      idle_timeout: IMAP_IDLE_TIMEOUT,
+      delete_after_delivery: false,
+      expunge_deleted: false,
+      delivery_options: {},
+      arbitration_method: 'noop',
+      arbitration_options: {},
+      logger: {}
     }
 
     # Store the configuration and require the appropriate delivery method
@@ -73,7 +82,7 @@ module MailRoom
             self[:logger]
           else
             self[:logger] ||= {}
-            MailRoom::Logger::Structured.new(self[:logger][:log_path])
+            MailRoom::Logger::Structured.new(normalize_log_path(self[:logger][:log_path]))
         end
     end
 
@@ -99,13 +108,13 @@ module MailRoom
       arbitrator.deliver?(uid)
     end
 
-    # deliver the imap email message
-    # @param message [Net::IMAP::FetchData]
+    # deliver the email message
+    # @param message [MailRoom::Message]
     def deliver(message)
-      body = message.attr['RFC822']
+      body = message.body
       return true unless body
 
-      logger.info({context: context, uid: message.attr['UID'], action: "sending to deliverer", deliverer: delivery.class.name, byte_size: message.attr['RFC822.SIZE']})
+      logger.info({context: context, uid: message.uid, action: "sending to deliverer", deliverer: delivery.class.name, byte_size: body.bytesize})
       delivery.deliver(body)
     end
 
@@ -118,14 +127,32 @@ module MailRoom
       { email: self.email, name: self.name }
     end
 
+    def imap?
+      !microsoft_graph?
+    end
+
+    def microsoft_graph?
+      self[:inbox_method].to_s == 'microsoft_graph'
+    end
+
     def validate!
+      if microsoft_graph?
+        validate_microsoft_graph!
+      else
+        validate_imap!
+      end
+    end
+
+    private
+
+    def validate_imap!
       if self[:idle_timeout] > IMAP_IDLE_TIMEOUT
         raise IdleTimeoutTooLarge,
               "Please use an idle timeout smaller than #{29*60} to prevent " \
               "IMAP server disconnects"
       end
 
-      REQUIRED_CONFIGURATION.each do |k|
+      IMAP_CONFIGURATION.each do |k|
         if self[k].nil?
           raise ConfigurationError,
                 "Field :#{k} is required in Mailbox: #{inspect}"
@@ -133,7 +160,23 @@ module MailRoom
       end
     end
 
-    private
+    def validate_microsoft_graph!
+      raise ConfigurationError, "Missing inbox_options in Mailbox: #{inspect}" unless self.inbox_options.is_a?(Hash)
+
+      MICROSOFT_GRAPH_CONFIGURATION.each do |k|
+        if self[k].nil?
+          raise ConfigurationError,
+                "Field :#{k} is required in Mailbox: #{inspect}"
+        end
+      end
+
+      MICROSOFT_GRAPH_INBOX_OPTIONS.each do |k|
+        if self[:inbox_options][k].nil?
+          raise ConfigurationError,
+                "inbox_options field :#{k} is required in Mailbox: #{inspect}"
+        end
+      end
+    end
 
     def parsed_arbitration_options
       arbitration_klass::Options.new(self)
@@ -164,5 +207,18 @@ module MailRoom
           OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
       end
     end
+
+    def normalize_log_path(log_path)
+      case log_path
+      when nil, ""
+        nil
+      when :stdout, "STDOUT"
+        STDOUT
+      when :stderr, "STDERR"
+        STDERR
+      else
+        log_path
+      end
+    end
   end
 end

data/lib/mail_room/mailbox_watcher.rb

@@ -1,3 +1,5 @@
+require "mail_room/connection"
+
 module MailRoom
   # TODO: split up between processing and idling?
 
@@ -55,8 +57,14 @@ module MailRoom
     end
 
     private
+
     def connection
-      @connection ||= Connection.new(@mailbox)
+      @connection ||=
+        if @mailbox.microsoft_graph?
+          ::MailRoom::MicrosoftGraph::Connection.new(@mailbox)
+        else
+          ::MailRoom::IMAP::Connection.new(@mailbox)
+        end
     end
   end
 end

data/lib/mail_room/message.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module MailRoom
+  class Message
+    attr_reader :uid, :body
+
+    def initialize(uid:, body:)
+      @uid = uid
+      @body = body
+    end
+
+    def ==(other)
+      self.class == other.class && uid == other.uid && body == other.body
+    end
+  end
+end