mail_room 0.9.1 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a31789d70291c209682a4b9ce5af4f5fa894cfba
-  data.tar.gz: 56f2f340095b95a616bac7a22a1786df517eb688
+SHA256:
+  metadata.gz: 50ef3164bc5ce5e15087123671dadb214280376dc4e92cc9bdf62ec3c92a6ca1
+  data.tar.gz: 1dd1a45af0fcd31f5b64aac56d6546a7b46eb2d2c0c1cb7e1753fe8d09e57b49
 SHA512:
-  metadata.gz: 67fff02a10229d437653d5e02a015fed44d3a184c7f5cc5742cdb08b0c30043b03027eedf1d6b8c36ffca6350eefb143a5a34548f3f2d8998468722c50d7c2b6
-  data.tar.gz: 696b3b95f5f96fb9a987c75e5cc169c9a346cbf3c9f392777c6fca5691a53045d96514123372c1f58f6767e2734ad442618a7ffc3488864c3b72828ef1c2b3d5
+  metadata.gz: a661b81b53b230a51b89a68d54a9711b8f68b7b9ed0ebafc206db9839894abb42f361efe49014ea325b0b528303b5ed9e78c38be7a301b2df91c6efd22565f5c
+  data.tar.gz: 97f6e54d35ea41c3935db82b57c932d89869c5dd76e8ba2fd92b5024e41bfc79a26dbaa67caffce3952d93ff3144de9ff0a8d10c78f1c52f2043c15282af3d22

data/.ruby-version

@@ -1 +1 @@
-2.2.2
+2.6

data/.travis.yml

@@ -1,8 +1,10 @@
 language: ruby
 rvm:
-  - 2.0.0
-  - 2.1.5
-  - 2.2.4
   - 2.3.0
+  - 2.4
+  - 2.5
+  - 2.6
+services:
+  - redis-server
 script: bundle exec rspec spec
 sudo: false

data/CHANGELOG.md

@@ -1,3 +1,24 @@
+## mail_room 0.10.0 ##
+
+* Remove imap backports
+* Increase minimum ruby version to 2.3
+* Postback basic_auth support - PR#92
+* Docs for ActionMailbox - PR#92
+* Configuration option for delivery_klass - PR#93
+* Expunge deleted - PR#90
+* Raise error on a few fields of missing configuration - PR#89
+* Remove fakeredis gem - PR#87
+
+    *Tony Pitale <@tpitale>*
+
+* Fix redis arbitration to use NX+EX - PR#86
+
+    *Craig Miskell <@craigmiskell-gitlab>*
+
+* Structured (JSON) logger - PR#88
+
+    *charlie <@cablett>*
+
 ## mail_room 0.9.1 ##
 
 * __FILE__ support in yml ERb config - PR#80

data/README.md

@@ -41,6 +41,8 @@ You will also need to install `faraday` or `letter_opener` if you use the `postb
     :password: "password"
     :name: "inbox"
     :search_command: 'NEW'
+    :logger:
+      :log_path: /path/to/logfile/for/mailroom
     :delivery_options:
       :delivery_url: "http://localhost:3000/inbox"
       :delivery_token: "abcdefg"
@@ -66,6 +68,7 @@ You will also need to install `faraday` or `letter_opener` if you use the `postb
     :name: "inbox"
     :delivery_method: letter_opener
     :delete_after_delivery: true
+    :expunge_deleted: true
     :delivery_options:
       :location: "/tmp/user4-email"
   -
@@ -92,6 +95,9 @@ You will also need to install `faraday` or `letter_opener` if you use the `postb
       :worker: EmailReceiverWorker
 ```
 
+**Note:** If using `delete_after_delivery`, you also probably want to use
+`expunge_deleted` unless you really know what you're doing.
+
 ## delivery_method ##
 
 ### postback ###
@@ -181,7 +187,7 @@ end
 
 Configured with `:delivery_method: logger`.
 
-If `:log_path:` is not provided, defaults to `STDOUT`
+If the `:log_path:` delivery option is not provided, defaults to `STDOUT`
 
 ### noop ###
 
@@ -197,6 +203,31 @@ Configured with `:delivery_method: letter_opener`.
 
 Uses Ryan Bates' excellent [letter_opener](https://github.com/ryanb/letter_opener) gem.
 
+## ActionMailbox in Rails ##
+
+MailRoom can deliver mail to Rails using the ActionMailbox [configuration options for an SMTP relay](https://edgeguides.rubyonrails.org/action_mailbox_basics.html#configuration).
+
+In summary (from the ActionMailbox docs)
+
+1. Configure Rails to use the `:relay` ingress option:
+```rb
+# config/environments/production.rb
+config.action_mailbox.ingress = :relay
+```
+
+2. Generate a strong password (e.g., using SecureRandom or something) and add it to Rails config:
+using `rails credentials:edit` under `action_mailbox.ingress_password`.
+
+And finally, configure MailRoom to use the postback configuration with the options:
+
+```yaml
+:delivery_method: postback
+:delivery_options:
+  :delivery_url: https://example.com/rails/action_mailbox/relay/inbound_emails
+  :delivery_username: actionmailbox
+  :delivery_password: <INGRESS_PASSWORD>
+```
+
 ## Receiving `postback` in Rails ##
 
 If you have a controller that you're sending to, with forgery protection
@@ -291,6 +322,12 @@ redis://:<password>@<master-name>/
 You also have to inform at least one pair of `host` and `port` for a sentinel in your cluster.
 To have a minimum reliable setup, you need at least `3` sentinel nodes and `3` redis servers (1 master, 2 slaves).
 
+## Logging ##
+
+MailRoom will output JSON-formatted logs to give some observability into its operations.
+
+Simply configure a `log_path` for the `logger` on any of your mailboxes. By default, nothing will be logged.
+
 ## Contributing ##
 
 1. Fork it

data/lib/mail_room.rb

@@ -6,10 +6,10 @@ module MailRoom
 end
 
 require "mail_room/version"
-require "mail_room/backports/imap"
 require "mail_room/configuration"
 require "mail_room/mailbox"
 require "mail_room/mailbox_watcher"
 require "mail_room/connection"
 require "mail_room/coordinator"
 require "mail_room/cli"
+require 'mail_room/logger/structured'

data/lib/mail_room/arbitration/redis.rb

@@ -22,23 +22,16 @@ module MailRoom
         @options = options
       end
 
-      def deliver?(uid)
+      def deliver?(uid, expiration = EXPIRATION)
         key = "delivered:#{uid}"
 
-        incr = nil
-        client.multi do |c|
-          # At this point, `incr` is a future, which will get its value after
-          # the MULTI command returns.
-          incr = c.incr(key)
-
-          c.expire(key, EXPIRATION)
-        end
-
-        # If INCR returns 1, that means the key didn't exist before, which means
-        # we are the first mail_room to try to deliver this message, so we get to.
-        # If we get any other value, another mail_room already (tried to) deliver
-        # the message, so we don't have to anymore.
-        incr.value == 1
+        # Set the key, but only if it doesn't already exist;
+        # the return value is true if successful, false if the key was already set,
+        # which is conveniently the correct return value for this method
+        # Any subsequent failure in the instance which gets the lock will be dealt
+        # with by the expiration, at which time another instance can pick up the
+        # message and try again.
+        client.set(key, 1, {:nx => true, :ex => expiration})
       end
 
       private

data/lib/mail_room/connection.rb

@@ -50,6 +50,7 @@ module MailRoom
 
         process_mailbox
       rescue Net::IMAP::Error, IOError
+        @mailbox.logger.warn({ context: @mailbox.context, action: "Disconnected. Resetting..." })
         reset
         setup
       end
@@ -64,14 +65,19 @@ module MailRoom
     end
 
     def setup
+      @mailbox.logger.info({ context: @mailbox.context, action: "Starting TLS session" })
       start_tls
+
+      @mailbox.logger.info({ context: @mailbox.context, action: "Logging into mailbox" })
       log_in
+
+      @mailbox.logger.info({ context: @mailbox.context, action: "Setting mailbox" })
       set_mailbox
     end
 
     # build a net/imap connection to google imap
     def imap
-      @imap ||= MailRoom::IMAP.new(@mailbox.host, :port => @mailbox.port, :ssl => @mailbox.ssl_options)
+      @imap ||= Net::IMAP.new(@mailbox.host, :port => @mailbox.port, :ssl => @mailbox.ssl_options)
     end
 
     # start a TLS session
@@ -106,6 +112,7 @@ module MailRoom
     def idle
       return unless ready_to_idle?
 
+      @mailbox.logger.info({ context: @mailbox.context, action: "Idling" })
       @idling = true
 
       imap.idle(@mailbox.idle_timeout, &idle_handler)
@@ -118,26 +125,35 @@ module MailRoom
       return unless idling?
 
       imap.idle_done
-      
+
       # idling_thread.join
       # self.idling_thread = nil
     end
 
     def process_mailbox
       return unless @new_message_handler
+      @mailbox.logger.info({ context: @mailbox.context, action: "Processing started" })
 
       msgs = new_messages
 
-      msgs.
-        map(&@new_message_handler). # deliver each new message, collect success
-        zip(msgs). # include messages with success
-        select(&:first).map(&:last). # filter failed deliveries, collect message
-        each {|message| scrub(message)} # scrub delivered messages
+      any_deletions = msgs.
+                      # deliver each new message, collect success
+                      map(&@new_message_handler).
+                      # include messages with success
+                      zip(msgs).
+                      # filter failed deliveries, collect message
+                      select(&:first).map(&:last).
+                      # scrub delivered messages
+                      map { |message| scrub(message) }.
+                      any?
+
+      imap.expunge if @mailbox.expunge_deleted && any_deletions
     end
 
     def scrub(message)
       if @mailbox.delete_after_delivery
         imap.store(message.seqno, "+FLAGS", [Net::IMAP::DELETED])
+        true
       end
     end
 
@@ -158,7 +174,11 @@ module MailRoom
     # @return [Array<Integer>] message ids
     def new_message_ids
       # uid_search still leaves messages UNSEEN
-      imap.uid_search(@mailbox.search_command).select { |uid| @mailbox.deliver?(uid) }
+      all_unread = @imap.uid_search(@mailbox.search_command)
+
+      to_deliver = all_unread.select { |uid| @mailbox.deliver?(uid) }
+      @mailbox.logger.info({ context: @mailbox.context, action: "Getting new messages", unread: {count: all_unread.count, ids: all_unread}, to_be_delivered: { count: to_deliver.count, ids: all_unread } })
+      to_deliver
     end
 
     # @private

data/lib/mail_room/delivery/postback.rb

@@ -5,18 +5,39 @@ module MailRoom
     # Postback Delivery method
     # @author Tony Pitale
     class Postback
-      Options = Struct.new(:delivery_url, :delivery_token) do
+      Options = Struct.new(:url, :token, :username, :password, :logger) do
         def initialize(mailbox)
-          delivery_url = mailbox.delivery_url || mailbox.delivery_options[:delivery_url]
-          delivery_token = mailbox.delivery_token || mailbox.delivery_options[:delivery_token]
+          url =
+            mailbox.delivery_url ||
+            mailbox.delivery_options[:delivery_url] ||
+            mailbox.delivery_options[:url]
 
-          super(delivery_url, delivery_token)
+          token =
+            mailbox.delivery_token ||
+            mailbox.delivery_options[:delivery_token] ||
+            mailbox.delivery_options[:token]
+
+          username = mailbox.delivery_options[:username]
+          password = mailbox.delivery_options[:password]
+
+          logger = mailbox.logger
+
+          super(url, token, username, password, logger)
+        end
+
+        def token_auth?
+          !self[:token].nil?
+        end
+
+        def basic_auth?
+          !self[:username].nil? && !self[:password].nil?
         end
       end
 
       # Build a new delivery, hold the delivery options
       # @param [MailRoom::Delivery::Postback::Options]
       def initialize(delivery_options)
+        puts delivery_options
         @delivery_options = delivery_options
       end
 
@@ -24,15 +45,24 @@ module MailRoom
       # @param message [String] the email message as a string, RFC822 format
       def deliver(message)
         connection = Faraday.new
-        connection.token_auth @delivery_options.delivery_token
+
+        if @delivery_options.token_auth?
+          connection.token_auth @delivery_options.token
+        elsif @delivery_options.basic_auth?
+          connection.basic_auth(
+            @delivery_options.username,
+            @delivery_options.password
+          )
+        end
 
         connection.post do |request|
-          request.url @delivery_options.delivery_url
+          request.url @delivery_options.url
           request.body = message
           # request.options[:timeout] = 3
           # request.headers['Content-Type'] = 'text/plain'
         end
 
+        @delivery_options.logger.info({ delivery_method: 'Postback', action: 'message pushed', url: @delivery_options.url })
         true
       end
     end

data/lib/mail_room/delivery/que.rb

@@ -6,7 +6,7 @@ module MailRoom
     # Que Delivery method
     # @author Tony Pitale
     class Que
-      Options = Struct.new(:host, :port, :database, :username, :password, :queue, :priority, :job_class) do
+      Options = Struct.new(:host, :port, :database, :username, :password, :queue, :priority, :job_class, :logger) do
         def initialize(mailbox)
           host = mailbox.delivery_options[:host] || "localhost"
           port = mailbox.delivery_options[:port] || 5432
@@ -17,8 +17,9 @@ module MailRoom
           queue = mailbox.delivery_options[:queue] || ''
           priority = mailbox.delivery_options[:priority] || 100 # lowest priority for Que
           job_class = mailbox.delivery_options[:job_class]
+          logger = mailbox.logger
 
-          super(host, port, database, username, password, queue, priority, job_class)
+          super(host, port, database, username, password, queue, priority, job_class, logger)
         end
       end
 
@@ -34,6 +35,7 @@ module MailRoom
       # @param message [String] the email message as a string, RFC822 format
       def deliver(message)
         queue_job(message)
+        @options.logger.info({ delivery_method: 'Que', action: 'message pushed' })
       end
 
       private

data/lib/mail_room/delivery/sidekiq.rb

@@ -8,15 +8,16 @@ module MailRoom
     # Sidekiq Delivery method
     # @author Douwe Maan
     class Sidekiq
-      Options = Struct.new(:redis_url, :namespace, :sentinels, :queue, :worker) do
+      Options = Struct.new(:redis_url, :namespace, :sentinels, :queue, :worker, :logger) do
         def initialize(mailbox)
           redis_url = mailbox.delivery_options[:redis_url] || "redis://localhost:6379"
           namespace = mailbox.delivery_options[:namespace]
           sentinels = mailbox.delivery_options[:sentinels]
           queue     = mailbox.delivery_options[:queue] || "default"
           worker    = mailbox.delivery_options[:worker]
+          logger = mailbox.logger
 
-          super(redis_url, namespace, sentinels, queue, worker)
+          super(redis_url, namespace, sentinels, queue, worker, logger)
         end
       end
 
@@ -35,6 +36,7 @@ module MailRoom
 
         client.lpush("queue:#{options.queue}", JSON.generate(item))
 
+        @options.logger.info({ delivery_method: 'Sidekiq', action: 'message pushed' })
         true
       end
 
@@ -62,7 +64,6 @@ module MailRoom
         {
           'class'       => options.worker,
           'args'        => [utf8_encode_message(message)],
-
           'queue'       => options.queue,
           'jid'         => SecureRandom.hex(12),
           'retry'       => false,

data/lib/mail_room/logger/structured.rb

@@ -0,0 +1,21 @@
+require 'logger'
+require 'json'
+
+module MailRoom
+  module Logger
+    class Structured < ::Logger
+
+      def format_message(severity, timestamp, progname, message)
+        raise ArgumentError.new("Message must be a Hash") unless message.is_a? Hash
+
+        data = {}
+        data[:severity] = severity
+        data[:time] = timestamp || Time.now.to_s
+        # only accept a Hash
+        data.merge!(message)
+
+        data.to_json + "\n"
+      end
+    end
+  end
+end

data/lib/mail_room/mailbox.rb

@@ -14,6 +14,8 @@ module MailRoom
     :search_command,
     :name,
     :delete_after_delivery,
+    :expunge_deleted,
+    :delivery_klass,
     :delivery_method, # :noop, :logger, :postback, :letter_opener
     :log_path, # for logger
     :delivery_url, # for postback
@@ -21,9 +23,11 @@ module MailRoom
     :location, # for letter_opener
     :delivery_options,
     :arbitration_method,
-    :arbitration_options
+    :arbitration_options,
+    :logger
   ]
 
+  ConfigurationError = Class.new(RuntimeError)
   IdleTimeoutTooLarge = Class.new(RuntimeError)
 
   # Holds configuration for each of the email accounts we wish to monitor
@@ -35,6 +39,8 @@ module MailRoom
     # 29 minutes, as suggested by the spec: https://tools.ietf.org/html/rfc2177
     IMAP_IDLE_TIMEOUT = 29 * 60 # 29 minutes in in seconds
 
+    REQUIRED_CONFIGURATION = [:name, :email, :password, :host, :port]
+
     # Default attributes for the mailbox configuration
     DEFAULTS = {
       :search_command => 'UNSEEN',
@@ -45,9 +51,11 @@ module MailRoom
       :start_tls => false,
       :idle_timeout => IMAP_IDLE_TIMEOUT,
       :delete_after_delivery => false,
+      :expunge_deleted => false,
       :delivery_options => {},
       :arbitration_method => 'noop',
-      :arbitration_options => {}
+      :arbitration_options => {},
+      :logger => {}
     }
 
     # Store the configuration and require the appropriate delivery method
@@ -58,8 +66,19 @@ module MailRoom
       validate!
     end
 
+    def logger
+      @logger ||=
+        case self[:logger]
+          when Logger
+            self[:logger]
+          else
+            self[:logger] ||= {}
+            MailRoom::Logger::Structured.new(self[:logger][:log_path])
+        end
+    end
+
     def delivery_klass
-      Delivery[delivery_method]
+      self[:delivery_klass] ||= Delivery[delivery_method]
     end
 
     def arbitration_klass
@@ -75,6 +94,8 @@ module MailRoom
     end
 
     def deliver?(uid)
+      logger.info({context: context, uid: uid, action: "asking arbiter to deliver", arbitrator: arbitrator.class.name})
+
       arbitrator.deliver?(uid)
     end
 
@@ -84,6 +105,7 @@ module MailRoom
       body = message.attr['RFC822']
       return true unless body
 
+      logger.info({context: context, uid: message.attr['UID'], action: "sending to deliverer", deliverer: delivery.class.name, byte_size: message.attr['RFC822.SIZE']})
       delivery.deliver(body)
     end
 
@@ -92,9 +114,22 @@ module MailRoom
       replace_verify_mode(ssl)
     end
 
+    def context
+      { email: self.email, name: self.name }
+    end
+
     def validate!
       if self[:idle_timeout] > IMAP_IDLE_TIMEOUT
-        raise IdleTimeoutTooLarge.new("Please use an idle timeout smaller than #{29*60} to prevent IMAP server disconnects")
+        raise IdleTimeoutTooLarge,
+              "Please use an idle timeout smaller than #{29*60} to prevent " \
+              "IMAP server disconnects"
+      end
+
+      REQUIRED_CONFIGURATION.each do |k|
+        if self[k].nil?
+          raise ConfigurationError,
+                "Field :#{k} is required in Mailbox: #{inspect}"
+        end
       end
     end
 
@@ -112,18 +147,22 @@ module MailRoom
       return options unless options.is_a?(Hash)
       return options unless options.has_key?(:verify_mode)
 
-      options[:verify_mode] = case options[:verify_mode]
-      when :none, 'none'
-        OpenSSL::SSL::VERIFY_NONE
-      when :peer, 'peer'
-        OpenSSL::SSL::VERIFY_PEER
-      when :client_once, 'client_once'
-        OpenSSL::SSL::VERIFY_CLIENT_ONCE
-      when :fail_if_no_peer_cert, 'fail_if_no_peer_cert'
-        OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
-      end
+      options[:verify_mode] = lookup_verify_mode(options[:verify_mode])
 
       options
     end
+
+    def lookup_verify_mode(verify_mode)
+      case verify_mode.to_sym
+        when :none
+          OpenSSL::SSL::VERIFY_NONE
+        when :peer
+          OpenSSL::SSL::VERIFY_PEER
+        when :client_once
+          OpenSSL::SSL::VERIFY_CLIENT_ONCE
+        when :fail_if_no_peer_cert
+          OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
+      end
+    end
   end
 end