gitlab-mail_room 0.0.10 → 0.0.14

data/lib/mail_room/arbitration/redis.rb

@@ -31,7 +31,7 @@ module MailRoom
         # Any subsequent failure in the instance which gets the lock will be dealt
         # with by the expiration, at which time another instance can pick up the
         # message and try again.
-        client.set(key, 1, {:nx => true, :ex => expiration})
+        client.set(key, 1, {nx: true, ex: expiration})
       end
 
       private

data/lib/mail_room/connection.rb

@@ -16,8 +16,6 @@ module MailRoom
       raise NotImplementedError
     end
 
-    def quit
-      raise NotImplementedError
-    end
+    def quit; end
   end
 end

data/lib/mail_room/delivery/letter_opener.rb

@@ -24,7 +24,7 @@ module MailRoom
       # Trigger `LetterOpener` to deliver our message
       # @param message [String] the email message as a string, RFC822 format
       def deliver(message)
-        method = ::LetterOpener::DeliveryMethod.new(:location => @delivery_options.location)
+        method = ::LetterOpener::DeliveryMethod.new(location: @delivery_options.location)
         method.deliver!(Mail.read_from_string(message))
 
         true

data/lib/mail_room/delivery/sidekiq.rb

@@ -8,16 +8,17 @@ module MailRoom
     # Sidekiq Delivery method
     # @author Douwe Maan
     class Sidekiq
-      Options = Struct.new(:redis_url, :namespace, :sentinels, :queue, :worker, :logger) do
+      Options = Struct.new(:redis_url, :namespace, :sentinels, :queue, :worker, :logger, :redis_db) do
         def initialize(mailbox)
           redis_url = mailbox.delivery_options[:redis_url] || "redis://localhost:6379"
+          redis_db  = mailbox.delivery_options[:redis_db] || 0
           namespace = mailbox.delivery_options[:namespace]
           sentinels = mailbox.delivery_options[:sentinels]
           queue     = mailbox.delivery_options[:queue] || "default"
           worker    = mailbox.delivery_options[:worker]
           logger = mailbox.logger
 
-          super(redis_url, namespace, sentinels, queue, worker, logger)
+          super(redis_url, namespace, sentinels, queue, worker, logger, redis_db)
         end
       end
 
@@ -45,7 +46,7 @@ module MailRoom
       def client
         @client ||= begin
           sentinels = options.sentinels
-          redis_options = { url: options.redis_url }
+          redis_options = { url: options.redis_url, db: options.redis_db }
           redis_options[:sentinels] = sentinels if sentinels
 
           redis = ::Redis.new(redis_options)

data/lib/mail_room/mailbox.rb

@@ -1,11 +1,14 @@
 require "mail_room/delivery"
 require "mail_room/arbitration"
 require "mail_room/imap"
+require "mail_room/microsoft_graph"
 
 module MailRoom
   # Mailbox Configuration fields
   MAILBOX_FIELDS = [
     :email,
+    :inbox_method,
+    :inbox_options,
     :password,
     :host,
     :port,
@@ -42,24 +45,26 @@ module MailRoom
     # 29 minutes, as suggested by the spec: https://tools.ietf.org/html/rfc2177
     IMAP_IDLE_TIMEOUT = 29 * 60 # 29 minutes in in seconds
 
-    REQUIRED_CONFIGURATION = [:name, :email, :password, :host, :port]
+    IMAP_CONFIGURATION = [:name, :email, :password, :host, :port].freeze
+    MICROSOFT_GRAPH_CONFIGURATION = [:name, :email].freeze
+    MICROSOFT_GRAPH_INBOX_OPTIONS = [:tenant_id, :client_id, :client_secret].freeze
 
     # Default attributes for the mailbox configuration
     DEFAULTS = {
-      :search_command => 'UNSEEN',
-      :delivery_method => 'postback',
-      :host => 'imap.gmail.com',
-      :port => 993,
-      :ssl => true,
-      :start_tls => false,
-      :limit_max_unread => 0,
-      :idle_timeout => IMAP_IDLE_TIMEOUT,
-      :delete_after_delivery => false,
-      :expunge_deleted => false,
-      :delivery_options => {},
-      :arbitration_method => 'noop',
-      :arbitration_options => {},
-      :logger => {}
+      search_command: 'UNSEEN',
+      delivery_method: 'postback',
+      host: 'imap.gmail.com',
+      port: 993,
+      ssl: true,
+      start_tls: false,
+      limit_max_unread: 0,
+      idle_timeout: IMAP_IDLE_TIMEOUT,
+      delete_after_delivery: false,
+      expunge_deleted: false,
+      delivery_options: {},
+      arbitration_method: 'noop',
+      arbitration_options: {},
+      logger: {}
     }
 
     # Store the configuration and require the appropriate delivery method
@@ -122,14 +127,32 @@ module MailRoom
       { email: self.email, name: self.name }
     end
 
+    def imap?
+      !microsoft_graph?
+    end
+
+    def microsoft_graph?
+      self[:inbox_method].to_s == 'microsoft_graph'
+    end
+
     def validate!
+      if microsoft_graph?
+        validate_microsoft_graph!
+      else
+        validate_imap!
+      end
+    end
+
+    private
+
+    def validate_imap!
       if self[:idle_timeout] > IMAP_IDLE_TIMEOUT
         raise IdleTimeoutTooLarge,
               "Please use an idle timeout smaller than #{29*60} to prevent " \
               "IMAP server disconnects"
       end
 
-      REQUIRED_CONFIGURATION.each do |k|
+      IMAP_CONFIGURATION.each do |k|
         if self[k].nil?
           raise ConfigurationError,
                 "Field :#{k} is required in Mailbox: #{inspect}"
@@ -137,7 +160,23 @@ module MailRoom
       end
     end
 
-    private
+    def validate_microsoft_graph!
+      raise ConfigurationError, "Missing inbox_options in Mailbox: #{inspect}" unless self.inbox_options.is_a?(Hash)
+
+      MICROSOFT_GRAPH_CONFIGURATION.each do |k|
+        if self[k].nil?
+          raise ConfigurationError,
+                "Field :#{k} is required in Mailbox: #{inspect}"
+        end
+      end
+
+      MICROSOFT_GRAPH_INBOX_OPTIONS.each do |k|
+        if self[:inbox_options][k].nil?
+          raise ConfigurationError,
+                "inbox_options field :#{k} is required in Mailbox: #{inspect}"
+        end
+      end
+    end
 
     def parsed_arbitration_options
       arbitration_klass::Options.new(self)

data/lib/mail_room/mailbox_watcher.rb

@@ -62,8 +62,14 @@ module MailRoom
     end
 
     private
+
     def connection
-      @connection ||= ::MailRoom::IMAP::Connection.new(@mailbox)
+      @connection ||=
+        if @mailbox.microsoft_graph?
+          ::MailRoom::MicrosoftGraph::Connection.new(@mailbox)
+        else
+          ::MailRoom::IMAP::Connection.new(@mailbox)
+        end
     end
   end
 end

data/lib/mail_room/microsoft_graph/connection.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'oauth2'
+
+module MailRoom
+  module MicrosoftGraph
+    class Connection < MailRoom::Connection
+      SCOPE = 'https://graph.microsoft.com/.default'
+      NEXT_PAGE_KEY = '@odata.nextLink'
+      DEFAULT_POLL_INTERVAL_S = 60
+
+      TooManyRequestsError = Class.new(RuntimeError)
+
+      attr_accessor :token, :throttled_count
+
+      def initialize(mailbox)
+        super
+
+        reset
+        setup
+      end
+
+      def wait
+        process_mailbox
+
+        @throttled_count = 0
+        wait_for_new_messages
+      rescue TooManyRequestsError => e
+        @throttled_count += 1
+
+        @mailbox.logger.warn({ context: @mailbox.context, action: 'Too many requests, backing off...', backoff_s: backoff_secs, error: e.message, error_backtrace: e.backtrace })
+
+        backoff
+      rescue IOError => e
+        @mailbox.logger.warn({ context: @mailbox.context, action: 'Disconnected. Resetting...', error: e.message, error_backtrace: e.backtrace })
+
+        reset
+        setup
+      end
+
+      private
+
+      def wait_for_new_messages
+        sleep poll_interval
+      end
+
+      def backoff
+        sleep backoff_secs
+      end
+
+      def backoff_secs
+        [60 * 10, 2**throttled_count].min
+      end
+
+      def reset
+        @token = nil
+        @throttled_count = 0
+      end
+
+      def setup
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Retrieving OAuth2 token...' })
+
+        @token = client.client_credentials.get_token({ scope: SCOPE })
+      end
+
+      def client
+        @client ||= OAuth2::Client.new(client_id, client_secret,
+                                       site: 'https://login.microsoftonline.com',
+                                       authorize_url: "/#{tenant_id}/oauth2/v2.0/authorize",
+                                       token_url: "/#{tenant_id}/oauth2/v2.0/token",
+                                       auth_scheme: :basic_auth)
+      end
+
+      def inbox_options
+        mailbox.inbox_options
+      end
+
+      def tenant_id
+        inbox_options[:tenant_id]
+      end
+
+      def client_id
+        inbox_options[:client_id]
+      end
+
+      def client_secret
+        inbox_options[:client_secret]
+      end
+
+      def poll_interval
+        @poll_interval ||= begin
+          interval = inbox_options[:poll_interval].to_i
+
+          if interval.positive?
+            interval
+          else
+            DEFAULT_POLL_INTERVAL_S
+          end
+        end
+      end
+
+      def process_mailbox
+        return unless @new_message_handler
+
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Processing started' })
+
+        new_messages.each do |msg|
+          success = @new_message_handler.call(msg)
+          handle_delivered(msg) if success
+        end
+      end
+
+      def handle_delivered(msg)
+        mark_as_read(msg)
+        delete_message(msg) if @mailbox.delete_after_delivery
+      end
+
+      def delete_message(msg)
+        token.delete(msg_url(msg.uid))
+      end
+
+      def mark_as_read(msg)
+        token.patch(msg_url(msg.uid),
+                    headers: { 'Content-Type' => 'application/json' },
+                    body: { isRead: true }.to_json)
+      end
+
+      def new_messages
+        messages_for_ids(new_message_ids)
+      end
+
+      # Yields a page of message IDs at a time
+      def new_message_ids
+        url = unread_messages_url
+
+        Enumerator.new do |block|
+          loop do
+            messages, next_page_url = unread_messages(url: url)
+            messages.each { |msg| block.yield msg }
+
+            break unless next_page_url
+
+            url = next_page_url
+          end
+        end
+      end
+
+      def unread_messages(url:)
+        body = get(url)
+
+        return [[], nil] unless body
+
+        all_unread = body['value'].map { |msg| msg['id'] }
+        to_deliver = all_unread.select { |uid| @mailbox.deliver?(uid) }
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Getting new messages',
+                               unread: { count: all_unread.count, ids: all_unread },
+                               to_be_delivered: { count: to_deliver.count, ids: to_deliver } })
+        [to_deliver, body[NEXT_PAGE_KEY]]
+      rescue TypeError, JSON::ParserError => e
+        log_exception('Error parsing JSON response', e)
+        [[], nil]
+      end
+
+      # Returns the JSON response
+      def get(url)
+        response = token.get(url, { raise_errors: false })
+
+        # https://docs.microsoft.com/en-us/graph/errors
+        case response.status
+        when 509, 429
+          raise TooManyRequestsError
+        when 400..599
+          raise OAuth2::Error, response
+        end
+
+        return unless response.body
+
+        body = JSON.parse(response.body)
+
+        raise TypeError, 'Response did not contain value hash' unless body.is_a?(Hash) && body.key?('value')
+
+        body
+      end
+
+      def messages_for_ids(message_ids)
+        message_ids.each_with_object([]) do |id, arr|
+          response = token.get(rfc822_msg_url(id))
+
+          arr << ::MailRoom::Message.new(uid: id, body: response.body)
+        end
+      end
+
+      def base_url
+        "https://graph.microsoft.com/v1.0/users/#{mailbox.email}/mailFolders/#{mailbox.name}/messages"
+      end
+
+      def unread_messages_url
+        "#{base_url}?$filter=isRead eq false"
+      end
+
+      def msg_url(id)
+        # Attempting to use the base_url fails with "The OData request is not supported"
+        "https://graph.microsoft.com/v1.0/users/#{mailbox.email}/messages/#{id}"
+      end
+
+      def rfc822_msg_url(id)
+        # Attempting to use the base_url fails with "The OData request is not supported"
+        "#{msg_url(id)}/$value"
+      end
+
+      def log_exception(message, exception)
+        @mailbox.logger.warn({ context: @mailbox.context, message: message, exception: exception.to_s })
+      end
+    end
+  end
+end

data/lib/mail_room/microsoft_graph.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module MailRoom
+  module MicrosoftGraph
+    autoload :Connection, 'mail_room/microsoft_graph/connection'
+  end
+end

data/lib/mail_room/version.rb

@@ -1,4 +1,4 @@
 module MailRoom
   # Current version of gitlab-mail_room gem
-  VERSION = "0.0.10"
+  VERSION = "0.0.14"
 end

data/mail_room.gemspec

@@ -18,9 +18,14 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_dependency             "net-imap", ">= 0.2.1"
+  gem.add_dependency             "oauth2", "~> 1.4.4"
+
+  # Pinning io-wait to 0.1.0, which is the last version to support Ruby < 3
+  gem.add_dependency              "io-wait", "~> 0.1.0"
 
   gem.add_development_dependency "rake"
   gem.add_development_dependency "rspec", "~> 3.9"
+  gem.add_development_dependency "rubocop", "~> 1.11"
   gem.add_development_dependency "mocha", "~> 1.11"
   gem.add_development_dependency "simplecov"
   gem.add_development_dependency "webrick", "~> 1.6"
@@ -33,4 +38,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "redis-namespace"
   gem.add_development_dependency "pg"
   gem.add_development_dependency "charlock_holmes"
+  gem.add_development_dependency "webmock"
 end

data/spec/lib/cli_spec.rb

@@ -2,8 +2,8 @@ require 'spec_helper'
 
 describe MailRoom::CLI do
   let(:config_path) {File.expand_path('../fixtures/test_config.yml', File.dirname(__FILE__))}
-  let!(:configuration) {MailRoom::Configuration.new({:config_path => config_path})}
-  let(:coordinator) {stub(:run => true, :quit => true)}
+  let!(:configuration) {MailRoom::Configuration.new({config_path: config_path})}
+  let(:coordinator) {stub(run: true, quit: true)}
   let(:configuration_args) { anything }
   let(:coordinator_args) { [anything, anything] }
 
@@ -17,7 +17,7 @@ describe MailRoom::CLI do
 
     context 'with configuration args' do
       let(:configuration_args) do
-        {:config_path => 'a path'}
+        {config_path: 'a path'}
       end
 
       it 'parses arguments into configuration' do

data/spec/lib/configuration_spec.rb

@@ -5,7 +5,7 @@ describe MailRoom::Configuration do
 
   describe '#initalize' do
     context 'with config_path' do
-      let(:configuration) { MailRoom::Configuration.new(:config_path => config_path) }
+      let(:configuration) { MailRoom::Configuration.new(config_path: config_path) }
 
       it 'parses yaml into mailbox objects' do
         MailRoom::Mailbox.stubs(:new).returns('mailbox1', 'mailbox2')

data/spec/lib/delivery/letter_opener_spec.rb

@@ -3,7 +3,7 @@ require 'mail_room/delivery/letter_opener'
 
 describe MailRoom::Delivery::LetterOpener do
   describe '#deliver' do
-    let(:mailbox) {build_mailbox(:location => '/tmp/somewhere')}
+    let(:mailbox) {build_mailbox(location: '/tmp/somewhere')}
     let(:delivery_method) {stub(:deliver!)}
     let(:mail) {stub}
 
@@ -13,7 +13,7 @@ describe MailRoom::Delivery::LetterOpener do
     end
 
     it 'creates a new LetterOpener::DeliveryMethod' do
-      ::LetterOpener::DeliveryMethod.expects(:new).with(:location => '/tmp/somewhere').returns(delivery_method)
+      ::LetterOpener::DeliveryMethod.expects(:new).with(location: '/tmp/somewhere').returns(delivery_method)
 
       MailRoom::Delivery::LetterOpener.new(mailbox).deliver('a message')
     end

data/spec/lib/delivery/logger_spec.rb

@@ -16,7 +16,7 @@ describe MailRoom::Delivery::Logger do
     end
 
     context "with a log path" do
-      let(:mailbox) {build_mailbox(:log_path => '/var/log/mail-room.log')}
+      let(:mailbox) {build_mailbox(log_path: '/var/log/mail-room.log')}
 
       it 'creates a new file to append to' do
         file = stub(:sync=)

data/spec/lib/delivery/postback_spec.rb

@@ -5,8 +5,8 @@ describe MailRoom::Delivery::Postback do
   describe '#deliver' do
     context 'with token auth delivery' do
       let(:mailbox) {build_mailbox({
-        :delivery_url => 'http://localhost/inbox',
-        :delivery_token => 'abcdefg'
+        delivery_url: 'http://localhost/inbox',
+        delivery_token: 'abcdefg'
       })}
 
       let(:delivery_options) {
@@ -30,10 +30,10 @@ describe MailRoom::Delivery::Postback do
 
     context 'with basic auth delivery options' do
       let(:mailbox) {build_mailbox({
-        :delivery_options => {
-          :url => 'http://localhost/inbox',
-          :username => 'user1',
-          :password => 'password123abc'
+        delivery_options: {
+          url: 'http://localhost/inbox',
+          username: 'user1',
+          password: 'password123abc'
         }
       })}
 
@@ -57,11 +57,11 @@ describe MailRoom::Delivery::Postback do
 
       context 'with content type in the delivery options' do
         let(:mailbox) {build_mailbox({
-          :delivery_options => {
-            :url => 'http://localhost/inbox',
-            :username => 'user1',
-            :password => 'password123abc',
-            :content_type => 'text/plain'
+          delivery_options: {
+            url: 'http://localhost/inbox',
+            username: 'user1',
+            password: 'password123abc',
+            content_type: 'text/plain'
           }
         })}
 

data/spec/lib/delivery/sidekiq_spec.rb

@@ -8,23 +8,45 @@ describe MailRoom::Delivery::Sidekiq do
 
   describe '#options' do
     let(:redis_url) { 'redis://localhost' }
+    let(:redis_options) { { redis_url: redis_url } }
 
     context 'when only redis_url is specified' do
       let(:mailbox) {
         build_mailbox(
           delivery_method: :sidekiq,
-          delivery_options: {
-            redis_url: redis_url
-          }
+          delivery_options: redis_options
         )
       }
 
-      it 'client has same specified redis_url' do
-        expect(redis.client.options[:url]).to eq(redis_url)
+      context 'with simple redis url' do
+        it 'client has same specified redis_url' do
+          expect(redis.client.options[:url]).to eq(redis_url)
+        end
+
+        it 'client is a instance of RedisNamespace class' do
+          expect(redis).to be_a ::Redis
+        end
+
+        it 'connection has correct values' do
+          expect(redis.connection[:host]).to eq('localhost')
+          expect(redis.connection[:db]).to eq(0)
+        end
       end
 
-      it 'client is a instance of RedisNamespace class' do
-        expect(redis).to be_a ::Redis
+      context 'with redis_db specified in options' do
+        before do
+          redis_options[:redis_db] = 4
+        end
+
+        it 'client has correct redis_url' do
+          expect(redis.client.options[:url]).to eq(redis_url)
+        end
+
+
+        it 'connection has correct values' do
+          expect(redis.connection[:host]).to eq('localhost')
+          expect(redis.connection[:db]).to eq(4)
+        end
       end
     end