gitlab-mail_room 0.0.9 → 0.0.23

data/lib/mail_room/microsoft_graph/connection.rb

@@ -0,0 +1,243 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'oauth2'
+
+module MailRoom
+  module MicrosoftGraph
+    class Connection < MailRoom::Connection
+      NEXT_PAGE_KEY = '@odata.nextLink'
+      DEFAULT_POLL_INTERVAL_S = 60
+
+      TooManyRequestsError = Class.new(RuntimeError)
+
+      attr_accessor :token, :throttled_count
+
+      def initialize(mailbox)
+        super
+
+        reset
+        setup
+      end
+
+      def wait
+        return if stopped?
+
+        process_mailbox
+
+        @throttled_count = 0
+        wait_for_new_messages
+      rescue TooManyRequestsError => e
+        @throttled_count += 1
+
+        @mailbox.logger.warn({ context: @mailbox.context, action: 'Too many requests, backing off...', backoff_s: backoff_secs, error: e.message, error_backtrace: e.backtrace })
+
+        backoff
+      rescue IOError => e
+        @mailbox.logger.warn({ context: @mailbox.context, action: 'Disconnected. Resetting...', error: e.message, error_backtrace: e.backtrace })
+
+        reset
+        setup
+      end
+
+      private
+
+      def wait_for_new_messages
+        sleep_while_running(poll_interval)
+      end
+
+      def backoff
+        sleep_while_running(backoff_secs)
+      end
+
+      def backoff_secs
+        [60 * 10, 2**throttled_count].min
+      end
+
+      # Unless wake up periodically, we won't notice that the thread was stopped
+      # if we sleep the entire interval.
+      def sleep_while_running(sleep_interval)
+        sleep_interval.times do
+          do_sleep(1)
+          return if stopped?
+        end
+      end
+
+      def do_sleep(interval)
+        sleep(interval)
+      end
+
+      def reset
+        @token = nil
+        @throttled_count = 0
+      end
+
+      def setup
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Retrieving OAuth2 token...' })
+
+        @token = client.client_credentials.get_token({ scope: scope })
+      end
+
+      def client
+        @client ||= OAuth2::Client.new(client_id, client_secret,
+                                       site: azure_ad_endpoint,
+                                       authorize_url: "/#{tenant_id}/oauth2/v2.0/authorize",
+                                       token_url: "/#{tenant_id}/oauth2/v2.0/token",
+                                       auth_scheme: :basic_auth)
+      end
+
+      def inbox_options
+        mailbox.inbox_options
+      end
+
+      def tenant_id
+        inbox_options[:tenant_id]
+      end
+
+      def client_id
+        inbox_options[:client_id]
+      end
+
+      def client_secret
+        inbox_options[:client_secret]
+      end
+
+      def poll_interval
+        @poll_interval ||= begin
+          interval = inbox_options[:poll_interval].to_i
+
+          if interval.positive?
+            interval
+          else
+            DEFAULT_POLL_INTERVAL_S
+          end
+        end
+      end
+
+      def process_mailbox
+        return unless @new_message_handler
+
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Processing started' })
+
+        new_messages.each do |msg|
+          success = @new_message_handler.call(msg)
+          handle_delivered(msg) if success
+        end
+      end
+
+      def handle_delivered(msg)
+        mark_as_read(msg)
+        delete_message(msg) if @mailbox.delete_after_delivery
+      end
+
+      def delete_message(msg)
+        token.delete(msg_url(msg.uid))
+      end
+
+      def mark_as_read(msg)
+        token.patch(msg_url(msg.uid),
+                    headers: { 'Content-Type' => 'application/json' },
+                    body: { isRead: true }.to_json)
+      end
+
+      def new_messages
+        messages_for_ids(new_message_ids)
+      end
+
+      # Yields a page of message IDs at a time
+      def new_message_ids
+        url = unread_messages_url
+
+        Enumerator.new do |block|
+          loop do
+            messages, next_page_url = unread_messages(url: url)
+            messages.each { |msg| block.yield msg }
+
+            break unless next_page_url
+
+            url = next_page_url
+          end
+        end
+      end
+
+      def unread_messages(url:)
+        body = get(url)
+
+        return [[], nil] unless body
+
+        all_unread = body['value'].map { |msg| msg['id'] }
+        to_deliver = all_unread.select { |uid| @mailbox.deliver?(uid) }
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Getting new messages',
+                               unread: { count: all_unread.count, ids: all_unread },
+                               to_be_delivered: { count: to_deliver.count, ids: to_deliver } })
+        [to_deliver, body[NEXT_PAGE_KEY]]
+      rescue TypeError, JSON::ParserError => e
+        log_exception('Error parsing JSON response', e)
+        [[], nil]
+      end
+
+      # Returns the JSON response
+      def get(url)
+        response = token.get(url, { raise_errors: false })
+
+        # https://docs.microsoft.com/en-us/graph/errors
+        case response.status
+        when 509, 429
+          raise TooManyRequestsError
+        when 400..599
+          raise OAuth2::Error, response
+        end
+
+        return unless response.body
+
+        body = JSON.parse(response.body)
+
+        raise TypeError, 'Response did not contain value hash' unless body.is_a?(Hash) && body.key?('value')
+
+        body
+      end
+
+      def messages_for_ids(message_ids)
+        message_ids.each_with_object([]) do |id, arr|
+          response = token.get(rfc822_msg_url(id))
+
+          arr << ::MailRoom::Message.new(uid: id, body: response.body)
+        end
+      end
+
+      def base_url
+        "#{graph_endpoint}/v1.0/users/#{mailbox.email}/mailFolders/#{mailbox.name}/messages"
+      end
+
+      def unread_messages_url
+        "#{base_url}?$filter=isRead eq false"
+      end
+
+      def msg_url(id)
+        # Attempting to use the base_url fails with "The OData request is not supported"
+        "#{graph_endpoint}/v1.0/users/#{mailbox.email}/messages/#{id}"
+      end
+
+      def rfc822_msg_url(id)
+        # Attempting to use the base_url fails with "The OData request is not supported"
+        "#{msg_url(id)}/$value"
+      end
+
+      def log_exception(message, exception)
+        @mailbox.logger.warn({ context: @mailbox.context, message: message, exception: exception.to_s })
+      end
+
+      def scope
+        "#{graph_endpoint}/.default"
+      end
+
+      def graph_endpoint
+        inbox_options[:graph_endpoint] || 'https://graph.microsoft.com'
+      end
+
+      def azure_ad_endpoint
+        inbox_options[:azure_ad_endpoint] || 'https://login.microsoftonline.com'
+      end
+    end
+  end
+end

data/lib/mail_room/microsoft_graph.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module MailRoom
+  module MicrosoftGraph
+    autoload :Connection, 'mail_room/microsoft_graph/connection'
+  end
+end

data/lib/mail_room/version.rb

@@ -1,4 +1,4 @@
 module MailRoom
   # Current version of gitlab-mail_room gem
-  VERSION = "0.0.9"
+  VERSION = "0.0.23"
 end

data/mail_room.gemspec

@@ -17,8 +17,13 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
+  gem.add_dependency             "net-imap", ">= 0.2.1"
+  gem.add_dependency             "oauth2", [">= 1.4.4", "< 3"]
+  gem.add_dependency             "jwt", ">= 2.0"
+
   gem.add_development_dependency "rake"
   gem.add_development_dependency "rspec", "~> 3.9"
+  gem.add_development_dependency "rubocop", "~> 1.11"
   gem.add_development_dependency "mocha", "~> 1.11"
   gem.add_development_dependency "simplecov"
   gem.add_development_dependency "webrick", "~> 1.6"
@@ -27,8 +32,9 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "faraday"
   gem.add_development_dependency "mail"
   gem.add_development_dependency "letter_opener"
-  gem.add_development_dependency "redis", "~> 3.3.1"
+  gem.add_development_dependency "redis", "~> 4"
   gem.add_development_dependency "redis-namespace"
   gem.add_development_dependency "pg"
   gem.add_development_dependency "charlock_holmes"
+  gem.add_development_dependency "webmock"
 end

data/spec/fixtures/jwt_secret

@@ -0,0 +1 @@
+aGVsbG93b3JsZA==

data/spec/lib/arbitration/redis_spec.rb

@@ -15,6 +15,7 @@ describe MailRoom::Arbitration::Redis do
 
   # Private, but we don't care.
   let(:redis) { subject.send(:client) }
+  let(:raw_client) { redis._client }
 
   describe '#deliver?' do
     context "when called the first time" do
@@ -95,7 +96,7 @@ describe MailRoom::Arbitration::Redis do
       it 'client has same specified url' do
         subject.deliver?(123)
 
-        expect(redis.client.options[:url]).to eq redis_url
+        expect(raw_client.options[:url]).to eq redis_url
       end
 
       it 'client is a instance of Redis class' do
@@ -137,10 +138,10 @@ describe MailRoom::Arbitration::Redis do
       before { ::Redis::Client::Connector::Sentinel.any_instance.stubs(:resolve).returns(sentinels) }
 
       it 'client has same specified sentinel params' do
-        expect(redis.client.instance_variable_get(:@connector)).to be_a Redis::Client::Connector::Sentinel
-        expect(redis.client.options[:host]).to eq('sentinel-master')
-        expect(redis.client.options[:password]).to eq('mypassword')
-        expect(redis.client.options[:sentinels]).to eq(sentinels)
+        expect(raw_client.instance_variable_get(:@connector)).to be_a Redis::Client::Connector::Sentinel
+        expect(raw_client.options[:host]).to eq('sentinel-master')
+        expect(raw_client.options[:password]).to eq('mypassword')
+        expect(raw_client.options[:sentinels]).to eq(sentinels)
       end
     end
   end

data/spec/lib/cli_spec.rb

@@ -2,8 +2,8 @@ require 'spec_helper'
 
 describe MailRoom::CLI do
   let(:config_path) {File.expand_path('../fixtures/test_config.yml', File.dirname(__FILE__))}
-  let!(:configuration) {MailRoom::Configuration.new({:config_path => config_path})}
-  let(:coordinator) {stub(:run => true, :quit => true)}
+  let!(:configuration) {MailRoom::Configuration.new({config_path: config_path})}
+  let(:coordinator) {stub(run: true, quit: true)}
   let(:configuration_args) { anything }
   let(:coordinator_args) { [anything, anything] }
 
@@ -17,7 +17,7 @@ describe MailRoom::CLI do
 
     context 'with configuration args' do
       let(:configuration_args) do
-        {:config_path => 'a path'}
+        {config_path: 'a path'}
       end
 
       it 'parses arguments into configuration' do

data/spec/lib/configuration_spec.rb

@@ -5,7 +5,7 @@ describe MailRoom::Configuration do
 
   describe '#initalize' do
     context 'with config_path' do
-      let(:configuration) { MailRoom::Configuration.new(:config_path => config_path) }
+      let(:configuration) { MailRoom::Configuration.new(config_path: config_path) }
 
       it 'parses yaml into mailbox objects' do
         MailRoom::Mailbox.stubs(:new).returns('mailbox1', 'mailbox2')

data/spec/lib/delivery/letter_opener_spec.rb

@@ -3,17 +3,18 @@ require 'mail_room/delivery/letter_opener'
 
 describe MailRoom::Delivery::LetterOpener do
   describe '#deliver' do
-    let(:mailbox) {build_mailbox(:location => '/tmp/somewhere')}
-    let(:delivery_method) {stub(:deliver!)}
+    let(:mailbox) {build_mailbox(location: '/tmp/somewhere')}
+    let(:delivery_method) {stub}
     let(:mail) {stub}
 
     before :each do
       Mail.stubs(:read_from_string).returns(mail)
       ::LetterOpener::DeliveryMethod.stubs(:new).returns(delivery_method)
+      delivery_method.stubs(:deliver!)
     end
 
     it 'creates a new LetterOpener::DeliveryMethod' do
-      ::LetterOpener::DeliveryMethod.expects(:new).with(:location => '/tmp/somewhere').returns(delivery_method)
+      ::LetterOpener::DeliveryMethod.expects(:new).with(location: '/tmp/somewhere').returns(delivery_method)
 
       MailRoom::Delivery::LetterOpener.new(mailbox).deliver('a message')
     end

data/spec/lib/delivery/logger_spec.rb

@@ -16,10 +16,11 @@ describe MailRoom::Delivery::Logger do
     end
 
     context "with a log path" do
-      let(:mailbox) {build_mailbox(:log_path => '/var/log/mail-room.log')}
+      let(:mailbox) {build_mailbox(log_path: '/var/log/mail-room.log')}
 
       it 'creates a new file to append to' do
-        file = stub(:sync=)
+        file = stub
+        file.stubs(:sync=)
 
         File.expects(:open).with('/var/log/mail-room.log', 'a').returns(file)
         ::Logger.stubs(:new).with(file)

data/spec/lib/delivery/postback_spec.rb

@@ -5,8 +5,8 @@ describe MailRoom::Delivery::Postback do
   describe '#deliver' do
     context 'with token auth delivery' do
       let(:mailbox) {build_mailbox({
-        :delivery_url => 'http://localhost/inbox',
-        :delivery_token => 'abcdefg'
+        delivery_url: 'http://localhost/inbox',
+        delivery_token: 'abcdefg'
       })}
 
       let(:delivery_options) {
@@ -30,10 +30,10 @@ describe MailRoom::Delivery::Postback do
 
     context 'with basic auth delivery options' do
       let(:mailbox) {build_mailbox({
-        :delivery_options => {
-          :url => 'http://localhost/inbox',
-          :username => 'user1',
-          :password => 'password123abc'
+        delivery_options: {
+          url: 'http://localhost/inbox',
+          username: 'user1',
+          password: 'password123abc'
         }
       })}
 
@@ -57,19 +57,18 @@ describe MailRoom::Delivery::Postback do
 
       context 'with content type in the delivery options' do
         let(:mailbox) {build_mailbox({
-          :delivery_options => {
-            :url => 'http://localhost/inbox',
-            :username => 'user1',
-            :password => 'password123abc',
-            :content_type => 'text/plain'
+          delivery_options: {
+            url: 'http://localhost/inbox',
+            username: 'user1',
+            password: 'password123abc',
+            content_type: 'text/plain'
           }
         })}
 
-  
         let(:delivery_options) {
           MailRoom::Delivery::Postback::Options.new(mailbox)
         }
-  
+
         it 'posts the message with faraday' do
           connection = stub
           request = stub
@@ -82,10 +81,59 @@ describe MailRoom::Delivery::Postback do
           connection.expects(:basic_auth).with('user1', 'password123abc')
 
           MailRoom::Delivery::Postback.new(delivery_options).deliver('a message')
-          
+
           expect(request.headers['Content-Type']).to eq('text/plain')
         end
       end
+
+      context 'with jwt token in the delivery options' do
+        let(:mailbox) {build_mailbox({
+          delivery_options: {
+            url: 'http://localhost/inbox',
+            jwt_auth_header: "Mailroom-Api-Request",
+            jwt_issuer: "mailroom",
+            jwt_algorithm: "HS256",
+            jwt_secret_path: "secret_path"
+          }
+        })}
+
+        let(:delivery_options) {
+          MailRoom::Delivery::Postback::Options.new(mailbox)
+        }
+
+        it 'posts the message with faraday' do
+          connection = stub
+          request = stub
+          Faraday.stubs(:new).returns(connection)
+
+          connection.expects(:post).yields(request).twice
+          request.stubs(:url)
+          request.stubs(:body=)
+          request.stubs(:headers).returns({})
+
+          jwt = stub
+          MailRoom::JWT.expects(:new).with(
+            header: 'Mailroom-Api-Request',
+            issuer: 'mailroom',
+            algorithm: 'HS256',
+            secret_path: 'secret_path'
+          ).returns(jwt)
+          jwt.stubs(:valid?).returns(true)
+          jwt.stubs(:header).returns('Mailroom-Api-Request')
+          jwt.stubs(:token).returns('a_jwt_token')
+
+          delivery = MailRoom::Delivery::Postback.new(delivery_options)
+
+          delivery.deliver('a message')
+          expect(request.headers['Mailroom-Api-Request']).to eql('a_jwt_token')
+
+          # A different jwt token for the second time
+          jwt.stubs(:token).returns('another_jwt_token')
+
+          delivery.deliver('another message')
+          expect(request.headers['Mailroom-Api-Request']).to eql('another_jwt_token')
+        end
+      end
     end
   end
 end

data/spec/lib/delivery/sidekiq_spec.rb

@@ -4,27 +4,49 @@ require 'mail_room/delivery/sidekiq'
 describe MailRoom::Delivery::Sidekiq do
   subject { described_class.new(options) }
   let(:redis) { subject.send(:client) }
+  let(:raw_client) { redis._client }
   let(:options) { MailRoom::Delivery::Sidekiq::Options.new(mailbox) }
 
   describe '#options' do
     let(:redis_url) { 'redis://localhost' }
+    let(:redis_options) { { redis_url: redis_url } }
 
     context 'when only redis_url is specified' do
       let(:mailbox) {
         build_mailbox(
           delivery_method: :sidekiq,
-          delivery_options: {
-            redis_url: redis_url
-          }
+          delivery_options: redis_options
         )
       }
 
-      it 'client has same specified redis_url' do
-        expect(redis.client.options[:url]).to eq(redis_url)
+      context 'with simple redis url' do
+        it 'client has same specified redis_url' do
+          expect(raw_client.options[:url]).to eq(redis_url)
+        end
+
+        it 'client is a instance of RedisNamespace class' do
+          expect(redis).to be_a ::Redis
+        end
+
+        it 'connection has correct values' do
+          expect(redis.connection[:host]).to eq('localhost')
+          expect(redis.connection[:db]).to eq(0)
+        end
       end
 
-      it 'client is a instance of RedisNamespace class' do
-        expect(redis).to be_a ::Redis
+      context 'with redis_db specified in options' do
+        before do
+          redis_options[:redis_db] = 4
+        end
+
+        it 'client has correct redis_url' do
+          expect(raw_client.options[:url]).to eq(redis_url)
+        end
+
+        it 'connection has correct values' do
+          expect(redis.connection[:host]).to eq('localhost')
+          expect(redis.connection[:db]).to eq(4)
+        end
       end
     end
 
@@ -65,10 +87,10 @@ describe MailRoom::Delivery::Sidekiq do
       before { ::Redis::Client::Connector::Sentinel.any_instance.stubs(:resolve).returns(sentinels) }
 
       it 'client has same specified sentinel params' do
-        expect(redis.client.instance_variable_get(:@connector)).to be_a Redis::Client::Connector::Sentinel
-        expect(redis.client.options[:host]).to eq('sentinel-master')
-        expect(redis.client.options[:password]).to eq('mypassword')
-        expect(redis.client.options[:sentinels]).to eq(sentinels)
+        expect(raw_client.instance_variable_get(:@connector)).to be_a Redis::Client::Connector::Sentinel
+        expect(raw_client.options[:host]).to eq('sentinel-master')
+        expect(raw_client.options[:password]).to eq('mypassword')
+        expect(raw_client.options[:sentinels]).to eq(sentinels)
       end
     end
 

data/spec/lib/jwt_spec.rb

@@ -0,0 +1,80 @@
+require 'spec_helper'
+
+require 'mail_room/jwt'
+
+describe MailRoom::JWT do
+  let(:secret_path) { File.expand_path('../fixtures/jwt_secret', File.dirname(__FILE__)) }
+  let(:secret) { Base64.strict_decode64(File.read(secret_path).chomp) }
+
+  let(:standard_config) do
+    {
+      secret_path: secret_path,
+      issuer: 'mailroom',
+      header: 'Mailroom-Api-Request',
+      algorithm: 'HS256'
+    }
+  end
+
+  describe '#token' do
+    let(:jwt) { described_class.new(**standard_config) }
+
+    it 'generates a valid jwt token' do
+      token = jwt.token
+      expect(token).not_to be_empty
+
+      payload = nil
+      expect do
+        payload = JWT.decode(token, secret, true, iss: 'mailroom', verify_iat: true, verify_iss: true, algorithm: 'HS256')
+      end.not_to raise_error
+      expect(payload).to be_an(Array)
+      expect(payload).to match(
+        [
+          a_hash_including(
+            'iss' => 'mailroom',
+            'nonce' => be_a(String),
+            'iat' => be_a(Integer)
+          ),
+          { 'alg' => 'HS256' }
+        ]
+      )
+    end
+
+    it 'generates a different token for each invocation' do
+      expect(jwt.token).not_to eql(jwt.token)
+    end
+  end
+
+  describe '#valid?' do
+    it 'returns true if all essential components are present' do
+      jwt = described_class.new(**standard_config)
+      expect(jwt.valid?).to eql(true)
+    end
+
+    it 'returns true if header and secret path are present' do
+      jwt = described_class.new(
+        secret_path: secret_path,
+        header: 'Mailroom-Api-Request',
+        issuer: nil,
+        algorithm: nil
+      )
+      expect(jwt.valid?).to eql(true)
+      expect(jwt.issuer).to eql(described_class::DEFAULT_ISSUER)
+      expect(jwt.algorithm).to eql(described_class::DEFAULT_ALGORITHM)
+    end
+
+    it 'returns false if either header or secret_path are missing' do
+      expect(described_class.new(
+        secret_path: nil,
+        header: 'Mailroom-Api-Request',
+        issuer: nil,
+        algorithm: nil
+      ).valid?).to eql(false)
+      expect(described_class.new(
+        secret_path: secret_path,
+        header: nil,
+        issuer: nil,
+        algorithm: nil
+      ).valid?).to eql(false)
+    end
+  end
+end