gitlab-mail_room 0.0.4 → 0.0.20

data/lib/mail_room/microsoft_graph/connection.rb

@@ -0,0 +1,243 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'oauth2'
+
+module MailRoom
+  module MicrosoftGraph
+    class Connection < MailRoom::Connection
+      NEXT_PAGE_KEY = '@odata.nextLink'
+      DEFAULT_POLL_INTERVAL_S = 60
+
+      TooManyRequestsError = Class.new(RuntimeError)
+
+      attr_accessor :token, :throttled_count
+
+      def initialize(mailbox)
+        super
+
+        reset
+        setup
+      end
+
+      def wait
+        return if stopped?
+
+        process_mailbox
+
+        @throttled_count = 0
+        wait_for_new_messages
+      rescue TooManyRequestsError => e
+        @throttled_count += 1
+
+        @mailbox.logger.warn({ context: @mailbox.context, action: 'Too many requests, backing off...', backoff_s: backoff_secs, error: e.message, error_backtrace: e.backtrace })
+
+        backoff
+      rescue IOError => e
+        @mailbox.logger.warn({ context: @mailbox.context, action: 'Disconnected. Resetting...', error: e.message, error_backtrace: e.backtrace })
+
+        reset
+        setup
+      end
+
+      private
+
+      def wait_for_new_messages
+        sleep_while_running(poll_interval)
+      end
+
+      def backoff
+        sleep_while_running(backoff_secs)
+      end
+
+      def backoff_secs
+        [60 * 10, 2**throttled_count].min
+      end
+
+      # Unless wake up periodically, we won't notice that the thread was stopped
+      # if we sleep the entire interval.
+      def sleep_while_running(sleep_interval)
+        sleep_interval.times do
+          do_sleep(1)
+          return if stopped?
+        end
+      end
+
+      def do_sleep(interval)
+        sleep(interval)
+      end
+
+      def reset
+        @token = nil
+        @throttled_count = 0
+      end
+
+      def setup
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Retrieving OAuth2 token...' })
+
+        @token = client.client_credentials.get_token({ scope: scope })
+      end
+
+      def client
+        @client ||= OAuth2::Client.new(client_id, client_secret,
+                                       site: azure_ad_endpoint,
+                                       authorize_url: "/#{tenant_id}/oauth2/v2.0/authorize",
+                                       token_url: "/#{tenant_id}/oauth2/v2.0/token",
+                                       auth_scheme: :basic_auth)
+      end
+
+      def inbox_options
+        mailbox.inbox_options
+      end
+
+      def tenant_id
+        inbox_options[:tenant_id]
+      end
+
+      def client_id
+        inbox_options[:client_id]
+      end
+
+      def client_secret
+        inbox_options[:client_secret]
+      end
+
+      def poll_interval
+        @poll_interval ||= begin
+          interval = inbox_options[:poll_interval].to_i
+
+          if interval.positive?
+            interval
+          else
+            DEFAULT_POLL_INTERVAL_S
+          end
+        end
+      end
+
+      def process_mailbox
+        return unless @new_message_handler
+
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Processing started' })
+
+        new_messages.each do |msg|
+          success = @new_message_handler.call(msg)
+          handle_delivered(msg) if success
+        end
+      end
+
+      def handle_delivered(msg)
+        mark_as_read(msg)
+        delete_message(msg) if @mailbox.delete_after_delivery
+      end
+
+      def delete_message(msg)
+        token.delete(msg_url(msg.uid))
+      end
+
+      def mark_as_read(msg)
+        token.patch(msg_url(msg.uid),
+                    headers: { 'Content-Type' => 'application/json' },
+                    body: { isRead: true }.to_json)
+      end
+
+      def new_messages
+        messages_for_ids(new_message_ids)
+      end
+
+      # Yields a page of message IDs at a time
+      def new_message_ids
+        url = unread_messages_url
+
+        Enumerator.new do |block|
+          loop do
+            messages, next_page_url = unread_messages(url: url)
+            messages.each { |msg| block.yield msg }
+
+            break unless next_page_url
+
+            url = next_page_url
+          end
+        end
+      end
+
+      def unread_messages(url:)
+        body = get(url)
+
+        return [[], nil] unless body
+
+        all_unread = body['value'].map { |msg| msg['id'] }
+        to_deliver = all_unread.select { |uid| @mailbox.deliver?(uid) }
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Getting new messages',
+                               unread: { count: all_unread.count, ids: all_unread },
+                               to_be_delivered: { count: to_deliver.count, ids: to_deliver } })
+        [to_deliver, body[NEXT_PAGE_KEY]]
+      rescue TypeError, JSON::ParserError => e
+        log_exception('Error parsing JSON response', e)
+        [[], nil]
+      end
+
+      # Returns the JSON response
+      def get(url)
+        response = token.get(url, { raise_errors: false })
+
+        # https://docs.microsoft.com/en-us/graph/errors
+        case response.status
+        when 509, 429
+          raise TooManyRequestsError
+        when 400..599
+          raise OAuth2::Error, response
+        end
+
+        return unless response.body
+
+        body = JSON.parse(response.body)
+
+        raise TypeError, 'Response did not contain value hash' unless body.is_a?(Hash) && body.key?('value')
+
+        body
+      end
+
+      def messages_for_ids(message_ids)
+        message_ids.each_with_object([]) do |id, arr|
+          response = token.get(rfc822_msg_url(id))
+
+          arr << ::MailRoom::Message.new(uid: id, body: response.body)
+        end
+      end
+
+      def base_url
+        "#{graph_endpoint}/v1.0/users/#{mailbox.email}/mailFolders/#{mailbox.name}/messages"
+      end
+
+      def unread_messages_url
+        "#{base_url}?$filter=isRead eq false"
+      end
+
+      def msg_url(id)
+        # Attempting to use the base_url fails with "The OData request is not supported"
+        "#{graph_endpoint}/v1.0/users/#{mailbox.email}/messages/#{id}"
+      end
+
+      def rfc822_msg_url(id)
+        # Attempting to use the base_url fails with "The OData request is not supported"
+        "#{msg_url(id)}/$value"
+      end
+
+      def log_exception(message, exception)
+        @mailbox.logger.warn({ context: @mailbox.context, message: message, exception: exception.to_s })
+      end
+
+      def scope
+        "#{graph_endpoint}/.default"
+      end
+
+      def graph_endpoint
+        inbox_options[:graph_endpoint] || 'https://graph.microsoft.com'
+      end
+
+      def azure_ad_endpoint
+        inbox_options[:azure_ad_endpoint] || 'https://login.microsoftonline.com'
+      end
+    end
+  end
+end

data/lib/mail_room/microsoft_graph.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module MailRoom
+  module MicrosoftGraph
+    autoload :Connection, 'mail_room/microsoft_graph/connection'
+  end
+end

data/lib/mail_room/version.rb

@@ -1,4 +1,4 @@
 module MailRoom
-  # Current version of MailRoom gem
-  VERSION = "0.0.4"
+  # Current version of gitlab-mail_room gem
+  VERSION = "0.0.20"
 end

data/lib/mail_room.rb

@@ -7,8 +7,10 @@ end
 
 require "mail_room/version"
 require "mail_room/configuration"
+require "mail_room/health_check"
 require "mail_room/mailbox"
 require "mail_room/mailbox_watcher"
+require "mail_room/message"
 require "mail_room/connection"
 require "mail_room/coordinator"
 require "mail_room/cli"

data/mail_room.gemspec

@@ -17,18 +17,27 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
+  gem.add_dependency             "net-imap", ">= 0.2.1"
+  gem.add_dependency             "oauth2", "~> 1.4.4"
+  gem.add_dependency             "jwt", ">= 2.0"
+
+  # Pinning io-wait to 0.1.0, which is the last version to support Ruby < 3
+  gem.add_dependency              "io-wait", "~> 0.1.0"
+
   gem.add_development_dependency "rake"
-  gem.add_development_dependency "rspec"
-  gem.add_development_dependency "mocha"
-  gem.add_development_dependency "bourne"
+  gem.add_development_dependency "rspec", "~> 3.9"
+  gem.add_development_dependency "rubocop", "~> 1.11"
+  gem.add_development_dependency "mocha", "~> 1.11"
   gem.add_development_dependency "simplecov"
+  gem.add_development_dependency "webrick", "~> 1.6"
 
   # for testing delivery methods
   gem.add_development_dependency "faraday"
   gem.add_development_dependency "mail"
   gem.add_development_dependency "letter_opener"
-  gem.add_development_dependency "redis", "~> 3.3.1"
+  gem.add_development_dependency "redis", "~> 4"
   gem.add_development_dependency "redis-namespace"
   gem.add_development_dependency "pg"
   gem.add_development_dependency "charlock_holmes"
+  gem.add_development_dependency "webmock"
 end

data/spec/fixtures/jwt_secret

@@ -0,0 +1 @@
+aGVsbG93b3JsZA==

data/spec/fixtures/test_config.yml

@@ -1,4 +1,7 @@
 ---
+:health_check:
+  :address: "127.0.0.1"
+  :port: 8080
 :mailboxes:
   -
     :email: "user1@gmail.com"

data/spec/lib/arbitration/redis_spec.rb

@@ -5,7 +5,8 @@ describe MailRoom::Arbitration::Redis do
   let(:mailbox) {
     build_mailbox(
       arbitration_options: {
-        namespace: "mail_room"
+        namespace: "mail_room",
+        redis_url: ENV['REDIS_URL']
       }
     )
   }
@@ -14,6 +15,7 @@ describe MailRoom::Arbitration::Redis do
 
   # Private, but we don't care.
   let(:redis) { subject.send(:client) }
+  let(:raw_client) { redis._client }
 
   describe '#deliver?' do
     context "when called the first time" do
@@ -78,7 +80,7 @@ describe MailRoom::Arbitration::Redis do
 
   context 'redis client connection params' do
     context 'when only url is present' do
-      let(:redis_url) { "redis://localhost:6379" }
+      let(:redis_url) { ENV.fetch('REDIS_URL', 'redis://localhost:6379') }
       let(:mailbox) {
         build_mailbox(
           arbitration_options: {
@@ -94,7 +96,7 @@ describe MailRoom::Arbitration::Redis do
       it 'client has same specified url' do
         subject.deliver?(123)
 
-        expect(redis.client.options[:url]).to eq redis_url
+        expect(raw_client.options[:url]).to eq redis_url
       end
 
       it 'client is a instance of Redis class' do
@@ -136,10 +138,10 @@ describe MailRoom::Arbitration::Redis do
       before { ::Redis::Client::Connector::Sentinel.any_instance.stubs(:resolve).returns(sentinels) }
 
       it 'client has same specified sentinel params' do
-        expect(redis.client.instance_variable_get(:@connector)).to be_a Redis::Client::Connector::Sentinel
-        expect(redis.client.options[:host]).to eq('sentinel-master')
-        expect(redis.client.options[:password]).to eq('mypassword')
-        expect(redis.client.options[:sentinels]).to eq(sentinels)
+        expect(raw_client.instance_variable_get(:@connector)).to be_a Redis::Client::Connector::Sentinel
+        expect(raw_client.options[:host]).to eq('sentinel-master')
+        expect(raw_client.options[:password]).to eq('mypassword')
+        expect(raw_client.options[:sentinels]).to eq(sentinels)
       end
     end
   end

data/spec/lib/cli_spec.rb

@@ -2,25 +2,37 @@ require 'spec_helper'
 
 describe MailRoom::CLI do
   let(:config_path) {File.expand_path('../fixtures/test_config.yml', File.dirname(__FILE__))}
-  let!(:configuration) {MailRoom::Configuration.new({:config_path => config_path})}
-  let(:coordinator) {stub(:run => true, :quit => true)}
+  let!(:configuration) {MailRoom::Configuration.new({config_path: config_path})}
+  let(:coordinator) {stub(run: true, quit: true)}
+  let(:configuration_args) { anything }
+  let(:coordinator_args) { [anything, anything] }
 
   describe '.new' do
     let(:args) {["-c", "a path"]}
 
     before :each do
-      MailRoom::Configuration.stubs(:new).returns(configuration)
-      MailRoom::Coordinator.stubs(:new).returns(coordinator)
+      MailRoom::Configuration.expects(:new).with(configuration_args).returns(configuration)
+      MailRoom::Coordinator.stubs(:new).with(*coordinator_args).returns(coordinator)
     end
 
-    it 'parses arguments into configuration' do
-      expect(MailRoom::CLI.new(args).configuration).to eq(configuration)
-      expect(MailRoom::Configuration).to have_received(:new).with({:config_path => 'a path'})
+    context 'with configuration args' do
+      let(:configuration_args) do
+        {config_path: 'a path'}
+      end
+
+      it 'parses arguments into configuration' do
+        expect(MailRoom::CLI.new(args).configuration).to eq configuration
+      end
     end
 
-    it 'creates a new coordinator with configuration' do
-      expect(MailRoom::CLI.new(args).coordinator).to eq(coordinator)
-      expect(MailRoom::Coordinator).to have_received(:new).with(configuration.mailboxes)
+    context 'with coordinator args' do
+      let(:coordinator_args) do
+        [configuration.mailboxes, anything]
+      end
+
+      it 'creates a new coordinator with configuration' do
+        expect(MailRoom::CLI.new(args).coordinator).to eq(coordinator)
+      end
     end
   end
 
@@ -30,30 +42,33 @@ describe MailRoom::CLI do
     before :each do
       cli.configuration = configuration
       cli.coordinator = coordinator
+      cli.stubs(:exit)
     end
 
     it 'starts running the coordinator' do
-      cli.start
+      coordinator.expects(:run)
 
-      expect(coordinator).to have_received(:run)
+      cli.start
     end
 
     context 'on error' do
-      let(:error_message) { "oh noes!" }
-      let(:coordinator) { OpenStruct.new(run: true, quit: true) }
+      let(:error) { RuntimeError.new("oh noes!") }
+      let(:coordinator) { stub(run: true, quit: true) }
+      let(:crash_handler) { stub(handle: nil) }
 
       before do
         cli.instance_variable_set(:@options, {exit_error_format: error_format})
-        coordinator.stubs(:run).raises(RuntimeError, error_message)
+        coordinator.stubs(:run).raises(error)
+        MailRoom::CrashHandler.stubs(:new).returns(crash_handler)
       end
 
       context 'json format provided' do
         let(:error_format) { 'json' }
 
         it 'passes onto CrashHandler' do
-          cli.start
+          crash_handler.expects(:handle).with(error, error_format)
 
-          expect(MailRoom::CrashHandler).to have_received(:new).with a_hash_including({format: error_format})
+          cli.start
         end
       end
     end

data/spec/lib/configuration_spec.rb

@@ -3,15 +3,19 @@ require 'spec_helper'
 describe MailRoom::Configuration do
   let(:config_path) {File.expand_path('../fixtures/test_config.yml', File.dirname(__FILE__))}
 
-  describe 'set_mailboxes' do
+  describe '#initalize' do
     context 'with config_path' do
-      let(:configuration) { MailRoom::Configuration.new(:config_path => config_path) }
+      let(:configuration) { MailRoom::Configuration.new(config_path: config_path) }
 
       it 'parses yaml into mailbox objects' do
         MailRoom::Mailbox.stubs(:new).returns('mailbox1', 'mailbox2')
 
         expect(configuration.mailboxes).to eq(['mailbox1', 'mailbox2'])
       end
+
+      it 'parses health check' do
+        expect(configuration.health_check).to be_a(MailRoom::HealthCheck)
+      end
     end
 
     context 'without config_path' do
@@ -19,10 +23,13 @@ describe MailRoom::Configuration do
 
       it 'sets mailboxes to an empty set' do
         MailRoom::Mailbox.stubs(:new)
+        MailRoom::Mailbox.expects(:new).never
 
         expect(configuration.mailboxes).to eq([])
+      end
 
-        expect(MailRoom::Mailbox).to have_received(:new).never
+      it 'sets the health check to nil' do
+        expect(configuration.health_check).to be_nil
       end
     end
   end

data/spec/lib/coordinator_spec.rb

@@ -3,20 +3,25 @@ require 'spec_helper'
 describe MailRoom::Coordinator do
   describe '#initialize' do
     it 'builds a watcher for each mailbox' do
-      MailRoom::MailboxWatcher.stubs(:new).returns('watcher1', 'watcher2')
+      MailRoom::MailboxWatcher.expects(:new).with('mailbox1').returns('watcher1')
+      MailRoom::MailboxWatcher.expects(:new).with('mailbox2').returns('watcher2')
 
       coordinator = MailRoom::Coordinator.new(['mailbox1', 'mailbox2'])
 
       expect(coordinator.watchers).to eq(['watcher1', 'watcher2'])
-
-      expect(MailRoom::MailboxWatcher).to have_received(:new).with('mailbox1')
-      expect(MailRoom::MailboxWatcher).to have_received(:new).with('mailbox2')
     end
 
     it 'makes no watchers when mailboxes is empty' do
       coordinator = MailRoom::Coordinator.new([])
       expect(coordinator.watchers).to eq([])
     end
+
+    it 'sets the health check' do
+      health_check = MailRoom::HealthCheck.new({ address: '127.0.0.1', port: 8080})
+      coordinator = MailRoom::Coordinator.new([], health_check)
+
+      expect(coordinator.health_check).to eq(health_check)
+    end
   end
 
   describe '#run' do
@@ -24,27 +29,37 @@ describe MailRoom::Coordinator do
       watcher = stub
       watcher.stubs(:run)
       watcher.stubs(:quit)
+
+      health_check = stub
+      health_check.stubs(:run)
+      health_check.stubs(:quit)
+
       MailRoom::MailboxWatcher.stubs(:new).returns(watcher)
-      coordinator = MailRoom::Coordinator.new(['mailbox1'])
+      coordinator = MailRoom::Coordinator.new(['mailbox1'], health_check)
       coordinator.stubs(:sleep_while_running)
+      watcher.expects(:run)
+      watcher.expects(:quit)
+      health_check.expects(:run)
+      health_check.expects(:quit)
+
       coordinator.run
-      expect(watcher).to have_received(:run)
-      expect(watcher).to have_received(:quit)
     end
-    
+
     it 'should go to sleep after running watchers' do
       coordinator = MailRoom::Coordinator.new([])
       coordinator.stubs(:running=)
       coordinator.stubs(:running?).returns(false)
+      coordinator.expects(:running=).with(true)
+      coordinator.expects(:running?)
+
       coordinator.run
-      expect(coordinator).to have_received(:running=).with(true)
-      expect(coordinator).to have_received(:running?)
     end
 
     it 'should set attribute running to true' do
       coordinator = MailRoom::Coordinator.new([])
       coordinator.stubs(:sleep_while_running)
       coordinator.run
+
       expect(coordinator.running).to eq(true)
     end
   end
@@ -54,8 +69,9 @@ describe MailRoom::Coordinator do
       watcher = stub(:quit)
       MailRoom::MailboxWatcher.stubs(:new).returns(watcher)
       coordinator = MailRoom::Coordinator.new(['mailbox1'])
+      watcher.expects(:quit)
+
       coordinator.quit
-      expect(watcher).to have_received(:quit)
     end
   end
 end

data/spec/lib/crash_handler_spec.rb

@@ -4,21 +4,22 @@ describe MailRoom::CrashHandler do
 
   let(:error_message) { "oh noes!" }
   let(:error) { RuntimeError.new(error_message) }
+  let(:stdout) { StringIO.new }
 
   describe '#handle' do
 
-    subject{ described_class.new(error: error, format: format) }
+    subject{ described_class.new(stdout).handle(error, format) }
 
     context 'when given a json format' do
       let(:format) { 'json' }
-      let(:fake_json) do
-        { message: error_message }.to_json
-      end
 
-      it 'outputs the result of json to stdout' do
-        subject.stubs(:json).returns(fake_json)
+      it 'writes a json message to stdout' do
+        subject
+        stdout.rewind
+        output = stdout.read
 
-        expect{ subject.handle }.to output(/\"message\":\"#{error_message}\"/).to_stdout
+        expect(output).to end_with("\n")
+        expect(JSON.parse(output)['message']).to eq(error_message)
       end
     end
 
@@ -26,7 +27,7 @@ describe MailRoom::CrashHandler do
       let(:format) { "" }
 
       it 'raises an error as designed' do
-        expect{ subject.handle }.to raise_error(error.class, error_message)
+        expect{ subject }.to raise_error(error.class, error_message)
       end
     end
 
@@ -34,7 +35,7 @@ describe MailRoom::CrashHandler do
       let(:format) { "nonsense" }
 
       it 'raises an error as designed' do
-        expect{ subject.handle }.to raise_error(error.class, error_message)
+        expect{ subject }.to raise_error(error.class, error_message)
       end
     end
   end

data/spec/lib/delivery/letter_opener_spec.rb

@@ -3,27 +3,31 @@ require 'mail_room/delivery/letter_opener'
 
 describe MailRoom::Delivery::LetterOpener do
   describe '#deliver' do
-    let(:mailbox) {build_mailbox(:location => '/tmp/somewhere')}
+    let(:mailbox) {build_mailbox(location: '/tmp/somewhere')}
     let(:delivery_method) {stub(:deliver!)}
     let(:mail) {stub}
 
     before :each do
       Mail.stubs(:read_from_string).returns(mail)
       ::LetterOpener::DeliveryMethod.stubs(:new).returns(delivery_method)
-
-      MailRoom::Delivery::LetterOpener.new(mailbox).deliver('a message')
     end
 
     it 'creates a new LetterOpener::DeliveryMethod' do
-      expect(::LetterOpener::DeliveryMethod).to have_received(:new).with(:location => '/tmp/somewhere')
+      ::LetterOpener::DeliveryMethod.expects(:new).with(location: '/tmp/somewhere').returns(delivery_method)
+
+      MailRoom::Delivery::LetterOpener.new(mailbox).deliver('a message')
     end
 
     it 'parses the message string with Mail' do
-      expect(::Mail).to have_received(:read_from_string).with('a message')
+      ::Mail.expects(:read_from_string).with('a message')
+
+      MailRoom::Delivery::LetterOpener.new(mailbox).deliver('a message')
     end
 
     it 'delivers the mail message' do
-      expect(delivery_method).to have_received(:deliver!).with(mail)
+      delivery_method.expects(:deliver!).with(mail)
+
+      MailRoom::Delivery::LetterOpener.new(mailbox).deliver('a message')
     end
   end
 end