gitlab-mail_room 0.0.7 → 0.0.12

data/lib/mail_room/mailbox_watcher.rb

@@ -1,3 +1,5 @@
+require "mail_room/connection"
+
 module MailRoom
   # TODO: split up between processing and idling?
 
@@ -49,14 +51,25 @@ module MailRoom
         @connection = nil
       end
 
+      @mailbox.logger.info({ context: @mailbox.context, action: "Terminating watching thread..." })
+
       if self.watching_thread
-        self.watching_thread.join
+        thr = self.watching_thread.join(60)
+        @mailbox.logger.info({ context: @mailbox.context, action: "Timeout waiting for watching thread" }) unless thr
       end
+
+      @mailbox.logger.info({ context: @mailbox.context, action: "Done with thread cleanup" })
     end
 
     private
+
     def connection
-      @connection ||= Connection.new(@mailbox)
+      @connection ||=
+        if @mailbox.microsoft_graph?
+          ::MailRoom::MicrosoftGraph::Connection.new(@mailbox)
+        else
+          ::MailRoom::IMAP::Connection.new(@mailbox)
+        end
     end
   end
 end

data/lib/mail_room/message.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module MailRoom
+  class Message
+    attr_reader :uid, :body
+
+    def initialize(uid:, body:)
+      @uid = uid
+      @body = body
+    end
+
+    def ==(other)
+      self.class == other.class && uid == other.uid && body == other.body
+    end
+  end
+end

data/lib/mail_room/microsoft_graph.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module MailRoom
+  module MicrosoftGraph
+    autoload :Connection, 'mail_room/microsoft_graph/connection'
+  end
+end

data/lib/mail_room/microsoft_graph/connection.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'oauth2'
+
+module MailRoom
+  module MicrosoftGraph
+    class Connection < MailRoom::Connection
+      SCOPE = 'https://graph.microsoft.com/.default'
+      NEXT_PAGE_KEY = '@odata.nextLink'
+      DEFAULT_POLL_INTERVAL_S = 60
+
+      TooManyRequestsError = Class.new(RuntimeError)
+
+      attr_accessor :token, :throttled_count
+
+      def initialize(mailbox)
+        super
+
+        reset
+        setup
+      end
+
+      def wait
+        process_mailbox
+
+        @throttled_count = 0
+        wait_for_new_messages
+      rescue TooManyRequestsError => e
+        @throttled_count += 1
+
+        @mailbox.logger.warn({ context: @mailbox.context, action: 'Too many requests, backing off...', backoff_s: backoff_secs, error: e.message, error_backtrace: e.backtrace })
+
+        backoff
+      rescue IOError => e
+        @mailbox.logger.warn({ context: @mailbox.context, action: 'Disconnected. Resetting...', error: e.message, error_backtrace: e.backtrace })
+
+        reset
+        setup
+      end
+
+      private
+
+      def wait_for_new_messages
+        sleep poll_interval
+      end
+
+      def backoff
+        sleep backoff_secs
+      end
+
+      def backoff_secs
+        [60 * 10, 2**throttled_count].min
+      end
+
+      def reset
+        @token = nil
+        @throttled_count = 0
+      end
+
+      def setup
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Retrieving OAuth2 token...' })
+
+        @token = client.client_credentials.get_token({ scope: SCOPE })
+      end
+
+      def client
+        @client ||= OAuth2::Client.new(client_id, client_secret,
+                                       site: 'https://login.microsoftonline.com',
+                                       authorize_url: "/#{tenant_id}/oauth2/v2.0/authorize",
+                                       token_url: "/#{tenant_id}/oauth2/v2.0/token",
+                                       auth_scheme: :basic_auth)
+      end
+
+      def inbox_options
+        mailbox.inbox_options
+      end
+
+      def tenant_id
+        inbox_options[:tenant_id]
+      end
+
+      def client_id
+        inbox_options[:client_id]
+      end
+
+      def client_secret
+        inbox_options[:client_secret]
+      end
+
+      def poll_interval
+        @poll_interval ||= begin
+          interval = inbox_options[:poll_interval].to_i
+
+          if interval.positive?
+            interval
+          else
+            DEFAULT_POLL_INTERVAL_S
+          end
+        end
+      end
+
+      def process_mailbox
+        return unless @new_message_handler
+
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Processing started' })
+
+        new_messages.each do |msg|
+          success = @new_message_handler.call(msg)
+          handle_delivered(msg) if success
+        end
+      end
+
+      def handle_delivered(msg)
+        mark_as_read(msg)
+        delete_message(msg) if @mailbox.delete_after_delivery
+      end
+
+      def delete_message(msg)
+        token.delete(msg_url(msg.uid))
+      end
+
+      def mark_as_read(msg)
+        token.patch(msg_url(msg.uid),
+                    headers: { 'Content-Type' => 'application/json' },
+                    body: { isRead: true }.to_json)
+      end
+
+      def new_messages
+        messages_for_ids(new_message_ids)
+      end
+
+      # Yields a page of message IDs at a time
+      def new_message_ids
+        url = unread_messages_url
+
+        Enumerator.new do |block|
+          loop do
+            messages, next_page_url = unread_messages(url: url)
+            messages.each { |msg| block.yield msg }
+
+            break unless next_page_url
+
+            url = next_page_url
+          end
+        end
+      end
+
+      def unread_messages(url:)
+        body = get(url)
+
+        return [[], nil] unless body
+
+        all_unread = body['value'].map { |msg| msg['id'] }
+        to_deliver = all_unread.select { |uid| @mailbox.deliver?(uid) }
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Getting new messages',
+                               unread: { count: all_unread.count, ids: all_unread },
+                               to_be_delivered: { count: to_deliver.count, ids: to_deliver } })
+        [to_deliver, body[NEXT_PAGE_KEY]]
+      rescue TypeError, JSON::ParserError => e
+        log_exception('Error parsing JSON response', e)
+        [[], nil]
+      end
+
+      # Returns the JSON response
+      def get(url)
+        response = token.get(url, { raise_errors: false })
+
+        # https://docs.microsoft.com/en-us/graph/errors
+        case response.status
+        when 509, 429
+          raise TooManyRequestsError
+        when 400..599
+          raise OAuth2::Error, response
+        end
+
+        return unless response.body
+
+        body = JSON.parse(response.body)
+
+        raise TypeError, 'Response did not contain value hash' unless body.is_a?(Hash) && body.key?('value')
+
+        body
+      end
+
+      def messages_for_ids(message_ids)
+        message_ids.each_with_object([]) do |id, arr|
+          response = token.get(rfc822_msg_url(id))
+
+          arr << ::MailRoom::Message.new(uid: id, body: response.body)
+        end
+      end
+
+      def base_url
+        "https://graph.microsoft.com/v1.0/users/#{mailbox.email}/mailFolders/#{mailbox.name}/messages"
+      end
+
+      def unread_messages_url
+        "#{base_url}?$filter=isRead eq false"
+      end
+
+      def msg_url(id)
+        # Attempting to use the base_url fails with "The OData request is not supported"
+        "https://graph.microsoft.com/v1.0/users/#{mailbox.email}/messages/#{id}"
+      end
+
+      def rfc822_msg_url(id)
+        # Attempting to use the base_url fails with "The OData request is not supported"
+        "#{msg_url(id)}/$value"
+      end
+
+      def log_exception(message, exception)
+        @mailbox.logger.warn({ context: @mailbox.context, message: message, exception: exception.to_s })
+      end
+    end
+  end
+end

data/lib/mail_room/version.rb

@@ -1,4 +1,4 @@
 module MailRoom
   # Current version of gitlab-mail_room gem
-  VERSION = "0.0.7"
+  VERSION = "0.0.12"
 end

data/mail_room.gemspec

@@ -17,10 +17,15 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
+  gem.add_dependency             "net-imap", ">= 0.2.1"
+  gem.add_dependency             "oauth2", "~> 1.4.4"
+
   gem.add_development_dependency "rake"
   gem.add_development_dependency "rspec", "~> 3.9"
+  gem.add_development_dependency "rubocop", "~> 1.11"
   gem.add_development_dependency "mocha", "~> 1.11"
   gem.add_development_dependency "simplecov"
+  gem.add_development_dependency "webrick", "~> 1.6"
 
   # for testing delivery methods
   gem.add_development_dependency "faraday"
@@ -30,4 +35,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "redis-namespace"
   gem.add_development_dependency "pg"
   gem.add_development_dependency "charlock_holmes"
+  gem.add_development_dependency "webmock"
 end

data/spec/fixtures/test_config.yml

@@ -1,4 +1,7 @@
 ---
+:health_check:
+  :address: "127.0.0.1"
+  :port: 8080
 :mailboxes:
   -
     :email: "user1@gmail.com"

data/spec/lib/cli_spec.rb

@@ -2,22 +2,22 @@ require 'spec_helper'
 
 describe MailRoom::CLI do
   let(:config_path) {File.expand_path('../fixtures/test_config.yml', File.dirname(__FILE__))}
-  let!(:configuration) {MailRoom::Configuration.new({:config_path => config_path})}
-  let(:coordinator) {stub(:run => true, :quit => true)}
+  let!(:configuration) {MailRoom::Configuration.new({config_path: config_path})}
+  let(:coordinator) {stub(run: true, quit: true)}
   let(:configuration_args) { anything }
-  let(:coordinator_args) { anything }
+  let(:coordinator_args) { [anything, anything] }
 
   describe '.new' do
     let(:args) {["-c", "a path"]}
 
     before :each do
       MailRoom::Configuration.expects(:new).with(configuration_args).returns(configuration)
-      MailRoom::Coordinator.stubs(:new).with(coordinator_args).returns(coordinator)
+      MailRoom::Coordinator.stubs(:new).with(*coordinator_args).returns(coordinator)
     end
 
     context 'with configuration args' do
       let(:configuration_args) do
-        {:config_path => 'a path'}
+        {config_path: 'a path'}
       end
 
       it 'parses arguments into configuration' do
@@ -27,7 +27,7 @@ describe MailRoom::CLI do
 
     context 'with coordinator args' do
       let(:coordinator_args) do
-        configuration.mailboxes
+        [configuration.mailboxes, anything]
       end
 
       it 'creates a new coordinator with configuration' do

data/spec/lib/configuration_spec.rb

@@ -3,15 +3,19 @@ require 'spec_helper'
 describe MailRoom::Configuration do
   let(:config_path) {File.expand_path('../fixtures/test_config.yml', File.dirname(__FILE__))}
 
-  describe 'set_mailboxes' do
+  describe '#initalize' do
     context 'with config_path' do
-      let(:configuration) { MailRoom::Configuration.new(:config_path => config_path) }
+      let(:configuration) { MailRoom::Configuration.new(config_path: config_path) }
 
       it 'parses yaml into mailbox objects' do
         MailRoom::Mailbox.stubs(:new).returns('mailbox1', 'mailbox2')
 
         expect(configuration.mailboxes).to eq(['mailbox1', 'mailbox2'])
       end
+
+      it 'parses health check' do
+        expect(configuration.health_check).to be_a(MailRoom::HealthCheck)
+      end
     end
 
     context 'without config_path' do
@@ -23,6 +27,10 @@ describe MailRoom::Configuration do
 
         expect(configuration.mailboxes).to eq([])
       end
+
+      it 'sets the health check to nil' do
+        expect(configuration.health_check).to be_nil
+      end
     end
   end
 end

data/spec/lib/coordinator_spec.rb

@@ -15,6 +15,13 @@ describe MailRoom::Coordinator do
       coordinator = MailRoom::Coordinator.new([])
       expect(coordinator.watchers).to eq([])
     end
+
+    it 'sets the health check' do
+      health_check = MailRoom::HealthCheck.new({ address: '127.0.0.1', port: 8080})
+      coordinator = MailRoom::Coordinator.new([], health_check)
+
+      expect(coordinator.health_check).to eq(health_check)
+    end
   end
 
   describe '#run' do
@@ -22,15 +29,22 @@ describe MailRoom::Coordinator do
       watcher = stub
       watcher.stubs(:run)
       watcher.stubs(:quit)
+
+      health_check = stub
+      health_check.stubs(:run)
+      health_check.stubs(:quit)
+
       MailRoom::MailboxWatcher.stubs(:new).returns(watcher)
-      coordinator = MailRoom::Coordinator.new(['mailbox1'])
+      coordinator = MailRoom::Coordinator.new(['mailbox1'], health_check)
       coordinator.stubs(:sleep_while_running)
       watcher.expects(:run)
       watcher.expects(:quit)
+      health_check.expects(:run)
+      health_check.expects(:quit)
 
       coordinator.run
     end
-    
+
     it 'should go to sleep after running watchers' do
       coordinator = MailRoom::Coordinator.new([])
       coordinator.stubs(:running=)

data/spec/lib/delivery/letter_opener_spec.rb

@@ -3,7 +3,7 @@ require 'mail_room/delivery/letter_opener'
 
 describe MailRoom::Delivery::LetterOpener do
   describe '#deliver' do
-    let(:mailbox) {build_mailbox(:location => '/tmp/somewhere')}
+    let(:mailbox) {build_mailbox(location: '/tmp/somewhere')}
     let(:delivery_method) {stub(:deliver!)}
     let(:mail) {stub}
 
@@ -13,7 +13,7 @@ describe MailRoom::Delivery::LetterOpener do
     end
 
     it 'creates a new LetterOpener::DeliveryMethod' do
-      ::LetterOpener::DeliveryMethod.expects(:new).with(:location => '/tmp/somewhere').returns(delivery_method)
+      ::LetterOpener::DeliveryMethod.expects(:new).with(location: '/tmp/somewhere').returns(delivery_method)
 
       MailRoom::Delivery::LetterOpener.new(mailbox).deliver('a message')
     end

data/spec/lib/delivery/logger_spec.rb

@@ -16,7 +16,7 @@ describe MailRoom::Delivery::Logger do
     end
 
     context "with a log path" do
-      let(:mailbox) {build_mailbox(:log_path => '/var/log/mail-room.log')}
+      let(:mailbox) {build_mailbox(log_path: '/var/log/mail-room.log')}
 
       it 'creates a new file to append to' do
         file = stub(:sync=)