gitlab-mail_room 0.0.3 → 0.0.9

data/lib/mail_room/health_check.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module MailRoom
+  class HealthCheck
+    attr_reader :address, :port, :running
+
+    def initialize(attributes = {})
+      @address = attributes[:address]
+      @port = attributes[:port]
+
+      validate!
+    end
+
+    def run
+      @server = create_server
+
+      @thread = Thread.new do
+        @server.start
+      end
+
+      @thread.abort_on_exception = true
+      @running = true
+    end
+
+    def quit
+      @running = false
+      @server&.shutdown
+      @thread&.join(60)
+    end
+
+    private
+
+    def validate!
+      raise 'No health check address specified' unless address
+      raise "Health check port #{@port.to_i} is invalid" unless port.to_i.positive?
+    end
+
+    def create_server
+      require 'webrick'
+
+      server = ::WEBrick::HTTPServer.new(Port: port, BindAddress: address, AccessLog: [])
+
+      server.mount_proc '/liveness' do |_req, res|
+        handle_liveness(res)
+      end
+
+      server
+    end
+
+    def handle_liveness(res)
+      if @running
+        res.status = 200
+        res.body = "OK\n"
+      else
+        res.status = 500
+        res.body = "Not running\n"
+      end
+    end
+  end
+end

data/lib/mail_room/imap.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+ 
+module MailRoom
+  module IMAP
+    autoload :Connection, 'mail_room/imap/connection'
+    autoload :Message, 'mail_room/imap/message'
+  end
+end

data/lib/mail_room/imap/connection.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+module MailRoom
+  module IMAP
+    class Connection < MailRoom::Connection
+      def initialize(mailbox)
+        super
+
+        # log in and set the mailbox
+        reset
+        setup
+      end
+
+      # is the connection logged in?
+      # @return [Boolean]
+      def logged_in?
+        @logged_in
+      end
+
+      # is the connection blocked idling?
+      # @return [Boolean]
+      def idling?
+        @idling
+      end
+
+      # is the imap connection closed?
+      # @return [Boolean]
+      def disconnected?
+        imap.disconnected?
+      end
+
+      # is the connection ready to idle?
+      # @return [Boolean]
+      def ready_to_idle?
+        logged_in? && !idling?
+      end
+
+      def quit
+        stop_idling
+        reset
+      end
+
+      def wait
+        # in case we missed any between idles
+        process_mailbox
+
+        idle
+
+        process_mailbox
+      rescue Net::IMAP::Error, IOError => e
+        @mailbox.logger.warn({ context: @mailbox.context, action: 'Disconnected. Resetting...', error: e.message })
+        reset
+        setup
+      end
+
+      private
+
+      def reset
+        @imap = nil
+        @logged_in = false
+        @idling = false
+      end
+
+      def setup
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Starting TLS session' })
+        start_tls
+
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Logging into mailbox' })
+        log_in
+
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Setting mailbox' })
+        set_mailbox
+      end
+
+      # build a net/imap connection to google imap
+      def imap
+        @imap ||= Net::IMAP.new(@mailbox.host, port: @mailbox.port, ssl: @mailbox.ssl_options)
+      end
+
+      # start a TLS session
+      def start_tls
+        imap.starttls if @mailbox.start_tls
+      end
+
+      # send the imap login command to google
+      def log_in
+        imap.login(@mailbox.email, @mailbox.password)
+        @logged_in = true
+      end
+
+      # select the mailbox name we want to use
+      def set_mailbox
+        imap.select(@mailbox.name) if logged_in?
+      end
+
+      # is the response for a new message?
+      # @param response [Net::IMAP::TaggedResponse] the imap response from idle
+      # @return [Boolean]
+      def message_exists?(response)
+        response.respond_to?(:name) && response.name == 'EXISTS'
+      end
+
+      # @private
+      def idle_handler
+        ->(response) { imap.idle_done if message_exists?(response) }
+      end
+
+      # maintain an imap idle connection
+      def idle
+        return unless ready_to_idle?
+
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Idling' })
+        @idling = true
+
+        imap.idle(@mailbox.idle_timeout, &idle_handler)
+      ensure
+        @idling = false
+      end
+
+      # trigger the idle to finish and wait for the thread to finish
+      def stop_idling
+        return unless idling?
+
+        imap.idle_done
+
+        # idling_thread.join
+        # self.idling_thread = nil
+      end
+
+      def process_mailbox
+        return unless @new_message_handler
+
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Processing started' })
+
+        msgs = new_messages
+        any_deletions = msgs.
+                        # deliver each new message, collect success
+                        map(&@new_message_handler).
+                        # include messages with success
+                        zip(msgs).
+                        # filter failed deliveries, collect message
+                        select(&:first).map(&:last).
+                        # scrub delivered messages
+                        map { |message| scrub(message) }
+                            .any?
+
+        imap.expunge if @mailbox.expunge_deleted && any_deletions
+      end
+
+      def scrub(message)
+        if @mailbox.delete_after_delivery
+          imap.store(message.seqno, '+FLAGS', [Net::IMAP::DELETED])
+          true
+        end
+      end
+
+      # @private
+      # fetch all messages for the new message ids
+      def new_messages
+        # Both of these calls may results in
+        #   imap raising an EOFError, we handle
+        #   this exception in the watcher
+        messages_for_ids(new_message_ids)
+      end
+
+      # TODO: label messages?
+      #   @imap.store(id, "+X-GM-LABELS", [label])
+
+      # @private
+      # search for all new (unseen) message ids
+      # @return [Array<Integer>] message ids
+      def new_message_ids
+        # uid_search still leaves messages UNSEEN
+        all_unread = imap.uid_search(@mailbox.search_command)
+
+        all_unread = all_unread.slice(0, @mailbox.limit_max_unread) if @mailbox.limit_max_unread.to_i > 0
+
+        to_deliver = all_unread.select { |uid| @mailbox.deliver?(uid) }
+        @mailbox.logger.info({ context: @mailbox.context, action: 'Getting new messages',
+                               unread: { count: all_unread.count, ids: all_unread }, to_be_delivered: { count: to_deliver.count, ids: to_deliver } })
+        to_deliver
+      end
+
+      # @private
+      # fetch the email for all given ids in RFC822 format
+      # @param ids [Array<Integer>] list of message ids
+      # @return [Array<MailRoom::IMAP::Message>] the net/imap messages for the given ids
+      def messages_for_ids(uids)
+        return [] if uids.empty?
+
+        # uid_fetch marks as SEEN, will not be re-fetched for UNSEEN
+        imap_messages = imap.uid_fetch(uids, 'RFC822')
+
+        imap_messages.each_with_object([]) do |msg, messages|
+          messages << ::MailRoom::IMAP::Message.new(uid: msg.attr['UID'], body: msg.attr['RFC822'], seqno: msg.seqno)
+        end
+      end
+    end
+  end
+end

data/lib/mail_room/imap/message.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal:true
+
+module MailRoom
+  module IMAP
+    class Message < MailRoom::Message
+      attr_reader :seqno
+
+      def initialize(uid:, body:, seqno:)
+        super(uid: uid, body: body)
+
+        @seqno = seqno
+      end
+
+      def ==(other)
+        super && seqno == other.seqno
+      end
+    end
+  end
+end

data/lib/mail_room/mailbox.rb

@@ -1,5 +1,6 @@
 require "mail_room/delivery"
 require "mail_room/arbitration"
+require "mail_room/imap"
 
 module MailRoom
   # Mailbox Configuration fields
@@ -10,6 +11,7 @@ module MailRoom
     :port,
     :ssl,
     :start_tls,
+    :limit_max_unread, #to avoid 'Error in IMAP command UID FETCH: Too long argument'
     :idle_timeout,
     :search_command,
     :name,
@@ -50,6 +52,7 @@ module MailRoom
       :port => 993,
       :ssl => true,
       :start_tls => false,
+      :limit_max_unread => 0,
       :idle_timeout => IMAP_IDLE_TIMEOUT,
       :delete_after_delivery => false,
       :expunge_deleted => false,
@@ -100,13 +103,13 @@ module MailRoom
       arbitrator.deliver?(uid)
     end
 
-    # deliver the imap email message
-    # @param message [Net::IMAP::FetchData]
+    # deliver the email message
+    # @param message [MailRoom::Message]
     def deliver(message)
-      body = message.attr['RFC822']
+      body = message.body
       return true unless body
 
-      logger.info({context: context, uid: message.attr['UID'], action: "sending to deliverer", deliverer: delivery.class.name, byte_size: message.attr['RFC822.SIZE']})
+      logger.info({context: context, uid: message.uid, action: "sending to deliverer", deliverer: delivery.class.name, byte_size: body.bytesize})
       delivery.deliver(body)
     end
 

data/lib/mail_room/mailbox_watcher.rb

@@ -1,3 +1,5 @@
+require "mail_room/connection"
+
 module MailRoom
   # TODO: split up between processing and idling?
 
@@ -49,14 +51,19 @@ module MailRoom
         @connection = nil
       end
 
+      @mailbox.logger.info({ context: @mailbox.context, action: "Terminating watching thread..." })
+
       if self.watching_thread
-        self.watching_thread.join
+        thr = self.watching_thread.join(60)
+        @mailbox.logger.info({ context: @mailbox.context, action: "Timeout waiting for watching thread" }) unless thr
       end
+
+      @mailbox.logger.info({ context: @mailbox.context, action: "Done with thread cleanup" })
     end
 
     private
     def connection
-      @connection ||= Connection.new(@mailbox)
+      @connection ||= ::MailRoom::IMAP::Connection.new(@mailbox)
     end
   end
 end

data/lib/mail_room/message.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module MailRoom
+  class Message
+    attr_reader :uid, :body
+
+    def initialize(uid:, body:)
+      @uid = uid
+      @body = body
+    end
+
+    def ==(other)
+      self.class == other.class && uid == other.uid && body == other.body
+    end
+  end
+end

data/lib/mail_room/version.rb

@@ -1,4 +1,4 @@
 module MailRoom
-  # Current version of MailRoom gem
-  VERSION = "0.0.3"
+  # Current version of gitlab-mail_room gem
+  VERSION = "0.0.9"
 end

data/mail_room.gemspec

@@ -18,10 +18,10 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_development_dependency "rake"
-  gem.add_development_dependency "rspec"
-  gem.add_development_dependency "mocha"
-  gem.add_development_dependency "bourne"
+  gem.add_development_dependency "rspec", "~> 3.9"
+  gem.add_development_dependency "mocha", "~> 1.11"
   gem.add_development_dependency "simplecov"
+  gem.add_development_dependency "webrick", "~> 1.6"
 
   # for testing delivery methods
   gem.add_development_dependency "faraday"

data/spec/fixtures/test_config.yml

@@ -1,4 +1,7 @@
 ---
+:health_check:
+  :address: "127.0.0.1"
+  :port: 8080
 :mailboxes:
   -
     :email: "user1@gmail.com"

data/spec/lib/arbitration/redis_spec.rb

@@ -5,7 +5,8 @@ describe MailRoom::Arbitration::Redis do
   let(:mailbox) {
     build_mailbox(
       arbitration_options: {
-        namespace: "mail_room"
+        namespace: "mail_room",
+        redis_url: ENV['REDIS_URL']
       }
     )
   }
@@ -78,7 +79,7 @@ describe MailRoom::Arbitration::Redis do
 
   context 'redis client connection params' do
     context 'when only url is present' do
-      let(:redis_url) { "redis://localhost:6379" }
+      let(:redis_url) { ENV.fetch('REDIS_URL', 'redis://localhost:6379') }
       let(:mailbox) {
         build_mailbox(
           arbitration_options: {

data/spec/lib/cli_spec.rb

@@ -4,23 +4,35 @@ describe MailRoom::CLI do
   let(:config_path) {File.expand_path('../fixtures/test_config.yml', File.dirname(__FILE__))}
   let!(:configuration) {MailRoom::Configuration.new({:config_path => config_path})}
   let(:coordinator) {stub(:run => true, :quit => true)}
+  let(:configuration_args) { anything }
+  let(:coordinator_args) { [anything, anything] }
 
   describe '.new' do
     let(:args) {["-c", "a path"]}
 
     before :each do
-      MailRoom::Configuration.stubs(:new).returns(configuration)
-      MailRoom::Coordinator.stubs(:new).returns(coordinator)
+      MailRoom::Configuration.expects(:new).with(configuration_args).returns(configuration)
+      MailRoom::Coordinator.stubs(:new).with(*coordinator_args).returns(coordinator)
     end
 
-    it 'parses arguments into configuration' do
-      expect(MailRoom::CLI.new(args).configuration).to eq(configuration)
-      expect(MailRoom::Configuration).to have_received(:new).with({:config_path => 'a path'})
+    context 'with configuration args' do
+      let(:configuration_args) do
+        {:config_path => 'a path'}
+      end
+
+      it 'parses arguments into configuration' do
+        expect(MailRoom::CLI.new(args).configuration).to eq configuration
+      end
     end
 
-    it 'creates a new coordinator with configuration' do
-      expect(MailRoom::CLI.new(args).coordinator).to eq(coordinator)
-      expect(MailRoom::Coordinator).to have_received(:new).with(configuration.mailboxes)
+    context 'with coordinator args' do
+      let(:coordinator_args) do
+        [configuration.mailboxes, anything]
+      end
+
+      it 'creates a new coordinator with configuration' do
+        expect(MailRoom::CLI.new(args).coordinator).to eq(coordinator)
+      end
     end
   end
 
@@ -30,30 +42,33 @@ describe MailRoom::CLI do
     before :each do
       cli.configuration = configuration
       cli.coordinator = coordinator
+      cli.stubs(:exit)
     end
 
     it 'starts running the coordinator' do
-      cli.start
+      coordinator.expects(:run)
 
-      expect(coordinator).to have_received(:run)
+      cli.start
     end
 
     context 'on error' do
-      let(:error_message) { "oh noes!" }
-      let(:coordinator) { OpenStruct.new(run: true, quit: true) }
+      let(:error) { RuntimeError.new("oh noes!") }
+      let(:coordinator) { stub(run: true, quit: true) }
+      let(:crash_handler) { stub(handle: nil) }
 
       before do
         cli.instance_variable_set(:@options, {exit_error_format: error_format})
-        coordinator.stubs(:run).raises(RuntimeError, error_message)
+        coordinator.stubs(:run).raises(error)
+        MailRoom::CrashHandler.stubs(:new).returns(crash_handler)
       end
 
       context 'json format provided' do
         let(:error_format) { 'json' }
 
         it 'passes onto CrashHandler' do
-          cli.start
+          crash_handler.expects(:handle).with(error, error_format)
 
-          expect(MailRoom::CrashHandler).to have_received(:new).with a_hash_including({format: error_format})
+          cli.start
         end
       end
     end