gitlab-mail_room 0.0.2

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/mail_room

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'mail_room'
+
+MailRoom::CLI.new(ARGV).start

data/lib/mail_room.rb

@@ -0,0 +1,16 @@
+require 'net/imap'
+require 'optparse'
+require 'yaml'
+
+module MailRoom
+end
+
+require "mail_room/version"
+require "mail_room/configuration"
+require "mail_room/mailbox"
+require "mail_room/mailbox_watcher"
+require "mail_room/connection"
+require "mail_room/coordinator"
+require "mail_room/cli"
+require 'mail_room/logger/structured'
+require 'mail_room/crash_handler'

data/lib/mail_room/arbitration.rb

@@ -0,0 +1,16 @@
+module MailRoom
+  module Arbitration
+    def [](name)
+      require_relative("./arbitration/#{name}")
+
+      case name
+      when "redis"
+        Arbitration::Redis
+      else
+        Arbitration::Noop
+      end
+    end
+
+    module_function :[]
+  end
+end

data/lib/mail_room/arbitration/noop.rb

@@ -0,0 +1,18 @@
+module MailRoom
+  module Arbitration
+    class Noop
+      Options = Class.new do
+        def initialize(*)
+          super()
+        end
+      end
+
+      def initialize(*)
+      end
+
+      def deliver?(*)
+        true
+      end
+    end
+  end
+end

data/lib/mail_room/arbitration/redis.rb

@@ -0,0 +1,58 @@
+require "redis"
+
+module MailRoom
+  module Arbitration
+    class Redis
+      Options = Struct.new(:redis_url, :namespace, :sentinels) do
+        def initialize(mailbox)
+          redis_url = mailbox.arbitration_options[:redis_url] || "redis://localhost:6379"
+          namespace = mailbox.arbitration_options[:namespace]
+          sentinels = mailbox.arbitration_options[:sentinels]
+
+          super(redis_url, namespace, sentinels)
+        end
+      end
+
+      # Expire after 10 minutes so Redis doesn't get filled up with outdated data.
+      EXPIRATION = 600
+
+      attr_accessor :options
+
+      def initialize(options)
+        @options = options
+      end
+
+      def deliver?(uid, expiration = EXPIRATION)
+        key = "delivered:#{uid}"
+
+        # Set the key, but only if it doesn't already exist;
+        # the return value is true if successful, false if the key was already set,
+        # which is conveniently the correct return value for this method
+        # Any subsequent failure in the instance which gets the lock will be dealt
+        # with by the expiration, at which time another instance can pick up the
+        # message and try again.
+        client.set(key, 1, {:nx => true, :ex => expiration})
+      end
+
+      private
+
+      def client
+        @client ||= begin
+          sentinels = options.sentinels
+          redis_options = { url: options.redis_url }
+          redis_options[:sentinels] = sentinels if sentinels
+
+          redis = ::Redis.new(redis_options)
+
+          namespace = options.namespace
+          if namespace
+            require 'redis/namespace'
+            ::Redis::Namespace.new(namespace, redis: redis)
+          else
+            redis
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mail_room/cli.rb

@@ -0,0 +1,62 @@
+module MailRoom
+  # The CLI parses ARGV into configuration to start the coordinator with.
+  # @author Tony Pitale
+  class CLI
+    attr_accessor :configuration, :coordinator, :options
+
+    # Initialize a new CLI instance to handle option parsing from arguments
+    #   into configuration to start the coordinator running on all mailboxes
+    #
+    # @param args [Array] `ARGV` passed from `bin/mail_room`
+    def initialize(args)
+      @options = {}
+
+      OptionParser.new do |parser|
+        parser.banner = [
+          "Usage: #{@name} [-c config_file]\n",
+          "       #{@name} --help\n"
+        ].compact.join
+
+        parser.on('-c', '--config FILE') do |path|
+          options[:config_path] = path
+        end
+
+        parser.on('-q', '--quiet') do
+          options[:quiet] = true
+        end
+
+        parser.on('--log-exit-as') do |format|
+          options[:exit_error_format] = 'json' unless format.nil?
+        end
+
+        # parser.on("-l", "--log FILE") do |path|
+        #   options[:log_path] = path
+        # end
+
+        parser.on_tail("-?", "--help", "Display this usage information.") do
+          puts "#{parser}\n"
+          exit
+        end
+      end.parse!(args)
+
+      self.configuration = Configuration.new(options)
+      self.coordinator = Coordinator.new(configuration.mailboxes)
+    end
+
+    # Start the coordinator running, sets up signal traps
+    def start
+      Signal.trap(:INT) do
+        coordinator.running = false
+      end
+
+      Signal.trap(:TERM) do
+        exit
+      end
+
+      coordinator.run
+    rescue Exception => e # not just Errors, but includes lower-level Exceptions
+      CrashHandler.new(error: e, format: @options[:exit_error_format]).handle
+      exit
+    end
+  end
+end

data/lib/mail_room/configuration.rb

@@ -0,0 +1,36 @@
+require "erb"
+
+module MailRoom
+  # Wraps configuration for a set of individual mailboxes with global config
+  # @author Tony Pitale
+  class Configuration
+    attr_accessor :mailboxes, :log_path, :quiet
+
+    # Initialize a new configuration of mailboxes
+    def initialize(options={})
+      self.mailboxes = []
+      self.quiet = options.fetch(:quiet, false)
+
+      if options.has_key?(:config_path)
+        begin
+          erb = ERB.new(File.read(options[:config_path]))
+          erb.filename = options[:config_path]
+          config_file = YAML.load(erb.result)
+
+          set_mailboxes(config_file[:mailboxes])
+        rescue => e
+          raise e unless quiet
+        end
+      end
+    end
+
+    # Builds individual mailboxes from YAML configuration
+    #
+    # @param mailboxes_config
+    def set_mailboxes(mailboxes_config)
+      mailboxes_config.each do |attributes|
+        self.mailboxes << Mailbox.new(attributes)
+      end
+    end
+  end
+end

data/lib/mail_room/connection.rb

@@ -0,0 +1,195 @@
+module MailRoom
+  class Connection
+    def initialize(mailbox)
+      @mailbox = mailbox
+
+      # log in and set the mailbox
+      reset
+      setup
+    end
+
+    def on_new_message(&block)
+      @new_message_handler = block
+    end
+
+    # is the connection logged in?
+    # @return [Boolean]
+    def logged_in?
+      @logged_in
+    end
+
+    # is the connection blocked idling?
+    # @return [Boolean]
+    def idling?
+      @idling
+    end
+
+    # is the imap connection closed?
+    # @return [Boolean]
+    def disconnected?
+      @imap.disconnected?
+    end
+
+    # is the connection ready to idle?
+    # @return [Boolean]
+    def ready_to_idle?
+      logged_in? && !idling?
+    end
+
+    def quit
+      stop_idling
+      reset
+    end
+
+    def wait
+      begin
+        # in case we missed any between idles
+        process_mailbox
+
+        idle
+
+        process_mailbox
+      rescue Net::IMAP::Error, IOError
+        @mailbox.logger.warn({ context: @mailbox.context, action: "Disconnected. Resetting..." })
+        reset
+        setup
+      end
+    end
+
+    private
+
+    def reset
+      @imap = nil
+      @logged_in = false
+      @idling = false
+    end
+
+    def setup
+      @mailbox.logger.info({ context: @mailbox.context, action: "Starting TLS session" })
+      start_tls
+
+      @mailbox.logger.info({ context: @mailbox.context, action: "Logging into mailbox" })
+      log_in
+
+      @mailbox.logger.info({ context: @mailbox.context, action: "Setting mailbox" })
+      set_mailbox
+    end
+
+    # build a net/imap connection to google imap
+    def imap
+      @imap ||= Net::IMAP.new(@mailbox.host, :port => @mailbox.port, :ssl => @mailbox.ssl_options)
+    end
+
+    # start a TLS session
+    def start_tls
+      imap.starttls if @mailbox.start_tls
+    end
+
+    # send the imap login command to google
+    def log_in
+      imap.login(@mailbox.email, @mailbox.password)
+      @logged_in = true
+    end
+
+    # select the mailbox name we want to use
+    def set_mailbox
+      imap.select(@mailbox.name) if logged_in?
+    end
+
+    # is the response for a new message?
+    # @param response [Net::IMAP::TaggedResponse] the imap response from idle
+    # @return [Boolean]
+    def message_exists?(response)
+      response.respond_to?(:name) && response.name == 'EXISTS'
+    end
+
+    # @private
+    def idle_handler
+      lambda {|response| imap.idle_done if message_exists?(response)}
+    end
+
+    # maintain an imap idle connection
+    def idle
+      return unless ready_to_idle?
+
+      @mailbox.logger.info({ context: @mailbox.context, action: "Idling" })
+      @idling = true
+
+      imap.idle(@mailbox.idle_timeout, &idle_handler)
+    ensure
+      @idling = false
+    end
+
+    # trigger the idle to finish and wait for the thread to finish
+    def stop_idling
+      return unless idling?
+
+      imap.idle_done
+
+      # idling_thread.join
+      # self.idling_thread = nil
+    end
+
+    def process_mailbox
+      return unless @new_message_handler
+      @mailbox.logger.info({ context: @mailbox.context, action: "Processing started" })
+
+      msgs = new_messages
+
+      any_deletions = msgs.
+                      # deliver each new message, collect success
+                      map(&@new_message_handler).
+                      # include messages with success
+                      zip(msgs).
+                      # filter failed deliveries, collect message
+                      select(&:first).map(&:last).
+                      # scrub delivered messages
+                      map { |message| scrub(message) }.
+                      any?
+
+      imap.expunge if @mailbox.expunge_deleted && any_deletions
+    end
+
+    def scrub(message)
+      if @mailbox.delete_after_delivery
+        imap.store(message.seqno, "+FLAGS", [Net::IMAP::DELETED])
+        true
+      end
+    end
+
+    # @private
+    # fetch all messages for the new message ids
+    def new_messages
+      # Both of these calls may results in
+      #   imap raising an EOFError, we handle
+      #   this exception in the watcher
+      messages_for_ids(new_message_ids)
+    end
+
+    # TODO: label messages?
+    #   @imap.store(id, "+X-GM-LABELS", [label])
+
+    # @private
+    # search for all new (unseen) message ids
+    # @return [Array<Integer>] message ids
+    def new_message_ids
+      # uid_search still leaves messages UNSEEN
+      all_unread = @imap.uid_search(@mailbox.search_command)
+
+      to_deliver = all_unread.select { |uid| @mailbox.deliver?(uid) }
+      @mailbox.logger.info({ context: @mailbox.context, action: "Getting new messages", unread: {count: all_unread.count, ids: all_unread}, to_be_delivered: { count: to_deliver.count, ids: all_unread } })
+      to_deliver
+    end
+
+    # @private
+    # fetch the email for all given ids in RFC822 format
+    # @param ids [Array<Integer>] list of message ids
+    # @return [Array<Net::IMAP::FetchData>] the net/imap messages for the given ids
+    def messages_for_ids(uids)
+      return [] if uids.empty?
+
+      # uid_fetch marks as SEEN, will not be re-fetched for UNSEEN
+      imap.uid_fetch(uids, "RFC822")
+    end
+  end
+end

data/lib/mail_room/coordinator.rb

@@ -0,0 +1,41 @@
+module MailRoom
+  # Coordinate the mailbox watchers
+  # @author Tony Pitale
+  class Coordinator
+    attr_accessor :watchers, :running
+
+    # build watchers for a set of mailboxes
+    # @params mailboxes [Array<MailRoom::Mailbox>] mailboxes to be watched
+    def initialize(mailboxes)
+      self.watchers = []
+
+      mailboxes.each {|box| self.watchers << MailboxWatcher.new(box)}
+    end
+
+    alias :running? :running
+
+    # start each of the watchers to running
+    def run
+      watchers.each(&:run)
+      
+      self.running = true
+      
+      sleep_while_running
+    ensure
+      quit
+    end
+
+    # quit each of the watchers when we're done running
+    def quit
+      watchers.each(&:quit)
+    end
+
+    private
+    # @private
+    def sleep_while_running
+      # do we need to sweep for dead watchers?
+      # or do we let the mailbox rebuild connections
+      while(running?) do; sleep 1; end
+    end
+  end
+end