gitlab-mail_room 0.0.3 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 219f2185b18471c72198ed5c0a55a1428139f7b944f90d1f2428bbc9961e95e9
-  data.tar.gz: f3fadcc3962ac8325c3c461cda65a81e6c7df40a1b575a303a9073b1773252e9
+  metadata.gz: 046d032389d041dfea1d880c47ff567ad6ccd97439ba05db04f49fe8f6fd8596
+  data.tar.gz: c1eda45ae5d2cbab863eabb843adfb89694ea563f73101bbf268fba18c25d175
 SHA512:
-  metadata.gz: e279af4924d7a87fabedbae769785ed2c5f258091651c2f3089225a0bacb1a2ab54b23931a925304e6b0f9cfaeb88f4d93d97763c663774b27bc0def8fbf1bac
-  data.tar.gz: 470d47c934a55ccd789c19da2188e889c6d953e5ebb5ff0be9d70be98db3ab770a3646646faf9596f1296660974d29b1ef02ba7c13fe1b6a1cfb36bc363330d6
+  metadata.gz: 58986de036f80e17d25440c25d4d6bbdb54a0340f865b455847f7a7bf4d2206ae4fefccb91edbb3b56990cf468505501ad1fa30f5223f589483a1e291a59c49a
+  data.tar.gz: a085cd90036662bf26516bd5574a64d921cfff6b0728ec3350cac8eba80f75faef08dc0a69bf4909adb6e31a1f62ff9e4f2d62560eff5226e493af777eaa5810

data/.gitlab-ci.yml

@@ -1,11 +1,16 @@
 # Cache gems in between builds
 
+services:
+  - redis:latest
+
 .test-template: &test
   cache:
     paths:
       - vendor/ruby
+  variables:
+    REDIS_URL: redis://redis:6379
   script:
-  - bundle exec rspec spec
+    - bundle exec rspec spec
   before_script:
     - apt update && apt install -y libicu-dev
     - ruby -v                                   # Print out ruby version for debugging
@@ -25,3 +30,11 @@ rspec-2.5:
 rspec-2.6:
   image: "ruby:2.6"
   <<: *test
+
+rspec-2.7:
+  image: "ruby:2.7"
+  <<: *test
+
+rspec-3.0:
+  image: "ruby:3.0"
+  <<: *test

data/.gitlab/issue_templates/Release.md

@@ -0,0 +1,7 @@
+# GitLab mail_room release checklist
+
+- [ ] create tag in https://gitlab.com/gitlab-org/gitlab-mail_room/
+- [ ] publish gem from this tag to rubygems.org
+- [ ] update https://gitlab.com/gitlab-org/gitlab/-/blob/master/Gemfile to use the new gem version
+- [ ] update gitlab-org/build/CNG to build container images from the new gem (example: https://gitlab.com/gitlab-org/build/CNG/-/merge_requests/451/diffs)
+- [ ] to deploy the new version to gitlab.com, update gitlab-com/gl-infra/k8s-workloads/gitlab-com to pin the new mailroom container image version and assign it the [release managers](https://about.gitlab.com/community/release-managers/) (example: https://gitlab.com/gitlab-com/gl-infra/k8s-workloads/gitlab-com/-/merge_requests/236/diffs)

data/.ruby-version

@@ -1 +1 @@
-2.6.0
+2.7.2

data/README.md

@@ -2,7 +2,21 @@
 
 ## Fork notice
 
-mail_room contains some merged functionality that GitLab requires, so this mirror fork is to help us release custom functionality.
+`mail_room` contains some merged functionality that GitLab requires, so this mirror fork is to help us release custom functionality.
+
+It needs to be more or less kept up to date with the original, so please feel free to incorporate changes to the upstream repo if you see them.
+
+### Rationale
+
+This fork is required to reduce dependency on the upstream releases.
+
+The [original JSON structured logging PR](https://github.com/tpitale/mail_room/pull/88) was [released](https://github.com/tpitale/mail_room/commit/deb8fe63bab21c5c3003346961a815d137ff6d2d) and we [bumped the version](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/3719) to incorporate it into [omnibus](https://gitlab.com/gitlab-org/omnibus-gitlab).
+It turned out that when Mailroom crashed out (which it's designed to do), the crash log [wasn't being pulled into elastic in a very useful way](https://github.com/tpitale/mail_room/commits/master) (that is, every line of the stack trace was a new event) so [another PR](https://github.com/tpitale/mail_room/pull/103) was raised.
+
+Rather than wait for the author (or bugging him more than once), we [opted for bias for action](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/19186#note_290758986) and made a fork of the gem.
+Here it is [in omnibus](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/3960).
+
+The fork is useful as we can post quick fixes to our own fork and release fixes quickly, and still contribute those fixes upstream to help others.
 
 ## README
 
@@ -41,6 +55,9 @@ You will also need to install `faraday` or `letter_opener` if you use the `postb
 
 ```yaml
 ---
+:health_check:
+  - :address: "127.0.0.1"
+    :port: 8080
 :mailboxes:
   -
     :email: "user1@gmail.com"
@@ -105,6 +122,18 @@ You will also need to install `faraday` or `letter_opener` if you use the `postb
 **Note:** If using `delete_after_delivery`, you also probably want to use
 `expunge_deleted` unless you really know what you're doing.
 
+## health_check ##
+
+Requires `webrick` gem to be installed.
+
+This option enables an HTTP server that listens to a bind address
+defined by `address` and `port`. The following endpoints are supported:
+
+* `/liveness`: This returns a 200 status code with `OK` as the body if
+the server is running. Otherwise, it returns a 500 status code.
+
+This feature is not included in upstream `mail_room` and is specific to GitLab.
+
 ## delivery_method ##
 
 ### postback ###

data/lib/mail_room.rb

@@ -7,8 +7,10 @@ end
 
 require "mail_room/version"
 require "mail_room/configuration"
+require "mail_room/health_check"
 require "mail_room/mailbox"
 require "mail_room/mailbox_watcher"
+require "mail_room/message"
 require "mail_room/connection"
 require "mail_room/coordinator"
 require "mail_room/cli"

data/lib/mail_room/cli.rb

@@ -42,7 +42,7 @@ module MailRoom
       end.parse!(args)
 
       self.configuration = Configuration.new(options)
-      self.coordinator = Coordinator.new(configuration.mailboxes)
+      self.coordinator = Coordinator.new(configuration.mailboxes, configuration.health_check)
     end
 
     # Start the coordinator running, sets up signal traps
@@ -57,7 +57,7 @@ module MailRoom
 
       coordinator.run
     rescue Exception => e # not just Errors, but includes lower-level Exceptions
-      CrashHandler.new(error: e, format: @options[:exit_error_format]).handle
+      CrashHandler.new.handle(e, @options[:exit_error_format])
       exit
     end
   end

data/lib/mail_room/configuration.rb

@@ -4,7 +4,7 @@ module MailRoom
   # Wraps configuration for a set of individual mailboxes with global config
   # @author Tony Pitale
   class Configuration
-    attr_accessor :mailboxes, :log_path, :quiet
+    attr_accessor :mailboxes, :log_path, :quiet, :health_check
 
     # Initialize a new configuration of mailboxes
     def initialize(options={})
@@ -18,6 +18,7 @@ module MailRoom
           config_file = YAML.load(erb.result)
 
           set_mailboxes(config_file[:mailboxes])
+          set_health_check(config_file[:health_check])
         rescue => e
           raise e unless quiet
         end
@@ -32,5 +33,14 @@ module MailRoom
         self.mailboxes << Mailbox.new(attributes)
       end
     end
+
+    # Builds the health checker from YAML configuration
+    #
+    # @param health_check_config nil or a Hash containing :address and :port
+    def set_health_check(health_check_config)
+      return unless health_check_config
+
+      self.health_check = HealthCheck.new(health_check_config)
+    end
   end
 end

data/lib/mail_room/connection.rb

@@ -1,195 +1,23 @@
+# frozen_string_literal: true
+
 module MailRoom
   class Connection
+    attr_reader :mailbox, :new_message_handler
+
     def initialize(mailbox)
       @mailbox = mailbox
-
-      # log in and set the mailbox
-      reset
-      setup
     end
 
     def on_new_message(&block)
       @new_message_handler = block
     end
 
-    # is the connection logged in?
-    # @return [Boolean]
-    def logged_in?
-      @logged_in
-    end
-
-    # is the connection blocked idling?
-    # @return [Boolean]
-    def idling?
-      @idling
-    end
-
-    # is the imap connection closed?
-    # @return [Boolean]
-    def disconnected?
-      @imap.disconnected?
-    end
-
-    # is the connection ready to idle?
-    # @return [Boolean]
-    def ready_to_idle?
-      logged_in? && !idling?
-    end
-
-    def quit
-      stop_idling
-      reset
-    end
-
     def wait
-      begin
-        # in case we missed any between idles
-        process_mailbox
-
-        idle
-
-        process_mailbox
-      rescue Net::IMAP::Error, IOError
-        @mailbox.logger.warn({ context: @mailbox.context, action: "Disconnected. Resetting..." })
-        reset
-        setup
-      end
-    end
-
-    private
-
-    def reset
-      @imap = nil
-      @logged_in = false
-      @idling = false
-    end
-
-    def setup
-      @mailbox.logger.info({ context: @mailbox.context, action: "Starting TLS session" })
-      start_tls
-
-      @mailbox.logger.info({ context: @mailbox.context, action: "Logging into mailbox" })
-      log_in
-
-      @mailbox.logger.info({ context: @mailbox.context, action: "Setting mailbox" })
-      set_mailbox
-    end
-
-    # build a net/imap connection to google imap
-    def imap
-      @imap ||= Net::IMAP.new(@mailbox.host, :port => @mailbox.port, :ssl => @mailbox.ssl_options)
-    end
-
-    # start a TLS session
-    def start_tls
-      imap.starttls if @mailbox.start_tls
-    end
-
-    # send the imap login command to google
-    def log_in
-      imap.login(@mailbox.email, @mailbox.password)
-      @logged_in = true
+      raise NotImplementedError
     end
 
-    # select the mailbox name we want to use
-    def set_mailbox
-      imap.select(@mailbox.name) if logged_in?
-    end
-
-    # is the response for a new message?
-    # @param response [Net::IMAP::TaggedResponse] the imap response from idle
-    # @return [Boolean]
-    def message_exists?(response)
-      response.respond_to?(:name) && response.name == 'EXISTS'
-    end
-
-    # @private
-    def idle_handler
-      lambda {|response| imap.idle_done if message_exists?(response)}
-    end
-
-    # maintain an imap idle connection
-    def idle
-      return unless ready_to_idle?
-
-      @mailbox.logger.info({ context: @mailbox.context, action: "Idling" })
-      @idling = true
-
-      imap.idle(@mailbox.idle_timeout, &idle_handler)
-    ensure
-      @idling = false
-    end
-
-    # trigger the idle to finish and wait for the thread to finish
-    def stop_idling
-      return unless idling?
-
-      imap.idle_done
-
-      # idling_thread.join
-      # self.idling_thread = nil
-    end
-
-    def process_mailbox
-      return unless @new_message_handler
-      @mailbox.logger.info({ context: @mailbox.context, action: "Processing started" })
-
-      msgs = new_messages
-
-      any_deletions = msgs.
-                      # deliver each new message, collect success
-                      map(&@new_message_handler).
-                      # include messages with success
-                      zip(msgs).
-                      # filter failed deliveries, collect message
-                      select(&:first).map(&:last).
-                      # scrub delivered messages
-                      map { |message| scrub(message) }.
-                      any?
-
-      imap.expunge if @mailbox.expunge_deleted && any_deletions
-    end
-
-    def scrub(message)
-      if @mailbox.delete_after_delivery
-        imap.store(message.seqno, "+FLAGS", [Net::IMAP::DELETED])
-        true
-      end
-    end
-
-    # @private
-    # fetch all messages for the new message ids
-    def new_messages
-      # Both of these calls may results in
-      #   imap raising an EOFError, we handle
-      #   this exception in the watcher
-      messages_for_ids(new_message_ids)
-    end
-
-    # TODO: label messages?
-    #   @imap.store(id, "+X-GM-LABELS", [label])
-
-    # @private
-    # search for all new (unseen) message ids
-    # @return [Array<Integer>] message ids
-    def new_message_ids
-      # uid_search still leaves messages UNSEEN
-      all_unread = @imap.uid_search(@mailbox.search_command)
-
-      to_deliver = all_unread.select { |uid| @mailbox.deliver?(uid) }
-      @mailbox.logger.info({ context: @mailbox.context, action: "Getting new messages", unread: {count: all_unread.count, ids: all_unread}, to_be_delivered: { count: to_deliver.count, ids: all_unread } })
-      to_deliver
-    end
-
-    # @private
-    # fetch the email for all given ids in RFC822 format
-    # @param ids [Array<Integer>] list of message ids
-    # @return [Array<Net::IMAP::FetchData>] the net/imap messages for the given ids
-    def messages_for_ids(uids)
-      return [] if uids.empty?
-
-      # uid_fetch marks as SEEN, will not be re-fetched for UNSEEN
-      imap.uid_fetch(uids, "RFC822")
+    def quit
+      raise NotImplementedError
     end
   end
 end

data/lib/mail_room/coordinator.rb

@@ -2,13 +2,15 @@ module MailRoom
   # Coordinate the mailbox watchers
   # @author Tony Pitale
   class Coordinator
-    attr_accessor :watchers, :running
+    attr_accessor :watchers, :running, :health_check
 
     # build watchers for a set of mailboxes
     # @params mailboxes [Array<MailRoom::Mailbox>] mailboxes to be watched
-    def initialize(mailboxes)
+    # @params health_check <MailRoom::HealthCheck> health checker to run
+    def initialize(mailboxes, health_check = nil)
       self.watchers = []
 
+      @health_check = health_check
       mailboxes.each {|box| self.watchers << MailboxWatcher.new(box)}
     end
 
@@ -16,10 +18,11 @@ module MailRoom
 
     # start each of the watchers to running
     def run
+      health_check&.run
       watchers.each(&:run)
-      
+
       self.running = true
-      
+
       sleep_while_running
     ensure
       quit
@@ -27,6 +30,7 @@ module MailRoom
 
     # quit each of the watchers when we're done running
     def quit
+      health_check&.quit
       watchers.each(&:quit)
     end
 

data/lib/mail_room/crash_handler.rb

@@ -1,19 +1,15 @@
 
 module MailRoom
   class CrashHandler
+    SUPPORTED_FORMATS = %w[json none]
 
-    attr_reader :error, :format
-
-    SUPPORTED_FORMATS = %w[json plain]
-
-    def initialize(error:, format:)
-      @error = error
-      @format = format
+    def initialize(stream=STDOUT)
+      @stream = stream
     end
 
-    def handle
+    def handle(error, format)
       if format == 'json'
-        puts json
+        @stream.puts json(error)
         return
       end
 
@@ -22,7 +18,7 @@ module MailRoom
 
     private
 
-    def json
+    def json(error)
       { time: Time.now, severity: :fatal, message: error.message, backtrace: error.backtrace }.to_json
     end
   end