gitlab-mail_room 0.0.7 → 0.0.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/.gitlab-ci.yml +7 -7
  3. data/.rubocop.yml +5 -0
  4. data/.rubocop_todo.yml +501 -0
  5. data/.ruby-version +1 -1
  6. data/.travis.yml +12 -5
  7. data/README.md +104 -10
  8. data/Rakefile +1 -1
  9. data/lib/mail_room.rb +2 -0
  10. data/lib/mail_room/arbitration/redis.rb +1 -1
  11. data/lib/mail_room/cli.rb +1 -1
  12. data/lib/mail_room/configuration.rb +11 -1
  13. data/lib/mail_room/connection.rb +6 -182
  14. data/lib/mail_room/coordinator.rb +8 -4
  15. data/lib/mail_room/crash_handler.rb +2 -2
  16. data/lib/mail_room/delivery/letter_opener.rb +1 -1
  17. data/lib/mail_room/health_check.rb +60 -0
  18. data/lib/mail_room/imap.rb +8 -0
  19. data/lib/mail_room/imap/connection.rb +200 -0
  20. data/lib/mail_room/imap/message.rb +19 -0
  21. data/lib/mail_room/logger/structured.rb +15 -1
  22. data/lib/mail_room/mailbox.rb +61 -21
  23. data/lib/mail_room/mailbox_watcher.rb +15 -2
  24. data/lib/mail_room/message.rb +16 -0
  25. data/lib/mail_room/microsoft_graph.rb +7 -0
  26. data/lib/mail_room/microsoft_graph/connection.rb +217 -0
  27. data/lib/mail_room/version.rb +1 -1
  28. data/mail_room.gemspec +6 -0
  29. data/spec/fixtures/test_config.yml +3 -0
  30. data/spec/lib/cli_spec.rb +6 -6
  31. data/spec/lib/configuration_spec.rb +10 -2
  32. data/spec/lib/coordinator_spec.rb +16 -2
  33. data/spec/lib/delivery/letter_opener_spec.rb +2 -2
  34. data/spec/lib/delivery/logger_spec.rb +1 -1
  35. data/spec/lib/delivery/postback_spec.rb +11 -11
  36. data/spec/lib/health_check_spec.rb +57 -0
  37. data/spec/lib/{connection_spec.rb → imap/connection_spec.rb} +12 -8
  38. data/spec/lib/imap/message_spec.rb +36 -0
  39. data/spec/lib/logger/structured_spec.rb +34 -2
  40. data/spec/lib/mailbox_spec.rb +75 -27
  41. data/spec/lib/mailbox_watcher_spec.rb +54 -38
  42. data/spec/lib/message_spec.rb +35 -0
  43. data/spec/lib/microsoft_graph/connection_spec.rb +190 -0
  44. data/spec/spec_helper.rb +14 -3
  45. metadata +92 -5
@@ -2,13 +2,15 @@ module MailRoom
2
2
  # Coordinate the mailbox watchers
3
3
  # @author Tony Pitale
4
4
  class Coordinator
5
- attr_accessor :watchers, :running
5
+ attr_accessor :watchers, :running, :health_check
6
6
 
7
7
  # build watchers for a set of mailboxes
8
8
  # @params mailboxes [Array<MailRoom::Mailbox>] mailboxes to be watched
9
- def initialize(mailboxes)
9
+ # @params health_check <MailRoom::HealthCheck> health checker to run
10
+ def initialize(mailboxes, health_check = nil)
10
11
  self.watchers = []
11
12
 
13
+ @health_check = health_check
12
14
  mailboxes.each {|box| self.watchers << MailboxWatcher.new(box)}
13
15
  end
14
16
 
@@ -16,10 +18,11 @@ module MailRoom
16
18
 
17
19
  # start each of the watchers to running
18
20
  def run
21
+ health_check&.run
19
22
  watchers.each(&:run)
20
-
23
+
21
24
  self.running = true
22
-
25
+
23
26
  sleep_while_running
24
27
  ensure
25
28
  quit
@@ -27,6 +30,7 @@ module MailRoom
27
30
 
28
31
  # quit each of the watchers when we're done running
29
32
  def quit
33
+ health_check&.quit
30
34
  watchers.each(&:quit)
31
35
  end
32
36
 
@@ -1,7 +1,7 @@
1
+ require 'date'
1
2
 
2
3
  module MailRoom
3
4
  class CrashHandler
4
-
5
5
  SUPPORTED_FORMATS = %w[json none]
6
6
 
7
7
  def initialize(stream=STDOUT)
@@ -20,7 +20,7 @@ module MailRoom
20
20
  private
21
21
 
22
22
  def json(error)
23
- { time: Time.now, severity: :fatal, message: error.message, backtrace: error.backtrace }.to_json
23
+ { time: DateTime.now.iso8601(3), severity: :fatal, message: error.message, backtrace: error.backtrace }.to_json
24
24
  end
25
25
  end
26
26
  end
@@ -24,7 +24,7 @@ module MailRoom
24
24
  # Trigger `LetterOpener` to deliver our message
25
25
  # @param message [String] the email message as a string, RFC822 format
26
26
  def deliver(message)
27
- method = ::LetterOpener::DeliveryMethod.new(:location => @delivery_options.location)
27
+ method = ::LetterOpener::DeliveryMethod.new(location: @delivery_options.location)
28
28
  method.deliver!(Mail.read_from_string(message))
29
29
 
30
30
  true
@@ -0,0 +1,60 @@
1
+ # frozen_string_literal: true
2
+
3
+ module MailRoom
4
+ class HealthCheck
5
+ attr_reader :address, :port, :running
6
+
7
+ def initialize(attributes = {})
8
+ @address = attributes[:address]
9
+ @port = attributes[:port]
10
+
11
+ validate!
12
+ end
13
+
14
+ def run
15
+ @server = create_server
16
+
17
+ @thread = Thread.new do
18
+ @server.start
19
+ end
20
+
21
+ @thread.abort_on_exception = true
22
+ @running = true
23
+ end
24
+
25
+ def quit
26
+ @running = false
27
+ @server&.shutdown
28
+ @thread&.join(60)
29
+ end
30
+
31
+ private
32
+
33
+ def validate!
34
+ raise 'No health check address specified' unless address
35
+ raise "Health check port #{@port.to_i} is invalid" unless port.to_i.positive?
36
+ end
37
+
38
+ def create_server
39
+ require 'webrick'
40
+
41
+ server = ::WEBrick::HTTPServer.new(Port: port, BindAddress: address, AccessLog: [])
42
+
43
+ server.mount_proc '/liveness' do |_req, res|
44
+ handle_liveness(res)
45
+ end
46
+
47
+ server
48
+ end
49
+
50
+ def handle_liveness(res)
51
+ if @running
52
+ res.status = 200
53
+ res.body = "OK\n"
54
+ else
55
+ res.status = 500
56
+ res.body = "Not running\n"
57
+ end
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ module MailRoom
4
+ module IMAP
5
+ autoload :Connection, 'mail_room/imap/connection'
6
+ autoload :Message, 'mail_room/imap/message'
7
+ end
8
+ end
@@ -0,0 +1,200 @@
1
+ # frozen_string_literal: true
2
+
3
+ module MailRoom
4
+ module IMAP
5
+ class Connection < MailRoom::Connection
6
+ def initialize(mailbox)
7
+ super
8
+
9
+ # log in and set the mailbox
10
+ reset
11
+ setup
12
+ end
13
+
14
+ # is the connection logged in?
15
+ # @return [Boolean]
16
+ def logged_in?
17
+ @logged_in
18
+ end
19
+
20
+ # is the connection blocked idling?
21
+ # @return [Boolean]
22
+ def idling?
23
+ @idling
24
+ end
25
+
26
+ # is the imap connection closed?
27
+ # @return [Boolean]
28
+ def disconnected?
29
+ imap.disconnected?
30
+ end
31
+
32
+ # is the connection ready to idle?
33
+ # @return [Boolean]
34
+ def ready_to_idle?
35
+ logged_in? && !idling?
36
+ end
37
+
38
+ def quit
39
+ stop_idling
40
+ reset
41
+ end
42
+
43
+ def wait
44
+ # in case we missed any between idles
45
+ process_mailbox
46
+
47
+ idle
48
+
49
+ process_mailbox
50
+ rescue Net::IMAP::Error, IOError => e
51
+ @mailbox.logger.warn({ context: @mailbox.context, action: 'Disconnected. Resetting...', error: e.message })
52
+ reset
53
+ setup
54
+ end
55
+
56
+ private
57
+
58
+ def reset
59
+ @imap = nil
60
+ @logged_in = false
61
+ @idling = false
62
+ end
63
+
64
+ def setup
65
+ @mailbox.logger.info({ context: @mailbox.context, action: 'Starting TLS session' })
66
+ start_tls
67
+
68
+ @mailbox.logger.info({ context: @mailbox.context, action: 'Logging into mailbox' })
69
+ log_in
70
+
71
+ @mailbox.logger.info({ context: @mailbox.context, action: 'Setting mailbox' })
72
+ set_mailbox
73
+ end
74
+
75
+ # build a net/imap connection to google imap
76
+ def imap
77
+ @imap ||= Net::IMAP.new(@mailbox.host, port: @mailbox.port, ssl: @mailbox.ssl_options)
78
+ end
79
+
80
+ # start a TLS session
81
+ def start_tls
82
+ imap.starttls if @mailbox.start_tls
83
+ end
84
+
85
+ # send the imap login command to google
86
+ def log_in
87
+ imap.login(@mailbox.email, @mailbox.password)
88
+ @logged_in = true
89
+ end
90
+
91
+ # select the mailbox name we want to use
92
+ def set_mailbox
93
+ imap.select(@mailbox.name) if logged_in?
94
+ end
95
+
96
+ # is the response for a new message?
97
+ # @param response [Net::IMAP::TaggedResponse] the imap response from idle
98
+ # @return [Boolean]
99
+ def message_exists?(response)
100
+ response.respond_to?(:name) && response.name == 'EXISTS'
101
+ end
102
+
103
+ # @private
104
+ def idle_handler
105
+ ->(response) { imap.idle_done if message_exists?(response) }
106
+ end
107
+
108
+ # maintain an imap idle connection
109
+ def idle
110
+ return unless ready_to_idle?
111
+
112
+ @mailbox.logger.info({ context: @mailbox.context, action: 'Idling' })
113
+ @idling = true
114
+
115
+ imap.idle(@mailbox.idle_timeout, &idle_handler)
116
+ ensure
117
+ @idling = false
118
+ end
119
+
120
+ # trigger the idle to finish and wait for the thread to finish
121
+ def stop_idling
122
+ return unless idling?
123
+
124
+ imap.idle_done
125
+
126
+ # idling_thread.join
127
+ # self.idling_thread = nil
128
+ end
129
+
130
+ def process_mailbox
131
+ return unless @new_message_handler
132
+
133
+ @mailbox.logger.info({ context: @mailbox.context, action: 'Processing started' })
134
+
135
+ msgs = new_messages
136
+ any_deletions = msgs.
137
+ # deliver each new message, collect success
138
+ map(&@new_message_handler).
139
+ # include messages with success
140
+ zip(msgs).
141
+ # filter failed deliveries, collect message
142
+ select(&:first).map(&:last).
143
+ # scrub delivered messages
144
+ map { |message| scrub(message) }
145
+ .any?
146
+
147
+ imap.expunge if @mailbox.expunge_deleted && any_deletions
148
+ end
149
+
150
+ def scrub(message)
151
+ if @mailbox.delete_after_delivery
152
+ imap.store(message.seqno, '+FLAGS', [Net::IMAP::DELETED])
153
+ true
154
+ end
155
+ end
156
+
157
+ # @private
158
+ # fetch all messages for the new message ids
159
+ def new_messages
160
+ # Both of these calls may results in
161
+ # imap raising an EOFError, we handle
162
+ # this exception in the watcher
163
+ messages_for_ids(new_message_ids)
164
+ end
165
+
166
+ # TODO: label messages?
167
+ # @imap.store(id, "+X-GM-LABELS", [label])
168
+
169
+ # @private
170
+ # search for all new (unseen) message ids
171
+ # @return [Array<Integer>] message ids
172
+ def new_message_ids
173
+ # uid_search still leaves messages UNSEEN
174
+ all_unread = imap.uid_search(@mailbox.search_command)
175
+
176
+ all_unread = all_unread.slice(0, @mailbox.limit_max_unread) if @mailbox.limit_max_unread.to_i > 0
177
+
178
+ to_deliver = all_unread.select { |uid| @mailbox.deliver?(uid) }
179
+ @mailbox.logger.info({ context: @mailbox.context, action: 'Getting new messages',
180
+ unread: { count: all_unread.count, ids: all_unread }, to_be_delivered: { count: to_deliver.count, ids: to_deliver } })
181
+ to_deliver
182
+ end
183
+
184
+ # @private
185
+ # fetch the email for all given ids in RFC822 format
186
+ # @param ids [Array<Integer>] list of message ids
187
+ # @return [Array<MailRoom::IMAP::Message>] the net/imap messages for the given ids
188
+ def messages_for_ids(uids)
189
+ return [] if uids.empty?
190
+
191
+ # uid_fetch marks as SEEN, will not be re-fetched for UNSEEN
192
+ imap_messages = imap.uid_fetch(uids, 'RFC822')
193
+
194
+ imap_messages.each_with_object([]) do |msg, messages|
195
+ messages << ::MailRoom::IMAP::Message.new(uid: msg.attr['UID'], body: msg.attr['RFC822'], seqno: msg.seqno)
196
+ end
197
+ end
198
+ end
199
+ end
200
+ end
@@ -0,0 +1,19 @@
1
+ # frozen_string_literal:true
2
+
3
+ module MailRoom
4
+ module IMAP
5
+ class Message < MailRoom::Message
6
+ attr_reader :seqno
7
+
8
+ def initialize(uid:, body:, seqno:)
9
+ super(uid: uid, body: body)
10
+
11
+ @seqno = seqno
12
+ end
13
+
14
+ def ==(other)
15
+ super && seqno == other.seqno
16
+ end
17
+ end
18
+ end
19
+ end
@@ -1,3 +1,4 @@
1
+ require 'date'
1
2
  require 'logger'
2
3
  require 'json'
3
4
 
@@ -10,12 +11,25 @@ module MailRoom
10
11
 
11
12
  data = {}
12
13
  data[:severity] = severity
13
- data[:time] = timestamp || Time.now.to_s
14
+ data[:time] = format_timestamp(timestamp || Time.now)
14
15
  # only accept a Hash
15
16
  data.merge!(message)
16
17
 
17
18
  data.to_json + "\n"
18
19
  end
20
+
21
+ private
22
+
23
+ def format_timestamp(timestamp)
24
+ case timestamp
25
+ when Time
26
+ timestamp.to_datetime.iso8601(3).to_s
27
+ when DateTime
28
+ timestamp.iso8601(3).to_s
29
+ else
30
+ timestamp
31
+ end
32
+ end
19
33
  end
20
34
  end
21
35
  end
@@ -1,10 +1,14 @@
1
1
  require "mail_room/delivery"
2
2
  require "mail_room/arbitration"
3
+ require "mail_room/imap"
4
+ require "mail_room/microsoft_graph"
3
5
 
4
6
  module MailRoom
5
7
  # Mailbox Configuration fields
6
8
  MAILBOX_FIELDS = [
7
9
  :email,
10
+ :inbox_method,
11
+ :inbox_options,
8
12
  :password,
9
13
  :host,
10
14
  :port,
@@ -41,24 +45,26 @@ module MailRoom
41
45
  # 29 minutes, as suggested by the spec: https://tools.ietf.org/html/rfc2177
42
46
  IMAP_IDLE_TIMEOUT = 29 * 60 # 29 minutes in in seconds
43
47
 
44
- REQUIRED_CONFIGURATION = [:name, :email, :password, :host, :port]
48
+ IMAP_CONFIGURATION = [:name, :email, :password, :host, :port].freeze
49
+ MICROSOFT_GRAPH_CONFIGURATION = [:name, :email].freeze
50
+ MICROSOFT_GRAPH_INBOX_OPTIONS = [:tenant_id, :client_id, :client_secret].freeze
45
51
 
46
52
  # Default attributes for the mailbox configuration
47
53
  DEFAULTS = {
48
- :search_command => 'UNSEEN',
49
- :delivery_method => 'postback',
50
- :host => 'imap.gmail.com',
51
- :port => 993,
52
- :ssl => true,
53
- :start_tls => false,
54
- :limit_max_unread => 0,
55
- :idle_timeout => IMAP_IDLE_TIMEOUT,
56
- :delete_after_delivery => false,
57
- :expunge_deleted => false,
58
- :delivery_options => {},
59
- :arbitration_method => 'noop',
60
- :arbitration_options => {},
61
- :logger => {}
54
+ search_command: 'UNSEEN',
55
+ delivery_method: 'postback',
56
+ host: 'imap.gmail.com',
57
+ port: 993,
58
+ ssl: true,
59
+ start_tls: false,
60
+ limit_max_unread: 0,
61
+ idle_timeout: IMAP_IDLE_TIMEOUT,
62
+ delete_after_delivery: false,
63
+ expunge_deleted: false,
64
+ delivery_options: {},
65
+ arbitration_method: 'noop',
66
+ arbitration_options: {},
67
+ logger: {}
62
68
  }
63
69
 
64
70
  # Store the configuration and require the appropriate delivery method
@@ -102,13 +108,13 @@ module MailRoom
102
108
  arbitrator.deliver?(uid)
103
109
  end
104
110
 
105
- # deliver the imap email message
106
- # @param message [Net::IMAP::FetchData]
111
+ # deliver the email message
112
+ # @param message [MailRoom::Message]
107
113
  def deliver(message)
108
- body = message.attr['RFC822']
114
+ body = message.body
109
115
  return true unless body
110
116
 
111
- logger.info({context: context, uid: message.attr['UID'], action: "sending to deliverer", deliverer: delivery.class.name, byte_size: body.bytesize})
117
+ logger.info({context: context, uid: message.uid, action: "sending to deliverer", deliverer: delivery.class.name, byte_size: body.bytesize})
112
118
  delivery.deliver(body)
113
119
  end
114
120
 
@@ -121,14 +127,32 @@ module MailRoom
121
127
  { email: self.email, name: self.name }
122
128
  end
123
129
 
130
+ def imap?
131
+ !microsoft_graph?
132
+ end
133
+
134
+ def microsoft_graph?
135
+ self[:inbox_method].to_s == 'microsoft_graph'
136
+ end
137
+
124
138
  def validate!
139
+ if microsoft_graph?
140
+ validate_microsoft_graph!
141
+ else
142
+ validate_imap!
143
+ end
144
+ end
145
+
146
+ private
147
+
148
+ def validate_imap!
125
149
  if self[:idle_timeout] > IMAP_IDLE_TIMEOUT
126
150
  raise IdleTimeoutTooLarge,
127
151
  "Please use an idle timeout smaller than #{29*60} to prevent " \
128
152
  "IMAP server disconnects"
129
153
  end
130
154
 
131
- REQUIRED_CONFIGURATION.each do |k|
155
+ IMAP_CONFIGURATION.each do |k|
132
156
  if self[k].nil?
133
157
  raise ConfigurationError,
134
158
  "Field :#{k} is required in Mailbox: #{inspect}"
@@ -136,7 +160,23 @@ module MailRoom
136
160
  end
137
161
  end
138
162
 
139
- private
163
+ def validate_microsoft_graph!
164
+ raise ConfigurationError, "Missing inbox_options in Mailbox: #{inspect}" unless self.inbox_options.is_a?(Hash)
165
+
166
+ MICROSOFT_GRAPH_CONFIGURATION.each do |k|
167
+ if self[k].nil?
168
+ raise ConfigurationError,
169
+ "Field :#{k} is required in Mailbox: #{inspect}"
170
+ end
171
+ end
172
+
173
+ MICROSOFT_GRAPH_INBOX_OPTIONS.each do |k|
174
+ if self[:inbox_options][k].nil?
175
+ raise ConfigurationError,
176
+ "inbox_options field :#{k} is required in Mailbox: #{inspect}"
177
+ end
178
+ end
179
+ end
140
180
 
141
181
  def parsed_arbitration_options
142
182
  arbitration_klass::Options.new(self)