gitlab-mail_room 0.0.6 → 0.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +9 -5
- data/.ruby-version +1 -1
- data/.travis.yml +3 -2
- data/README.md +104 -10
- data/lib/mail_room.rb +2 -0
- data/lib/mail_room/cli.rb +1 -1
- data/lib/mail_room/configuration.rb +11 -1
- data/lib/mail_room/connection.rb +7 -179
- data/lib/mail_room/coordinator.rb +8 -4
- data/lib/mail_room/crash_handler.rb +2 -2
- data/lib/mail_room/health_check.rb +60 -0
- data/lib/mail_room/imap.rb +8 -0
- data/lib/mail_room/imap/connection.rb +200 -0
- data/lib/mail_room/imap/message.rb +19 -0
- data/lib/mail_room/logger/structured.rb +15 -1
- data/lib/mail_room/mailbox.rb +49 -7
- data/lib/mail_room/mailbox_watcher.rb +15 -2
- data/lib/mail_room/message.rb +16 -0
- data/lib/mail_room/microsoft_graph.rb +7 -0
- data/lib/mail_room/microsoft_graph/connection.rb +216 -0
- data/lib/mail_room/version.rb +1 -1
- data/mail_room.gemspec +5 -0
- data/spec/fixtures/test_config.yml +3 -0
- data/spec/lib/cli_spec.rb +3 -3
- data/spec/lib/configuration_spec.rb +9 -1
- data/spec/lib/coordinator_spec.rb +16 -2
- data/spec/lib/health_check_spec.rb +57 -0
- data/spec/lib/{connection_spec.rb → imap/connection_spec.rb} +12 -8
- data/spec/lib/imap/message_spec.rb +36 -0
- data/spec/lib/logger/structured_spec.rb +34 -2
- data/spec/lib/mailbox_spec.rb +58 -10
- data/spec/lib/mailbox_watcher_spec.rb +54 -38
- data/spec/lib/message_spec.rb +35 -0
- data/spec/lib/microsoft_graph/connection_spec.rb +183 -0
- data/spec/spec_helper.rb +11 -0
- metadata +76 -5
@@ -2,13 +2,15 @@ module MailRoom
|
|
2
2
|
# Coordinate the mailbox watchers
|
3
3
|
# @author Tony Pitale
|
4
4
|
class Coordinator
|
5
|
-
attr_accessor :watchers, :running
|
5
|
+
attr_accessor :watchers, :running, :health_check
|
6
6
|
|
7
7
|
# build watchers for a set of mailboxes
|
8
8
|
# @params mailboxes [Array<MailRoom::Mailbox>] mailboxes to be watched
|
9
|
-
|
9
|
+
# @params health_check <MailRoom::HealthCheck> health checker to run
|
10
|
+
def initialize(mailboxes, health_check = nil)
|
10
11
|
self.watchers = []
|
11
12
|
|
13
|
+
@health_check = health_check
|
12
14
|
mailboxes.each {|box| self.watchers << MailboxWatcher.new(box)}
|
13
15
|
end
|
14
16
|
|
@@ -16,10 +18,11 @@ module MailRoom
|
|
16
18
|
|
17
19
|
# start each of the watchers to running
|
18
20
|
def run
|
21
|
+
health_check&.run
|
19
22
|
watchers.each(&:run)
|
20
|
-
|
23
|
+
|
21
24
|
self.running = true
|
22
|
-
|
25
|
+
|
23
26
|
sleep_while_running
|
24
27
|
ensure
|
25
28
|
quit
|
@@ -27,6 +30,7 @@ module MailRoom
|
|
27
30
|
|
28
31
|
# quit each of the watchers when we're done running
|
29
32
|
def quit
|
33
|
+
health_check&.quit
|
30
34
|
watchers.each(&:quit)
|
31
35
|
end
|
32
36
|
|
@@ -1,7 +1,7 @@
|
|
1
|
+
require 'date'
|
1
2
|
|
2
3
|
module MailRoom
|
3
4
|
class CrashHandler
|
4
|
-
|
5
5
|
SUPPORTED_FORMATS = %w[json none]
|
6
6
|
|
7
7
|
def initialize(stream=STDOUT)
|
@@ -20,7 +20,7 @@ module MailRoom
|
|
20
20
|
private
|
21
21
|
|
22
22
|
def json(error)
|
23
|
-
{ time:
|
23
|
+
{ time: DateTime.now.iso8601(3), severity: :fatal, message: error.message, backtrace: error.backtrace }.to_json
|
24
24
|
end
|
25
25
|
end
|
26
26
|
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module MailRoom
|
4
|
+
class HealthCheck
|
5
|
+
attr_reader :address, :port, :running
|
6
|
+
|
7
|
+
def initialize(attributes = {})
|
8
|
+
@address = attributes[:address]
|
9
|
+
@port = attributes[:port]
|
10
|
+
|
11
|
+
validate!
|
12
|
+
end
|
13
|
+
|
14
|
+
def run
|
15
|
+
@server = create_server
|
16
|
+
|
17
|
+
@thread = Thread.new do
|
18
|
+
@server.start
|
19
|
+
end
|
20
|
+
|
21
|
+
@thread.abort_on_exception = true
|
22
|
+
@running = true
|
23
|
+
end
|
24
|
+
|
25
|
+
def quit
|
26
|
+
@running = false
|
27
|
+
@server&.shutdown
|
28
|
+
@thread&.join(60)
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def validate!
|
34
|
+
raise 'No health check address specified' unless address
|
35
|
+
raise "Health check port #{@port.to_i} is invalid" unless port.to_i.positive?
|
36
|
+
end
|
37
|
+
|
38
|
+
def create_server
|
39
|
+
require 'webrick'
|
40
|
+
|
41
|
+
server = ::WEBrick::HTTPServer.new(Port: port, BindAddress: address, AccessLog: [])
|
42
|
+
|
43
|
+
server.mount_proc '/liveness' do |_req, res|
|
44
|
+
handle_liveness(res)
|
45
|
+
end
|
46
|
+
|
47
|
+
server
|
48
|
+
end
|
49
|
+
|
50
|
+
def handle_liveness(res)
|
51
|
+
if @running
|
52
|
+
res.status = 200
|
53
|
+
res.body = "OK\n"
|
54
|
+
else
|
55
|
+
res.status = 500
|
56
|
+
res.body = "Not running\n"
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,200 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module MailRoom
|
4
|
+
module IMAP
|
5
|
+
class Connection < MailRoom::Connection
|
6
|
+
def initialize(mailbox)
|
7
|
+
super
|
8
|
+
|
9
|
+
# log in and set the mailbox
|
10
|
+
reset
|
11
|
+
setup
|
12
|
+
end
|
13
|
+
|
14
|
+
# is the connection logged in?
|
15
|
+
# @return [Boolean]
|
16
|
+
def logged_in?
|
17
|
+
@logged_in
|
18
|
+
end
|
19
|
+
|
20
|
+
# is the connection blocked idling?
|
21
|
+
# @return [Boolean]
|
22
|
+
def idling?
|
23
|
+
@idling
|
24
|
+
end
|
25
|
+
|
26
|
+
# is the imap connection closed?
|
27
|
+
# @return [Boolean]
|
28
|
+
def disconnected?
|
29
|
+
imap.disconnected?
|
30
|
+
end
|
31
|
+
|
32
|
+
# is the connection ready to idle?
|
33
|
+
# @return [Boolean]
|
34
|
+
def ready_to_idle?
|
35
|
+
logged_in? && !idling?
|
36
|
+
end
|
37
|
+
|
38
|
+
def quit
|
39
|
+
stop_idling
|
40
|
+
reset
|
41
|
+
end
|
42
|
+
|
43
|
+
def wait
|
44
|
+
# in case we missed any between idles
|
45
|
+
process_mailbox
|
46
|
+
|
47
|
+
idle
|
48
|
+
|
49
|
+
process_mailbox
|
50
|
+
rescue Net::IMAP::Error, IOError => e
|
51
|
+
@mailbox.logger.warn({ context: @mailbox.context, action: 'Disconnected. Resetting...', error: e.message })
|
52
|
+
reset
|
53
|
+
setup
|
54
|
+
end
|
55
|
+
|
56
|
+
private
|
57
|
+
|
58
|
+
def reset
|
59
|
+
@imap = nil
|
60
|
+
@logged_in = false
|
61
|
+
@idling = false
|
62
|
+
end
|
63
|
+
|
64
|
+
def setup
|
65
|
+
@mailbox.logger.info({ context: @mailbox.context, action: 'Starting TLS session' })
|
66
|
+
start_tls
|
67
|
+
|
68
|
+
@mailbox.logger.info({ context: @mailbox.context, action: 'Logging into mailbox' })
|
69
|
+
log_in
|
70
|
+
|
71
|
+
@mailbox.logger.info({ context: @mailbox.context, action: 'Setting mailbox' })
|
72
|
+
set_mailbox
|
73
|
+
end
|
74
|
+
|
75
|
+
# build a net/imap connection to google imap
|
76
|
+
def imap
|
77
|
+
@imap ||= Net::IMAP.new(@mailbox.host, port: @mailbox.port, ssl: @mailbox.ssl_options)
|
78
|
+
end
|
79
|
+
|
80
|
+
# start a TLS session
|
81
|
+
def start_tls
|
82
|
+
imap.starttls if @mailbox.start_tls
|
83
|
+
end
|
84
|
+
|
85
|
+
# send the imap login command to google
|
86
|
+
def log_in
|
87
|
+
imap.login(@mailbox.email, @mailbox.password)
|
88
|
+
@logged_in = true
|
89
|
+
end
|
90
|
+
|
91
|
+
# select the mailbox name we want to use
|
92
|
+
def set_mailbox
|
93
|
+
imap.select(@mailbox.name) if logged_in?
|
94
|
+
end
|
95
|
+
|
96
|
+
# is the response for a new message?
|
97
|
+
# @param response [Net::IMAP::TaggedResponse] the imap response from idle
|
98
|
+
# @return [Boolean]
|
99
|
+
def message_exists?(response)
|
100
|
+
response.respond_to?(:name) && response.name == 'EXISTS'
|
101
|
+
end
|
102
|
+
|
103
|
+
# @private
|
104
|
+
def idle_handler
|
105
|
+
->(response) { imap.idle_done if message_exists?(response) }
|
106
|
+
end
|
107
|
+
|
108
|
+
# maintain an imap idle connection
|
109
|
+
def idle
|
110
|
+
return unless ready_to_idle?
|
111
|
+
|
112
|
+
@mailbox.logger.info({ context: @mailbox.context, action: 'Idling' })
|
113
|
+
@idling = true
|
114
|
+
|
115
|
+
imap.idle(@mailbox.idle_timeout, &idle_handler)
|
116
|
+
ensure
|
117
|
+
@idling = false
|
118
|
+
end
|
119
|
+
|
120
|
+
# trigger the idle to finish and wait for the thread to finish
|
121
|
+
def stop_idling
|
122
|
+
return unless idling?
|
123
|
+
|
124
|
+
imap.idle_done
|
125
|
+
|
126
|
+
# idling_thread.join
|
127
|
+
# self.idling_thread = nil
|
128
|
+
end
|
129
|
+
|
130
|
+
def process_mailbox
|
131
|
+
return unless @new_message_handler
|
132
|
+
|
133
|
+
@mailbox.logger.info({ context: @mailbox.context, action: 'Processing started' })
|
134
|
+
|
135
|
+
msgs = new_messages
|
136
|
+
any_deletions = msgs.
|
137
|
+
# deliver each new message, collect success
|
138
|
+
map(&@new_message_handler).
|
139
|
+
# include messages with success
|
140
|
+
zip(msgs).
|
141
|
+
# filter failed deliveries, collect message
|
142
|
+
select(&:first).map(&:last).
|
143
|
+
# scrub delivered messages
|
144
|
+
map { |message| scrub(message) }
|
145
|
+
.any?
|
146
|
+
|
147
|
+
imap.expunge if @mailbox.expunge_deleted && any_deletions
|
148
|
+
end
|
149
|
+
|
150
|
+
def scrub(message)
|
151
|
+
if @mailbox.delete_after_delivery
|
152
|
+
imap.store(message.seqno, '+FLAGS', [Net::IMAP::DELETED])
|
153
|
+
true
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
# @private
|
158
|
+
# fetch all messages for the new message ids
|
159
|
+
def new_messages
|
160
|
+
# Both of these calls may results in
|
161
|
+
# imap raising an EOFError, we handle
|
162
|
+
# this exception in the watcher
|
163
|
+
messages_for_ids(new_message_ids)
|
164
|
+
end
|
165
|
+
|
166
|
+
# TODO: label messages?
|
167
|
+
# @imap.store(id, "+X-GM-LABELS", [label])
|
168
|
+
|
169
|
+
# @private
|
170
|
+
# search for all new (unseen) message ids
|
171
|
+
# @return [Array<Integer>] message ids
|
172
|
+
def new_message_ids
|
173
|
+
# uid_search still leaves messages UNSEEN
|
174
|
+
all_unread = imap.uid_search(@mailbox.search_command)
|
175
|
+
|
176
|
+
all_unread = all_unread.slice(0, @mailbox.limit_max_unread) if @mailbox.limit_max_unread.to_i > 0
|
177
|
+
|
178
|
+
to_deliver = all_unread.select { |uid| @mailbox.deliver?(uid) }
|
179
|
+
@mailbox.logger.info({ context: @mailbox.context, action: 'Getting new messages',
|
180
|
+
unread: { count: all_unread.count, ids: all_unread }, to_be_delivered: { count: to_deliver.count, ids: to_deliver } })
|
181
|
+
to_deliver
|
182
|
+
end
|
183
|
+
|
184
|
+
# @private
|
185
|
+
# fetch the email for all given ids in RFC822 format
|
186
|
+
# @param ids [Array<Integer>] list of message ids
|
187
|
+
# @return [Array<MailRoom::IMAP::Message>] the net/imap messages for the given ids
|
188
|
+
def messages_for_ids(uids)
|
189
|
+
return [] if uids.empty?
|
190
|
+
|
191
|
+
# uid_fetch marks as SEEN, will not be re-fetched for UNSEEN
|
192
|
+
imap_messages = imap.uid_fetch(uids, 'RFC822')
|
193
|
+
|
194
|
+
imap_messages.each_with_object([]) do |msg, messages|
|
195
|
+
messages << ::MailRoom::IMAP::Message.new(uid: msg.attr['UID'], body: msg.attr['RFC822'], seqno: msg.seqno)
|
196
|
+
end
|
197
|
+
end
|
198
|
+
end
|
199
|
+
end
|
200
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal:true
|
2
|
+
|
3
|
+
module MailRoom
|
4
|
+
module IMAP
|
5
|
+
class Message < MailRoom::Message
|
6
|
+
attr_reader :seqno
|
7
|
+
|
8
|
+
def initialize(uid:, body:, seqno:)
|
9
|
+
super(uid: uid, body: body)
|
10
|
+
|
11
|
+
@seqno = seqno
|
12
|
+
end
|
13
|
+
|
14
|
+
def ==(other)
|
15
|
+
super && seqno == other.seqno
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'date'
|
1
2
|
require 'logger'
|
2
3
|
require 'json'
|
3
4
|
|
@@ -10,12 +11,25 @@ module MailRoom
|
|
10
11
|
|
11
12
|
data = {}
|
12
13
|
data[:severity] = severity
|
13
|
-
data[:time] = timestamp || Time.now
|
14
|
+
data[:time] = format_timestamp(timestamp || Time.now)
|
14
15
|
# only accept a Hash
|
15
16
|
data.merge!(message)
|
16
17
|
|
17
18
|
data.to_json + "\n"
|
18
19
|
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def format_timestamp(timestamp)
|
24
|
+
case timestamp
|
25
|
+
when Time
|
26
|
+
timestamp.to_datetime.iso8601(3).to_s
|
27
|
+
when DateTime
|
28
|
+
timestamp.iso8601(3).to_s
|
29
|
+
else
|
30
|
+
timestamp
|
31
|
+
end
|
32
|
+
end
|
19
33
|
end
|
20
34
|
end
|
21
35
|
end
|
data/lib/mail_room/mailbox.rb
CHANGED
@@ -1,15 +1,20 @@
|
|
1
1
|
require "mail_room/delivery"
|
2
2
|
require "mail_room/arbitration"
|
3
|
+
require "mail_room/imap"
|
4
|
+
require "mail_room/microsoft_graph"
|
3
5
|
|
4
6
|
module MailRoom
|
5
7
|
# Mailbox Configuration fields
|
6
8
|
MAILBOX_FIELDS = [
|
7
9
|
:email,
|
10
|
+
:inbox_method,
|
11
|
+
:inbox_options,
|
8
12
|
:password,
|
9
13
|
:host,
|
10
14
|
:port,
|
11
15
|
:ssl,
|
12
16
|
:start_tls,
|
17
|
+
:limit_max_unread, #to avoid 'Error in IMAP command UID FETCH: Too long argument'
|
13
18
|
:idle_timeout,
|
14
19
|
:search_command,
|
15
20
|
:name,
|
@@ -40,7 +45,9 @@ module MailRoom
|
|
40
45
|
# 29 minutes, as suggested by the spec: https://tools.ietf.org/html/rfc2177
|
41
46
|
IMAP_IDLE_TIMEOUT = 29 * 60 # 29 minutes in in seconds
|
42
47
|
|
43
|
-
|
48
|
+
IMAP_CONFIGURATION = [:name, :email, :password, :host, :port].freeze
|
49
|
+
MICROSOFT_GRAPH_CONFIGURATION = [:name, :email].freeze
|
50
|
+
MICROSOFT_GRAPH_INBOX_OPTIONS = [:tenant_id, :client_id, :client_secret].freeze
|
44
51
|
|
45
52
|
# Default attributes for the mailbox configuration
|
46
53
|
DEFAULTS = {
|
@@ -50,6 +57,7 @@ module MailRoom
|
|
50
57
|
:port => 993,
|
51
58
|
:ssl => true,
|
52
59
|
:start_tls => false,
|
60
|
+
:limit_max_unread => 0,
|
53
61
|
:idle_timeout => IMAP_IDLE_TIMEOUT,
|
54
62
|
:delete_after_delivery => false,
|
55
63
|
:expunge_deleted => false,
|
@@ -100,13 +108,13 @@ module MailRoom
|
|
100
108
|
arbitrator.deliver?(uid)
|
101
109
|
end
|
102
110
|
|
103
|
-
# deliver the
|
104
|
-
# @param message [
|
111
|
+
# deliver the email message
|
112
|
+
# @param message [MailRoom::Message]
|
105
113
|
def deliver(message)
|
106
|
-
body = message.
|
114
|
+
body = message.body
|
107
115
|
return true unless body
|
108
116
|
|
109
|
-
logger.info({context: context, uid: message.
|
117
|
+
logger.info({context: context, uid: message.uid, action: "sending to deliverer", deliverer: delivery.class.name, byte_size: body.bytesize})
|
110
118
|
delivery.deliver(body)
|
111
119
|
end
|
112
120
|
|
@@ -119,14 +127,32 @@ module MailRoom
|
|
119
127
|
{ email: self.email, name: self.name }
|
120
128
|
end
|
121
129
|
|
130
|
+
def imap?
|
131
|
+
!microsoft_graph?
|
132
|
+
end
|
133
|
+
|
134
|
+
def microsoft_graph?
|
135
|
+
self[:inbox_method].to_s == 'microsoft_graph'
|
136
|
+
end
|
137
|
+
|
122
138
|
def validate!
|
139
|
+
if microsoft_graph?
|
140
|
+
validate_microsoft_graph!
|
141
|
+
else
|
142
|
+
validate_imap!
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
private
|
147
|
+
|
148
|
+
def validate_imap!
|
123
149
|
if self[:idle_timeout] > IMAP_IDLE_TIMEOUT
|
124
150
|
raise IdleTimeoutTooLarge,
|
125
151
|
"Please use an idle timeout smaller than #{29*60} to prevent " \
|
126
152
|
"IMAP server disconnects"
|
127
153
|
end
|
128
154
|
|
129
|
-
|
155
|
+
IMAP_CONFIGURATION.each do |k|
|
130
156
|
if self[k].nil?
|
131
157
|
raise ConfigurationError,
|
132
158
|
"Field :#{k} is required in Mailbox: #{inspect}"
|
@@ -134,7 +160,23 @@ module MailRoom
|
|
134
160
|
end
|
135
161
|
end
|
136
162
|
|
137
|
-
|
163
|
+
def validate_microsoft_graph!
|
164
|
+
raise ConfigurationError, "Missing inbox_options in Mailbox: #{inspect}" unless self.inbox_options.is_a?(Hash)
|
165
|
+
|
166
|
+
MICROSOFT_GRAPH_CONFIGURATION.each do |k|
|
167
|
+
if self[k].nil?
|
168
|
+
raise ConfigurationError,
|
169
|
+
"Field :#{k} is required in Mailbox: #{inspect}"
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
MICROSOFT_GRAPH_INBOX_OPTIONS.each do |k|
|
174
|
+
if self[:inbox_options][k].nil?
|
175
|
+
raise ConfigurationError,
|
176
|
+
"inbox_options field :#{k} is required in Mailbox: #{inspect}"
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
138
180
|
|
139
181
|
def parsed_arbitration_options
|
140
182
|
arbitration_klass::Options.new(self)
|