gitlab-mail_room 0.0.7 → 0.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +7 -7
- data/.rubocop.yml +5 -0
- data/.rubocop_todo.yml +501 -0
- data/.ruby-version +1 -1
- data/.travis.yml +12 -5
- data/README.md +104 -10
- data/Rakefile +1 -1
- data/lib/mail_room.rb +2 -0
- data/lib/mail_room/arbitration/redis.rb +1 -1
- data/lib/mail_room/cli.rb +1 -1
- data/lib/mail_room/configuration.rb +11 -1
- data/lib/mail_room/connection.rb +6 -182
- data/lib/mail_room/coordinator.rb +8 -4
- data/lib/mail_room/crash_handler.rb +2 -2
- data/lib/mail_room/delivery/letter_opener.rb +1 -1
- data/lib/mail_room/health_check.rb +60 -0
- data/lib/mail_room/imap.rb +8 -0
- data/lib/mail_room/imap/connection.rb +200 -0
- data/lib/mail_room/imap/message.rb +19 -0
- data/lib/mail_room/logger/structured.rb +15 -1
- data/lib/mail_room/mailbox.rb +61 -21
- data/lib/mail_room/mailbox_watcher.rb +15 -2
- data/lib/mail_room/message.rb +16 -0
- data/lib/mail_room/microsoft_graph.rb +7 -0
- data/lib/mail_room/microsoft_graph/connection.rb +217 -0
- data/lib/mail_room/version.rb +1 -1
- data/mail_room.gemspec +6 -0
- data/spec/fixtures/test_config.yml +3 -0
- data/spec/lib/cli_spec.rb +6 -6
- data/spec/lib/configuration_spec.rb +10 -2
- data/spec/lib/coordinator_spec.rb +16 -2
- data/spec/lib/delivery/letter_opener_spec.rb +2 -2
- data/spec/lib/delivery/logger_spec.rb +1 -1
- data/spec/lib/delivery/postback_spec.rb +11 -11
- data/spec/lib/health_check_spec.rb +57 -0
- data/spec/lib/{connection_spec.rb → imap/connection_spec.rb} +12 -8
- data/spec/lib/imap/message_spec.rb +36 -0
- data/spec/lib/logger/structured_spec.rb +34 -2
- data/spec/lib/mailbox_spec.rb +75 -27
- data/spec/lib/mailbox_watcher_spec.rb +54 -38
- data/spec/lib/message_spec.rb +35 -0
- data/spec/lib/microsoft_graph/connection_spec.rb +190 -0
- data/spec/spec_helper.rb +14 -3
- metadata +92 -5
@@ -2,13 +2,15 @@ module MailRoom
|
|
2
2
|
# Coordinate the mailbox watchers
|
3
3
|
# @author Tony Pitale
|
4
4
|
class Coordinator
|
5
|
-
attr_accessor :watchers, :running
|
5
|
+
attr_accessor :watchers, :running, :health_check
|
6
6
|
|
7
7
|
# build watchers for a set of mailboxes
|
8
8
|
# @params mailboxes [Array<MailRoom::Mailbox>] mailboxes to be watched
|
9
|
-
|
9
|
+
# @params health_check <MailRoom::HealthCheck> health checker to run
|
10
|
+
def initialize(mailboxes, health_check = nil)
|
10
11
|
self.watchers = []
|
11
12
|
|
13
|
+
@health_check = health_check
|
12
14
|
mailboxes.each {|box| self.watchers << MailboxWatcher.new(box)}
|
13
15
|
end
|
14
16
|
|
@@ -16,10 +18,11 @@ module MailRoom
|
|
16
18
|
|
17
19
|
# start each of the watchers to running
|
18
20
|
def run
|
21
|
+
health_check&.run
|
19
22
|
watchers.each(&:run)
|
20
|
-
|
23
|
+
|
21
24
|
self.running = true
|
22
|
-
|
25
|
+
|
23
26
|
sleep_while_running
|
24
27
|
ensure
|
25
28
|
quit
|
@@ -27,6 +30,7 @@ module MailRoom
|
|
27
30
|
|
28
31
|
# quit each of the watchers when we're done running
|
29
32
|
def quit
|
33
|
+
health_check&.quit
|
30
34
|
watchers.each(&:quit)
|
31
35
|
end
|
32
36
|
|
@@ -1,7 +1,7 @@
|
|
1
|
+
require 'date'
|
1
2
|
|
2
3
|
module MailRoom
|
3
4
|
class CrashHandler
|
4
|
-
|
5
5
|
SUPPORTED_FORMATS = %w[json none]
|
6
6
|
|
7
7
|
def initialize(stream=STDOUT)
|
@@ -20,7 +20,7 @@ module MailRoom
|
|
20
20
|
private
|
21
21
|
|
22
22
|
def json(error)
|
23
|
-
{ time:
|
23
|
+
{ time: DateTime.now.iso8601(3), severity: :fatal, message: error.message, backtrace: error.backtrace }.to_json
|
24
24
|
end
|
25
25
|
end
|
26
26
|
end
|
@@ -24,7 +24,7 @@ module MailRoom
|
|
24
24
|
# Trigger `LetterOpener` to deliver our message
|
25
25
|
# @param message [String] the email message as a string, RFC822 format
|
26
26
|
def deliver(message)
|
27
|
-
method = ::LetterOpener::DeliveryMethod.new(:
|
27
|
+
method = ::LetterOpener::DeliveryMethod.new(location: @delivery_options.location)
|
28
28
|
method.deliver!(Mail.read_from_string(message))
|
29
29
|
|
30
30
|
true
|
@@ -0,0 +1,60 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module MailRoom
|
4
|
+
class HealthCheck
|
5
|
+
attr_reader :address, :port, :running
|
6
|
+
|
7
|
+
def initialize(attributes = {})
|
8
|
+
@address = attributes[:address]
|
9
|
+
@port = attributes[:port]
|
10
|
+
|
11
|
+
validate!
|
12
|
+
end
|
13
|
+
|
14
|
+
def run
|
15
|
+
@server = create_server
|
16
|
+
|
17
|
+
@thread = Thread.new do
|
18
|
+
@server.start
|
19
|
+
end
|
20
|
+
|
21
|
+
@thread.abort_on_exception = true
|
22
|
+
@running = true
|
23
|
+
end
|
24
|
+
|
25
|
+
def quit
|
26
|
+
@running = false
|
27
|
+
@server&.shutdown
|
28
|
+
@thread&.join(60)
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def validate!
|
34
|
+
raise 'No health check address specified' unless address
|
35
|
+
raise "Health check port #{@port.to_i} is invalid" unless port.to_i.positive?
|
36
|
+
end
|
37
|
+
|
38
|
+
def create_server
|
39
|
+
require 'webrick'
|
40
|
+
|
41
|
+
server = ::WEBrick::HTTPServer.new(Port: port, BindAddress: address, AccessLog: [])
|
42
|
+
|
43
|
+
server.mount_proc '/liveness' do |_req, res|
|
44
|
+
handle_liveness(res)
|
45
|
+
end
|
46
|
+
|
47
|
+
server
|
48
|
+
end
|
49
|
+
|
50
|
+
def handle_liveness(res)
|
51
|
+
if @running
|
52
|
+
res.status = 200
|
53
|
+
res.body = "OK\n"
|
54
|
+
else
|
55
|
+
res.status = 500
|
56
|
+
res.body = "Not running\n"
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,200 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module MailRoom
|
4
|
+
module IMAP
|
5
|
+
class Connection < MailRoom::Connection
|
6
|
+
def initialize(mailbox)
|
7
|
+
super
|
8
|
+
|
9
|
+
# log in and set the mailbox
|
10
|
+
reset
|
11
|
+
setup
|
12
|
+
end
|
13
|
+
|
14
|
+
# is the connection logged in?
|
15
|
+
# @return [Boolean]
|
16
|
+
def logged_in?
|
17
|
+
@logged_in
|
18
|
+
end
|
19
|
+
|
20
|
+
# is the connection blocked idling?
|
21
|
+
# @return [Boolean]
|
22
|
+
def idling?
|
23
|
+
@idling
|
24
|
+
end
|
25
|
+
|
26
|
+
# is the imap connection closed?
|
27
|
+
# @return [Boolean]
|
28
|
+
def disconnected?
|
29
|
+
imap.disconnected?
|
30
|
+
end
|
31
|
+
|
32
|
+
# is the connection ready to idle?
|
33
|
+
# @return [Boolean]
|
34
|
+
def ready_to_idle?
|
35
|
+
logged_in? && !idling?
|
36
|
+
end
|
37
|
+
|
38
|
+
def quit
|
39
|
+
stop_idling
|
40
|
+
reset
|
41
|
+
end
|
42
|
+
|
43
|
+
def wait
|
44
|
+
# in case we missed any between idles
|
45
|
+
process_mailbox
|
46
|
+
|
47
|
+
idle
|
48
|
+
|
49
|
+
process_mailbox
|
50
|
+
rescue Net::IMAP::Error, IOError => e
|
51
|
+
@mailbox.logger.warn({ context: @mailbox.context, action: 'Disconnected. Resetting...', error: e.message })
|
52
|
+
reset
|
53
|
+
setup
|
54
|
+
end
|
55
|
+
|
56
|
+
private
|
57
|
+
|
58
|
+
def reset
|
59
|
+
@imap = nil
|
60
|
+
@logged_in = false
|
61
|
+
@idling = false
|
62
|
+
end
|
63
|
+
|
64
|
+
def setup
|
65
|
+
@mailbox.logger.info({ context: @mailbox.context, action: 'Starting TLS session' })
|
66
|
+
start_tls
|
67
|
+
|
68
|
+
@mailbox.logger.info({ context: @mailbox.context, action: 'Logging into mailbox' })
|
69
|
+
log_in
|
70
|
+
|
71
|
+
@mailbox.logger.info({ context: @mailbox.context, action: 'Setting mailbox' })
|
72
|
+
set_mailbox
|
73
|
+
end
|
74
|
+
|
75
|
+
# build a net/imap connection to google imap
|
76
|
+
def imap
|
77
|
+
@imap ||= Net::IMAP.new(@mailbox.host, port: @mailbox.port, ssl: @mailbox.ssl_options)
|
78
|
+
end
|
79
|
+
|
80
|
+
# start a TLS session
|
81
|
+
def start_tls
|
82
|
+
imap.starttls if @mailbox.start_tls
|
83
|
+
end
|
84
|
+
|
85
|
+
# send the imap login command to google
|
86
|
+
def log_in
|
87
|
+
imap.login(@mailbox.email, @mailbox.password)
|
88
|
+
@logged_in = true
|
89
|
+
end
|
90
|
+
|
91
|
+
# select the mailbox name we want to use
|
92
|
+
def set_mailbox
|
93
|
+
imap.select(@mailbox.name) if logged_in?
|
94
|
+
end
|
95
|
+
|
96
|
+
# is the response for a new message?
|
97
|
+
# @param response [Net::IMAP::TaggedResponse] the imap response from idle
|
98
|
+
# @return [Boolean]
|
99
|
+
def message_exists?(response)
|
100
|
+
response.respond_to?(:name) && response.name == 'EXISTS'
|
101
|
+
end
|
102
|
+
|
103
|
+
# @private
|
104
|
+
def idle_handler
|
105
|
+
->(response) { imap.idle_done if message_exists?(response) }
|
106
|
+
end
|
107
|
+
|
108
|
+
# maintain an imap idle connection
|
109
|
+
def idle
|
110
|
+
return unless ready_to_idle?
|
111
|
+
|
112
|
+
@mailbox.logger.info({ context: @mailbox.context, action: 'Idling' })
|
113
|
+
@idling = true
|
114
|
+
|
115
|
+
imap.idle(@mailbox.idle_timeout, &idle_handler)
|
116
|
+
ensure
|
117
|
+
@idling = false
|
118
|
+
end
|
119
|
+
|
120
|
+
# trigger the idle to finish and wait for the thread to finish
|
121
|
+
def stop_idling
|
122
|
+
return unless idling?
|
123
|
+
|
124
|
+
imap.idle_done
|
125
|
+
|
126
|
+
# idling_thread.join
|
127
|
+
# self.idling_thread = nil
|
128
|
+
end
|
129
|
+
|
130
|
+
def process_mailbox
|
131
|
+
return unless @new_message_handler
|
132
|
+
|
133
|
+
@mailbox.logger.info({ context: @mailbox.context, action: 'Processing started' })
|
134
|
+
|
135
|
+
msgs = new_messages
|
136
|
+
any_deletions = msgs.
|
137
|
+
# deliver each new message, collect success
|
138
|
+
map(&@new_message_handler).
|
139
|
+
# include messages with success
|
140
|
+
zip(msgs).
|
141
|
+
# filter failed deliveries, collect message
|
142
|
+
select(&:first).map(&:last).
|
143
|
+
# scrub delivered messages
|
144
|
+
map { |message| scrub(message) }
|
145
|
+
.any?
|
146
|
+
|
147
|
+
imap.expunge if @mailbox.expunge_deleted && any_deletions
|
148
|
+
end
|
149
|
+
|
150
|
+
def scrub(message)
|
151
|
+
if @mailbox.delete_after_delivery
|
152
|
+
imap.store(message.seqno, '+FLAGS', [Net::IMAP::DELETED])
|
153
|
+
true
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
# @private
|
158
|
+
# fetch all messages for the new message ids
|
159
|
+
def new_messages
|
160
|
+
# Both of these calls may results in
|
161
|
+
# imap raising an EOFError, we handle
|
162
|
+
# this exception in the watcher
|
163
|
+
messages_for_ids(new_message_ids)
|
164
|
+
end
|
165
|
+
|
166
|
+
# TODO: label messages?
|
167
|
+
# @imap.store(id, "+X-GM-LABELS", [label])
|
168
|
+
|
169
|
+
# @private
|
170
|
+
# search for all new (unseen) message ids
|
171
|
+
# @return [Array<Integer>] message ids
|
172
|
+
def new_message_ids
|
173
|
+
# uid_search still leaves messages UNSEEN
|
174
|
+
all_unread = imap.uid_search(@mailbox.search_command)
|
175
|
+
|
176
|
+
all_unread = all_unread.slice(0, @mailbox.limit_max_unread) if @mailbox.limit_max_unread.to_i > 0
|
177
|
+
|
178
|
+
to_deliver = all_unread.select { |uid| @mailbox.deliver?(uid) }
|
179
|
+
@mailbox.logger.info({ context: @mailbox.context, action: 'Getting new messages',
|
180
|
+
unread: { count: all_unread.count, ids: all_unread }, to_be_delivered: { count: to_deliver.count, ids: to_deliver } })
|
181
|
+
to_deliver
|
182
|
+
end
|
183
|
+
|
184
|
+
# @private
|
185
|
+
# fetch the email for all given ids in RFC822 format
|
186
|
+
# @param ids [Array<Integer>] list of message ids
|
187
|
+
# @return [Array<MailRoom::IMAP::Message>] the net/imap messages for the given ids
|
188
|
+
def messages_for_ids(uids)
|
189
|
+
return [] if uids.empty?
|
190
|
+
|
191
|
+
# uid_fetch marks as SEEN, will not be re-fetched for UNSEEN
|
192
|
+
imap_messages = imap.uid_fetch(uids, 'RFC822')
|
193
|
+
|
194
|
+
imap_messages.each_with_object([]) do |msg, messages|
|
195
|
+
messages << ::MailRoom::IMAP::Message.new(uid: msg.attr['UID'], body: msg.attr['RFC822'], seqno: msg.seqno)
|
196
|
+
end
|
197
|
+
end
|
198
|
+
end
|
199
|
+
end
|
200
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal:true
|
2
|
+
|
3
|
+
module MailRoom
|
4
|
+
module IMAP
|
5
|
+
class Message < MailRoom::Message
|
6
|
+
attr_reader :seqno
|
7
|
+
|
8
|
+
def initialize(uid:, body:, seqno:)
|
9
|
+
super(uid: uid, body: body)
|
10
|
+
|
11
|
+
@seqno = seqno
|
12
|
+
end
|
13
|
+
|
14
|
+
def ==(other)
|
15
|
+
super && seqno == other.seqno
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'date'
|
1
2
|
require 'logger'
|
2
3
|
require 'json'
|
3
4
|
|
@@ -10,12 +11,25 @@ module MailRoom
|
|
10
11
|
|
11
12
|
data = {}
|
12
13
|
data[:severity] = severity
|
13
|
-
data[:time] = timestamp || Time.now
|
14
|
+
data[:time] = format_timestamp(timestamp || Time.now)
|
14
15
|
# only accept a Hash
|
15
16
|
data.merge!(message)
|
16
17
|
|
17
18
|
data.to_json + "\n"
|
18
19
|
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def format_timestamp(timestamp)
|
24
|
+
case timestamp
|
25
|
+
when Time
|
26
|
+
timestamp.to_datetime.iso8601(3).to_s
|
27
|
+
when DateTime
|
28
|
+
timestamp.iso8601(3).to_s
|
29
|
+
else
|
30
|
+
timestamp
|
31
|
+
end
|
32
|
+
end
|
19
33
|
end
|
20
34
|
end
|
21
35
|
end
|
data/lib/mail_room/mailbox.rb
CHANGED
@@ -1,10 +1,14 @@
|
|
1
1
|
require "mail_room/delivery"
|
2
2
|
require "mail_room/arbitration"
|
3
|
+
require "mail_room/imap"
|
4
|
+
require "mail_room/microsoft_graph"
|
3
5
|
|
4
6
|
module MailRoom
|
5
7
|
# Mailbox Configuration fields
|
6
8
|
MAILBOX_FIELDS = [
|
7
9
|
:email,
|
10
|
+
:inbox_method,
|
11
|
+
:inbox_options,
|
8
12
|
:password,
|
9
13
|
:host,
|
10
14
|
:port,
|
@@ -41,24 +45,26 @@ module MailRoom
|
|
41
45
|
# 29 minutes, as suggested by the spec: https://tools.ietf.org/html/rfc2177
|
42
46
|
IMAP_IDLE_TIMEOUT = 29 * 60 # 29 minutes in in seconds
|
43
47
|
|
44
|
-
|
48
|
+
IMAP_CONFIGURATION = [:name, :email, :password, :host, :port].freeze
|
49
|
+
MICROSOFT_GRAPH_CONFIGURATION = [:name, :email].freeze
|
50
|
+
MICROSOFT_GRAPH_INBOX_OPTIONS = [:tenant_id, :client_id, :client_secret].freeze
|
45
51
|
|
46
52
|
# Default attributes for the mailbox configuration
|
47
53
|
DEFAULTS = {
|
48
|
-
:
|
49
|
-
:
|
50
|
-
:
|
51
|
-
:
|
52
|
-
:
|
53
|
-
:
|
54
|
-
:
|
55
|
-
:
|
56
|
-
:
|
57
|
-
:
|
58
|
-
:
|
59
|
-
:
|
60
|
-
:
|
61
|
-
:
|
54
|
+
search_command: 'UNSEEN',
|
55
|
+
delivery_method: 'postback',
|
56
|
+
host: 'imap.gmail.com',
|
57
|
+
port: 993,
|
58
|
+
ssl: true,
|
59
|
+
start_tls: false,
|
60
|
+
limit_max_unread: 0,
|
61
|
+
idle_timeout: IMAP_IDLE_TIMEOUT,
|
62
|
+
delete_after_delivery: false,
|
63
|
+
expunge_deleted: false,
|
64
|
+
delivery_options: {},
|
65
|
+
arbitration_method: 'noop',
|
66
|
+
arbitration_options: {},
|
67
|
+
logger: {}
|
62
68
|
}
|
63
69
|
|
64
70
|
# Store the configuration and require the appropriate delivery method
|
@@ -102,13 +108,13 @@ module MailRoom
|
|
102
108
|
arbitrator.deliver?(uid)
|
103
109
|
end
|
104
110
|
|
105
|
-
# deliver the
|
106
|
-
# @param message [
|
111
|
+
# deliver the email message
|
112
|
+
# @param message [MailRoom::Message]
|
107
113
|
def deliver(message)
|
108
|
-
body = message.
|
114
|
+
body = message.body
|
109
115
|
return true unless body
|
110
116
|
|
111
|
-
logger.info({context: context, uid: message.
|
117
|
+
logger.info({context: context, uid: message.uid, action: "sending to deliverer", deliverer: delivery.class.name, byte_size: body.bytesize})
|
112
118
|
delivery.deliver(body)
|
113
119
|
end
|
114
120
|
|
@@ -121,14 +127,32 @@ module MailRoom
|
|
121
127
|
{ email: self.email, name: self.name }
|
122
128
|
end
|
123
129
|
|
130
|
+
def imap?
|
131
|
+
!microsoft_graph?
|
132
|
+
end
|
133
|
+
|
134
|
+
def microsoft_graph?
|
135
|
+
self[:inbox_method].to_s == 'microsoft_graph'
|
136
|
+
end
|
137
|
+
|
124
138
|
def validate!
|
139
|
+
if microsoft_graph?
|
140
|
+
validate_microsoft_graph!
|
141
|
+
else
|
142
|
+
validate_imap!
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
private
|
147
|
+
|
148
|
+
def validate_imap!
|
125
149
|
if self[:idle_timeout] > IMAP_IDLE_TIMEOUT
|
126
150
|
raise IdleTimeoutTooLarge,
|
127
151
|
"Please use an idle timeout smaller than #{29*60} to prevent " \
|
128
152
|
"IMAP server disconnects"
|
129
153
|
end
|
130
154
|
|
131
|
-
|
155
|
+
IMAP_CONFIGURATION.each do |k|
|
132
156
|
if self[k].nil?
|
133
157
|
raise ConfigurationError,
|
134
158
|
"Field :#{k} is required in Mailbox: #{inspect}"
|
@@ -136,7 +160,23 @@ module MailRoom
|
|
136
160
|
end
|
137
161
|
end
|
138
162
|
|
139
|
-
|
163
|
+
def validate_microsoft_graph!
|
164
|
+
raise ConfigurationError, "Missing inbox_options in Mailbox: #{inspect}" unless self.inbox_options.is_a?(Hash)
|
165
|
+
|
166
|
+
MICROSOFT_GRAPH_CONFIGURATION.each do |k|
|
167
|
+
if self[k].nil?
|
168
|
+
raise ConfigurationError,
|
169
|
+
"Field :#{k} is required in Mailbox: #{inspect}"
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
MICROSOFT_GRAPH_INBOX_OPTIONS.each do |k|
|
174
|
+
if self[:inbox_options][k].nil?
|
175
|
+
raise ConfigurationError,
|
176
|
+
"inbox_options field :#{k} is required in Mailbox: #{inspect}"
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
140
180
|
|
141
181
|
def parsed_arbitration_options
|
142
182
|
arbitration_klass::Options.new(self)
|