gitlab-mail_room 0.0.4 → 0.0.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/.gitlab-ci.yml +14 -5
  3. data/.gitlab/issue_templates/Release.md +7 -0
  4. data/.ruby-version +1 -1
  5. data/.travis.yml +3 -2
  6. data/README.md +16 -1
  7. data/lib/mail_room.rb +2 -0
  8. data/lib/mail_room/cli.rb +2 -2
  9. data/lib/mail_room/configuration.rb +11 -1
  10. data/lib/mail_room/connection.rb +7 -179
  11. data/lib/mail_room/coordinator.rb +8 -4
  12. data/lib/mail_room/crash_handler.rb +8 -12
  13. data/lib/mail_room/health_check.rb +60 -0
  14. data/lib/mail_room/imap.rb +8 -0
  15. data/lib/mail_room/imap/connection.rb +200 -0
  16. data/lib/mail_room/imap/message.rb +19 -0
  17. data/lib/mail_room/logger/structured.rb +15 -1
  18. data/lib/mail_room/mailbox.rb +7 -4
  19. data/lib/mail_room/mailbox_watcher.rb +9 -2
  20. data/lib/mail_room/message.rb +16 -0
  21. data/lib/mail_room/version.rb +2 -2
  22. data/mail_room.gemspec +5 -3
  23. data/spec/fixtures/test_config.yml +3 -0
  24. data/spec/lib/arbitration/redis_spec.rb +3 -2
  25. data/spec/lib/cli_spec.rb +30 -15
  26. data/spec/lib/configuration_spec.rb +9 -2
  27. data/spec/lib/coordinator_spec.rb +27 -11
  28. data/spec/lib/crash_handler_spec.rb +10 -9
  29. data/spec/lib/delivery/letter_opener_spec.rb +9 -5
  30. data/spec/lib/delivery/logger_spec.rb +7 -9
  31. data/spec/lib/delivery/postback_spec.rb +11 -27
  32. data/spec/lib/delivery/que_spec.rb +5 -8
  33. data/spec/lib/health_check_spec.rb +57 -0
  34. data/spec/lib/{connection_spec.rb → imap/connection_spec.rb} +13 -17
  35. data/spec/lib/imap/message_spec.rb +36 -0
  36. data/spec/lib/logger/structured_spec.rb +34 -2
  37. data/spec/lib/mailbox_spec.rb +14 -17
  38. data/spec/lib/mailbox_watcher_spec.rb +9 -12
  39. data/spec/lib/message_spec.rb +35 -0
  40. data/spec/spec_helper.rb +1 -1
  41. metadata +45 -19
@@ -0,0 +1,60 @@
1
+ # frozen_string_literal: true
2
+
3
+ module MailRoom
4
+ class HealthCheck
5
+ attr_reader :address, :port, :running
6
+
7
+ def initialize(attributes = {})
8
+ @address = attributes[:address]
9
+ @port = attributes[:port]
10
+
11
+ validate!
12
+ end
13
+
14
+ def run
15
+ @server = create_server
16
+
17
+ @thread = Thread.new do
18
+ @server.start
19
+ end
20
+
21
+ @thread.abort_on_exception = true
22
+ @running = true
23
+ end
24
+
25
+ def quit
26
+ @running = false
27
+ @server&.shutdown
28
+ @thread&.join(60)
29
+ end
30
+
31
+ private
32
+
33
+ def validate!
34
+ raise 'No health check address specified' unless address
35
+ raise "Health check port #{@port.to_i} is invalid" unless port.to_i.positive?
36
+ end
37
+
38
+ def create_server
39
+ require 'webrick'
40
+
41
+ server = ::WEBrick::HTTPServer.new(Port: port, BindAddress: address, AccessLog: [])
42
+
43
+ server.mount_proc '/liveness' do |_req, res|
44
+ handle_liveness(res)
45
+ end
46
+
47
+ server
48
+ end
49
+
50
+ def handle_liveness(res)
51
+ if @running
52
+ res.status = 200
53
+ res.body = "OK\n"
54
+ else
55
+ res.status = 500
56
+ res.body = "Not running\n"
57
+ end
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ module MailRoom
4
+ module IMAP
5
+ autoload :Connection, 'mail_room/imap/connection'
6
+ autoload :Message, 'mail_room/imap/message'
7
+ end
8
+ end
@@ -0,0 +1,200 @@
1
+ # frozen_string_literal: true
2
+
3
+ module MailRoom
4
+ module IMAP
5
+ class Connection < MailRoom::Connection
6
+ def initialize(mailbox)
7
+ super
8
+
9
+ # log in and set the mailbox
10
+ reset
11
+ setup
12
+ end
13
+
14
+ # is the connection logged in?
15
+ # @return [Boolean]
16
+ def logged_in?
17
+ @logged_in
18
+ end
19
+
20
+ # is the connection blocked idling?
21
+ # @return [Boolean]
22
+ def idling?
23
+ @idling
24
+ end
25
+
26
+ # is the imap connection closed?
27
+ # @return [Boolean]
28
+ def disconnected?
29
+ imap.disconnected?
30
+ end
31
+
32
+ # is the connection ready to idle?
33
+ # @return [Boolean]
34
+ def ready_to_idle?
35
+ logged_in? && !idling?
36
+ end
37
+
38
+ def quit
39
+ stop_idling
40
+ reset
41
+ end
42
+
43
+ def wait
44
+ # in case we missed any between idles
45
+ process_mailbox
46
+
47
+ idle
48
+
49
+ process_mailbox
50
+ rescue Net::IMAP::Error, IOError => e
51
+ @mailbox.logger.warn({ context: @mailbox.context, action: 'Disconnected. Resetting...', error: e.message })
52
+ reset
53
+ setup
54
+ end
55
+
56
+ private
57
+
58
+ def reset
59
+ @imap = nil
60
+ @logged_in = false
61
+ @idling = false
62
+ end
63
+
64
+ def setup
65
+ @mailbox.logger.info({ context: @mailbox.context, action: 'Starting TLS session' })
66
+ start_tls
67
+
68
+ @mailbox.logger.info({ context: @mailbox.context, action: 'Logging into mailbox' })
69
+ log_in
70
+
71
+ @mailbox.logger.info({ context: @mailbox.context, action: 'Setting mailbox' })
72
+ set_mailbox
73
+ end
74
+
75
+ # build a net/imap connection to google imap
76
+ def imap
77
+ @imap ||= Net::IMAP.new(@mailbox.host, port: @mailbox.port, ssl: @mailbox.ssl_options)
78
+ end
79
+
80
+ # start a TLS session
81
+ def start_tls
82
+ imap.starttls if @mailbox.start_tls
83
+ end
84
+
85
+ # send the imap login command to google
86
+ def log_in
87
+ imap.login(@mailbox.email, @mailbox.password)
88
+ @logged_in = true
89
+ end
90
+
91
+ # select the mailbox name we want to use
92
+ def set_mailbox
93
+ imap.select(@mailbox.name) if logged_in?
94
+ end
95
+
96
+ # is the response for a new message?
97
+ # @param response [Net::IMAP::TaggedResponse] the imap response from idle
98
+ # @return [Boolean]
99
+ def message_exists?(response)
100
+ response.respond_to?(:name) && response.name == 'EXISTS'
101
+ end
102
+
103
+ # @private
104
+ def idle_handler
105
+ ->(response) { imap.idle_done if message_exists?(response) }
106
+ end
107
+
108
+ # maintain an imap idle connection
109
+ def idle
110
+ return unless ready_to_idle?
111
+
112
+ @mailbox.logger.info({ context: @mailbox.context, action: 'Idling' })
113
+ @idling = true
114
+
115
+ imap.idle(@mailbox.idle_timeout, &idle_handler)
116
+ ensure
117
+ @idling = false
118
+ end
119
+
120
+ # trigger the idle to finish and wait for the thread to finish
121
+ def stop_idling
122
+ return unless idling?
123
+
124
+ imap.idle_done
125
+
126
+ # idling_thread.join
127
+ # self.idling_thread = nil
128
+ end
129
+
130
+ def process_mailbox
131
+ return unless @new_message_handler
132
+
133
+ @mailbox.logger.info({ context: @mailbox.context, action: 'Processing started' })
134
+
135
+ msgs = new_messages
136
+ any_deletions = msgs.
137
+ # deliver each new message, collect success
138
+ map(&@new_message_handler).
139
+ # include messages with success
140
+ zip(msgs).
141
+ # filter failed deliveries, collect message
142
+ select(&:first).map(&:last).
143
+ # scrub delivered messages
144
+ map { |message| scrub(message) }
145
+ .any?
146
+
147
+ imap.expunge if @mailbox.expunge_deleted && any_deletions
148
+ end
149
+
150
+ def scrub(message)
151
+ if @mailbox.delete_after_delivery
152
+ imap.store(message.seqno, '+FLAGS', [Net::IMAP::DELETED])
153
+ true
154
+ end
155
+ end
156
+
157
+ # @private
158
+ # fetch all messages for the new message ids
159
+ def new_messages
160
+ # Both of these calls may results in
161
+ # imap raising an EOFError, we handle
162
+ # this exception in the watcher
163
+ messages_for_ids(new_message_ids)
164
+ end
165
+
166
+ # TODO: label messages?
167
+ # @imap.store(id, "+X-GM-LABELS", [label])
168
+
169
+ # @private
170
+ # search for all new (unseen) message ids
171
+ # @return [Array<Integer>] message ids
172
+ def new_message_ids
173
+ # uid_search still leaves messages UNSEEN
174
+ all_unread = imap.uid_search(@mailbox.search_command)
175
+
176
+ all_unread = all_unread.slice(0, @mailbox.limit_max_unread) if @mailbox.limit_max_unread.to_i > 0
177
+
178
+ to_deliver = all_unread.select { |uid| @mailbox.deliver?(uid) }
179
+ @mailbox.logger.info({ context: @mailbox.context, action: 'Getting new messages',
180
+ unread: { count: all_unread.count, ids: all_unread }, to_be_delivered: { count: to_deliver.count, ids: to_deliver } })
181
+ to_deliver
182
+ end
183
+
184
+ # @private
185
+ # fetch the email for all given ids in RFC822 format
186
+ # @param ids [Array<Integer>] list of message ids
187
+ # @return [Array<MailRoom::IMAP::Message>] the net/imap messages for the given ids
188
+ def messages_for_ids(uids)
189
+ return [] if uids.empty?
190
+
191
+ # uid_fetch marks as SEEN, will not be re-fetched for UNSEEN
192
+ imap_messages = imap.uid_fetch(uids, 'RFC822')
193
+
194
+ imap_messages.each_with_object([]) do |msg, messages|
195
+ messages << ::MailRoom::IMAP::Message.new(uid: msg.attr['UID'], body: msg.attr['RFC822'], seqno: msg.seqno)
196
+ end
197
+ end
198
+ end
199
+ end
200
+ end
@@ -0,0 +1,19 @@
1
+ # frozen_string_literal:true
2
+
3
+ module MailRoom
4
+ module IMAP
5
+ class Message < MailRoom::Message
6
+ attr_reader :seqno
7
+
8
+ def initialize(uid:, body:, seqno:)
9
+ super(uid: uid, body: body)
10
+
11
+ @seqno = seqno
12
+ end
13
+
14
+ def ==(other)
15
+ super && seqno == other.seqno
16
+ end
17
+ end
18
+ end
19
+ end
@@ -1,3 +1,4 @@
1
+ require 'date'
1
2
  require 'logger'
2
3
  require 'json'
3
4
 
@@ -10,12 +11,25 @@ module MailRoom
10
11
 
11
12
  data = {}
12
13
  data[:severity] = severity
13
- data[:time] = timestamp || Time.now.to_s
14
+ data[:time] = format_timestamp(timestamp || Time.now)
14
15
  # only accept a Hash
15
16
  data.merge!(message)
16
17
 
17
18
  data.to_json + "\n"
18
19
  end
20
+
21
+ private
22
+
23
+ def format_timestamp(timestamp)
24
+ case timestamp
25
+ when Time
26
+ timestamp.to_datetime.iso8601(3).to_s
27
+ when DateTime
28
+ timestamp.iso8601(3).to_s
29
+ else
30
+ timestamp
31
+ end
32
+ end
19
33
  end
20
34
  end
21
35
  end
@@ -1,5 +1,6 @@
1
1
  require "mail_room/delivery"
2
2
  require "mail_room/arbitration"
3
+ require "mail_room/imap"
3
4
 
4
5
  module MailRoom
5
6
  # Mailbox Configuration fields
@@ -10,6 +11,7 @@ module MailRoom
10
11
  :port,
11
12
  :ssl,
12
13
  :start_tls,
14
+ :limit_max_unread, #to avoid 'Error in IMAP command UID FETCH: Too long argument'
13
15
  :idle_timeout,
14
16
  :search_command,
15
17
  :name,
@@ -50,6 +52,7 @@ module MailRoom
50
52
  :port => 993,
51
53
  :ssl => true,
52
54
  :start_tls => false,
55
+ :limit_max_unread => 0,
53
56
  :idle_timeout => IMAP_IDLE_TIMEOUT,
54
57
  :delete_after_delivery => false,
55
58
  :expunge_deleted => false,
@@ -100,13 +103,13 @@ module MailRoom
100
103
  arbitrator.deliver?(uid)
101
104
  end
102
105
 
103
- # deliver the imap email message
104
- # @param message [Net::IMAP::FetchData]
106
+ # deliver the email message
107
+ # @param message [MailRoom::Message]
105
108
  def deliver(message)
106
- body = message.attr['RFC822']
109
+ body = message.body
107
110
  return true unless body
108
111
 
109
- logger.info({context: context, uid: message.attr['UID'], action: "sending to deliverer", deliverer: delivery.class.name, byte_size: body.bytesize})
112
+ logger.info({context: context, uid: message.uid, action: "sending to deliverer", deliverer: delivery.class.name, byte_size: body.bytesize})
110
113
  delivery.deliver(body)
111
114
  end
112
115
 
@@ -1,3 +1,5 @@
1
+ require "mail_room/connection"
2
+
1
3
  module MailRoom
2
4
  # TODO: split up between processing and idling?
3
5
 
@@ -49,14 +51,19 @@ module MailRoom
49
51
  @connection = nil
50
52
  end
51
53
 
54
+ @mailbox.logger.info({ context: @mailbox.context, action: "Terminating watching thread..." })
55
+
52
56
  if self.watching_thread
53
- self.watching_thread.join
57
+ thr = self.watching_thread.join(60)
58
+ @mailbox.logger.info({ context: @mailbox.context, action: "Timeout waiting for watching thread" }) unless thr
54
59
  end
60
+
61
+ @mailbox.logger.info({ context: @mailbox.context, action: "Done with thread cleanup" })
55
62
  end
56
63
 
57
64
  private
58
65
  def connection
59
- @connection ||= Connection.new(@mailbox)
66
+ @connection ||= ::MailRoom::IMAP::Connection.new(@mailbox)
60
67
  end
61
68
  end
62
69
  end
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ module MailRoom
4
+ class Message
5
+ attr_reader :uid, :body
6
+
7
+ def initialize(uid:, body:)
8
+ @uid = uid
9
+ @body = body
10
+ end
11
+
12
+ def ==(other)
13
+ self.class == other.class && uid == other.uid && body == other.body
14
+ end
15
+ end
16
+ end
@@ -1,4 +1,4 @@
1
1
  module MailRoom
2
- # Current version of MailRoom gem
3
- VERSION = "0.0.4"
2
+ # Current version of gitlab-mail_room gem
3
+ VERSION = "0.0.10"
4
4
  end
data/mail_room.gemspec CHANGED
@@ -17,11 +17,13 @@ Gem::Specification.new do |gem|
17
17
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
18
18
  gem.require_paths = ["lib"]
19
19
 
20
+ gem.add_dependency "net-imap", ">= 0.2.1"
21
+
20
22
  gem.add_development_dependency "rake"
21
- gem.add_development_dependency "rspec"
22
- gem.add_development_dependency "mocha"
23
- gem.add_development_dependency "bourne"
23
+ gem.add_development_dependency "rspec", "~> 3.9"
24
+ gem.add_development_dependency "mocha", "~> 1.11"
24
25
  gem.add_development_dependency "simplecov"
26
+ gem.add_development_dependency "webrick", "~> 1.6"
25
27
 
26
28
  # for testing delivery methods
27
29
  gem.add_development_dependency "faraday"
@@ -1,4 +1,7 @@
1
1
  ---
2
+ :health_check:
3
+ :address: "127.0.0.1"
4
+ :port: 8080
2
5
  :mailboxes:
3
6
  -
4
7
  :email: "user1@gmail.com"