gitlab-mail_room 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +18 -0
- data/.gitlab-ci.yml +27 -0
- data/.ruby-version +1 -0
- data/.travis.yml +10 -0
- data/CHANGELOG.md +125 -0
- data/CODE_OF_CONDUCT.md +24 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +361 -0
- data/Rakefile +6 -0
- data/bin/mail_room +5 -0
- data/lib/mail_room.rb +16 -0
- data/lib/mail_room/arbitration.rb +16 -0
- data/lib/mail_room/arbitration/noop.rb +18 -0
- data/lib/mail_room/arbitration/redis.rb +58 -0
- data/lib/mail_room/cli.rb +62 -0
- data/lib/mail_room/configuration.rb +36 -0
- data/lib/mail_room/connection.rb +195 -0
- data/lib/mail_room/coordinator.rb +41 -0
- data/lib/mail_room/crash_handler.rb +29 -0
- data/lib/mail_room/delivery.rb +24 -0
- data/lib/mail_room/delivery/letter_opener.rb +34 -0
- data/lib/mail_room/delivery/logger.rb +37 -0
- data/lib/mail_room/delivery/noop.rb +22 -0
- data/lib/mail_room/delivery/postback.rb +72 -0
- data/lib/mail_room/delivery/que.rb +63 -0
- data/lib/mail_room/delivery/sidekiq.rb +88 -0
- data/lib/mail_room/logger/structured.rb +21 -0
- data/lib/mail_room/mailbox.rb +182 -0
- data/lib/mail_room/mailbox_watcher.rb +62 -0
- data/lib/mail_room/version.rb +4 -0
- data/logfile.log +1 -0
- data/mail_room.gemspec +34 -0
- data/spec/fixtures/test_config.yml +16 -0
- data/spec/lib/arbitration/redis_spec.rb +146 -0
- data/spec/lib/cli_spec.rb +61 -0
- data/spec/lib/configuration_spec.rb +29 -0
- data/spec/lib/connection_spec.rb +65 -0
- data/spec/lib/coordinator_spec.rb +61 -0
- data/spec/lib/crash_handler_spec.rb +41 -0
- data/spec/lib/delivery/letter_opener_spec.rb +29 -0
- data/spec/lib/delivery/logger_spec.rb +46 -0
- data/spec/lib/delivery/postback_spec.rb +107 -0
- data/spec/lib/delivery/que_spec.rb +45 -0
- data/spec/lib/delivery/sidekiq_spec.rb +76 -0
- data/spec/lib/logger/structured_spec.rb +55 -0
- data/spec/lib/mailbox_spec.rb +132 -0
- data/spec/lib/mailbox_watcher_spec.rb +64 -0
- data/spec/spec_helper.rb +32 -0
- metadata +277 -0
data/Rakefile
ADDED
data/bin/mail_room
ADDED
data/lib/mail_room.rb
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'net/imap'
|
2
|
+
require 'optparse'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
module MailRoom
|
6
|
+
end
|
7
|
+
|
8
|
+
require "mail_room/version"
|
9
|
+
require "mail_room/configuration"
|
10
|
+
require "mail_room/mailbox"
|
11
|
+
require "mail_room/mailbox_watcher"
|
12
|
+
require "mail_room/connection"
|
13
|
+
require "mail_room/coordinator"
|
14
|
+
require "mail_room/cli"
|
15
|
+
require 'mail_room/logger/structured'
|
16
|
+
require 'mail_room/crash_handler'
|
@@ -0,0 +1,58 @@
|
|
1
|
+
require "redis"
|
2
|
+
|
3
|
+
module MailRoom
|
4
|
+
module Arbitration
|
5
|
+
class Redis
|
6
|
+
Options = Struct.new(:redis_url, :namespace, :sentinels) do
|
7
|
+
def initialize(mailbox)
|
8
|
+
redis_url = mailbox.arbitration_options[:redis_url] || "redis://localhost:6379"
|
9
|
+
namespace = mailbox.arbitration_options[:namespace]
|
10
|
+
sentinels = mailbox.arbitration_options[:sentinels]
|
11
|
+
|
12
|
+
super(redis_url, namespace, sentinels)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
# Expire after 10 minutes so Redis doesn't get filled up with outdated data.
|
17
|
+
EXPIRATION = 600
|
18
|
+
|
19
|
+
attr_accessor :options
|
20
|
+
|
21
|
+
def initialize(options)
|
22
|
+
@options = options
|
23
|
+
end
|
24
|
+
|
25
|
+
def deliver?(uid, expiration = EXPIRATION)
|
26
|
+
key = "delivered:#{uid}"
|
27
|
+
|
28
|
+
# Set the key, but only if it doesn't already exist;
|
29
|
+
# the return value is true if successful, false if the key was already set,
|
30
|
+
# which is conveniently the correct return value for this method
|
31
|
+
# Any subsequent failure in the instance which gets the lock will be dealt
|
32
|
+
# with by the expiration, at which time another instance can pick up the
|
33
|
+
# message and try again.
|
34
|
+
client.set(key, 1, {:nx => true, :ex => expiration})
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def client
|
40
|
+
@client ||= begin
|
41
|
+
sentinels = options.sentinels
|
42
|
+
redis_options = { url: options.redis_url }
|
43
|
+
redis_options[:sentinels] = sentinels if sentinels
|
44
|
+
|
45
|
+
redis = ::Redis.new(redis_options)
|
46
|
+
|
47
|
+
namespace = options.namespace
|
48
|
+
if namespace
|
49
|
+
require 'redis/namespace'
|
50
|
+
::Redis::Namespace.new(namespace, redis: redis)
|
51
|
+
else
|
52
|
+
redis
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
module MailRoom
|
2
|
+
# The CLI parses ARGV into configuration to start the coordinator with.
|
3
|
+
# @author Tony Pitale
|
4
|
+
class CLI
|
5
|
+
attr_accessor :configuration, :coordinator, :options
|
6
|
+
|
7
|
+
# Initialize a new CLI instance to handle option parsing from arguments
|
8
|
+
# into configuration to start the coordinator running on all mailboxes
|
9
|
+
#
|
10
|
+
# @param args [Array] `ARGV` passed from `bin/mail_room`
|
11
|
+
def initialize(args)
|
12
|
+
@options = {}
|
13
|
+
|
14
|
+
OptionParser.new do |parser|
|
15
|
+
parser.banner = [
|
16
|
+
"Usage: #{@name} [-c config_file]\n",
|
17
|
+
" #{@name} --help\n"
|
18
|
+
].compact.join
|
19
|
+
|
20
|
+
parser.on('-c', '--config FILE') do |path|
|
21
|
+
options[:config_path] = path
|
22
|
+
end
|
23
|
+
|
24
|
+
parser.on('-q', '--quiet') do
|
25
|
+
options[:quiet] = true
|
26
|
+
end
|
27
|
+
|
28
|
+
parser.on('--log-exit-as') do |format|
|
29
|
+
options[:exit_error_format] = 'json' unless format.nil?
|
30
|
+
end
|
31
|
+
|
32
|
+
# parser.on("-l", "--log FILE") do |path|
|
33
|
+
# options[:log_path] = path
|
34
|
+
# end
|
35
|
+
|
36
|
+
parser.on_tail("-?", "--help", "Display this usage information.") do
|
37
|
+
puts "#{parser}\n"
|
38
|
+
exit
|
39
|
+
end
|
40
|
+
end.parse!(args)
|
41
|
+
|
42
|
+
self.configuration = Configuration.new(options)
|
43
|
+
self.coordinator = Coordinator.new(configuration.mailboxes)
|
44
|
+
end
|
45
|
+
|
46
|
+
# Start the coordinator running, sets up signal traps
|
47
|
+
def start
|
48
|
+
Signal.trap(:INT) do
|
49
|
+
coordinator.running = false
|
50
|
+
end
|
51
|
+
|
52
|
+
Signal.trap(:TERM) do
|
53
|
+
exit
|
54
|
+
end
|
55
|
+
|
56
|
+
coordinator.run
|
57
|
+
rescue Exception => e # not just Errors, but includes lower-level Exceptions
|
58
|
+
CrashHandler.new(error: e, format: @options[:exit_error_format]).handle
|
59
|
+
exit
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require "erb"
|
2
|
+
|
3
|
+
module MailRoom
|
4
|
+
# Wraps configuration for a set of individual mailboxes with global config
|
5
|
+
# @author Tony Pitale
|
6
|
+
class Configuration
|
7
|
+
attr_accessor :mailboxes, :log_path, :quiet
|
8
|
+
|
9
|
+
# Initialize a new configuration of mailboxes
|
10
|
+
def initialize(options={})
|
11
|
+
self.mailboxes = []
|
12
|
+
self.quiet = options.fetch(:quiet, false)
|
13
|
+
|
14
|
+
if options.has_key?(:config_path)
|
15
|
+
begin
|
16
|
+
erb = ERB.new(File.read(options[:config_path]))
|
17
|
+
erb.filename = options[:config_path]
|
18
|
+
config_file = YAML.load(erb.result)
|
19
|
+
|
20
|
+
set_mailboxes(config_file[:mailboxes])
|
21
|
+
rescue => e
|
22
|
+
raise e unless quiet
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
# Builds individual mailboxes from YAML configuration
|
28
|
+
#
|
29
|
+
# @param mailboxes_config
|
30
|
+
def set_mailboxes(mailboxes_config)
|
31
|
+
mailboxes_config.each do |attributes|
|
32
|
+
self.mailboxes << Mailbox.new(attributes)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,195 @@
|
|
1
|
+
module MailRoom
|
2
|
+
class Connection
|
3
|
+
def initialize(mailbox)
|
4
|
+
@mailbox = mailbox
|
5
|
+
|
6
|
+
# log in and set the mailbox
|
7
|
+
reset
|
8
|
+
setup
|
9
|
+
end
|
10
|
+
|
11
|
+
def on_new_message(&block)
|
12
|
+
@new_message_handler = block
|
13
|
+
end
|
14
|
+
|
15
|
+
# is the connection logged in?
|
16
|
+
# @return [Boolean]
|
17
|
+
def logged_in?
|
18
|
+
@logged_in
|
19
|
+
end
|
20
|
+
|
21
|
+
# is the connection blocked idling?
|
22
|
+
# @return [Boolean]
|
23
|
+
def idling?
|
24
|
+
@idling
|
25
|
+
end
|
26
|
+
|
27
|
+
# is the imap connection closed?
|
28
|
+
# @return [Boolean]
|
29
|
+
def disconnected?
|
30
|
+
@imap.disconnected?
|
31
|
+
end
|
32
|
+
|
33
|
+
# is the connection ready to idle?
|
34
|
+
# @return [Boolean]
|
35
|
+
def ready_to_idle?
|
36
|
+
logged_in? && !idling?
|
37
|
+
end
|
38
|
+
|
39
|
+
def quit
|
40
|
+
stop_idling
|
41
|
+
reset
|
42
|
+
end
|
43
|
+
|
44
|
+
def wait
|
45
|
+
begin
|
46
|
+
# in case we missed any between idles
|
47
|
+
process_mailbox
|
48
|
+
|
49
|
+
idle
|
50
|
+
|
51
|
+
process_mailbox
|
52
|
+
rescue Net::IMAP::Error, IOError
|
53
|
+
@mailbox.logger.warn({ context: @mailbox.context, action: "Disconnected. Resetting..." })
|
54
|
+
reset
|
55
|
+
setup
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
private
|
60
|
+
|
61
|
+
def reset
|
62
|
+
@imap = nil
|
63
|
+
@logged_in = false
|
64
|
+
@idling = false
|
65
|
+
end
|
66
|
+
|
67
|
+
def setup
|
68
|
+
@mailbox.logger.info({ context: @mailbox.context, action: "Starting TLS session" })
|
69
|
+
start_tls
|
70
|
+
|
71
|
+
@mailbox.logger.info({ context: @mailbox.context, action: "Logging into mailbox" })
|
72
|
+
log_in
|
73
|
+
|
74
|
+
@mailbox.logger.info({ context: @mailbox.context, action: "Setting mailbox" })
|
75
|
+
set_mailbox
|
76
|
+
end
|
77
|
+
|
78
|
+
# build a net/imap connection to google imap
|
79
|
+
def imap
|
80
|
+
@imap ||= Net::IMAP.new(@mailbox.host, :port => @mailbox.port, :ssl => @mailbox.ssl_options)
|
81
|
+
end
|
82
|
+
|
83
|
+
# start a TLS session
|
84
|
+
def start_tls
|
85
|
+
imap.starttls if @mailbox.start_tls
|
86
|
+
end
|
87
|
+
|
88
|
+
# send the imap login command to google
|
89
|
+
def log_in
|
90
|
+
imap.login(@mailbox.email, @mailbox.password)
|
91
|
+
@logged_in = true
|
92
|
+
end
|
93
|
+
|
94
|
+
# select the mailbox name we want to use
|
95
|
+
def set_mailbox
|
96
|
+
imap.select(@mailbox.name) if logged_in?
|
97
|
+
end
|
98
|
+
|
99
|
+
# is the response for a new message?
|
100
|
+
# @param response [Net::IMAP::TaggedResponse] the imap response from idle
|
101
|
+
# @return [Boolean]
|
102
|
+
def message_exists?(response)
|
103
|
+
response.respond_to?(:name) && response.name == 'EXISTS'
|
104
|
+
end
|
105
|
+
|
106
|
+
# @private
|
107
|
+
def idle_handler
|
108
|
+
lambda {|response| imap.idle_done if message_exists?(response)}
|
109
|
+
end
|
110
|
+
|
111
|
+
# maintain an imap idle connection
|
112
|
+
def idle
|
113
|
+
return unless ready_to_idle?
|
114
|
+
|
115
|
+
@mailbox.logger.info({ context: @mailbox.context, action: "Idling" })
|
116
|
+
@idling = true
|
117
|
+
|
118
|
+
imap.idle(@mailbox.idle_timeout, &idle_handler)
|
119
|
+
ensure
|
120
|
+
@idling = false
|
121
|
+
end
|
122
|
+
|
123
|
+
# trigger the idle to finish and wait for the thread to finish
|
124
|
+
def stop_idling
|
125
|
+
return unless idling?
|
126
|
+
|
127
|
+
imap.idle_done
|
128
|
+
|
129
|
+
# idling_thread.join
|
130
|
+
# self.idling_thread = nil
|
131
|
+
end
|
132
|
+
|
133
|
+
def process_mailbox
|
134
|
+
return unless @new_message_handler
|
135
|
+
@mailbox.logger.info({ context: @mailbox.context, action: "Processing started" })
|
136
|
+
|
137
|
+
msgs = new_messages
|
138
|
+
|
139
|
+
any_deletions = msgs.
|
140
|
+
# deliver each new message, collect success
|
141
|
+
map(&@new_message_handler).
|
142
|
+
# include messages with success
|
143
|
+
zip(msgs).
|
144
|
+
# filter failed deliveries, collect message
|
145
|
+
select(&:first).map(&:last).
|
146
|
+
# scrub delivered messages
|
147
|
+
map { |message| scrub(message) }.
|
148
|
+
any?
|
149
|
+
|
150
|
+
imap.expunge if @mailbox.expunge_deleted && any_deletions
|
151
|
+
end
|
152
|
+
|
153
|
+
def scrub(message)
|
154
|
+
if @mailbox.delete_after_delivery
|
155
|
+
imap.store(message.seqno, "+FLAGS", [Net::IMAP::DELETED])
|
156
|
+
true
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
# @private
|
161
|
+
# fetch all messages for the new message ids
|
162
|
+
def new_messages
|
163
|
+
# Both of these calls may results in
|
164
|
+
# imap raising an EOFError, we handle
|
165
|
+
# this exception in the watcher
|
166
|
+
messages_for_ids(new_message_ids)
|
167
|
+
end
|
168
|
+
|
169
|
+
# TODO: label messages?
|
170
|
+
# @imap.store(id, "+X-GM-LABELS", [label])
|
171
|
+
|
172
|
+
# @private
|
173
|
+
# search for all new (unseen) message ids
|
174
|
+
# @return [Array<Integer>] message ids
|
175
|
+
def new_message_ids
|
176
|
+
# uid_search still leaves messages UNSEEN
|
177
|
+
all_unread = @imap.uid_search(@mailbox.search_command)
|
178
|
+
|
179
|
+
to_deliver = all_unread.select { |uid| @mailbox.deliver?(uid) }
|
180
|
+
@mailbox.logger.info({ context: @mailbox.context, action: "Getting new messages", unread: {count: all_unread.count, ids: all_unread}, to_be_delivered: { count: to_deliver.count, ids: all_unread } })
|
181
|
+
to_deliver
|
182
|
+
end
|
183
|
+
|
184
|
+
# @private
|
185
|
+
# fetch the email for all given ids in RFC822 format
|
186
|
+
# @param ids [Array<Integer>] list of message ids
|
187
|
+
# @return [Array<Net::IMAP::FetchData>] the net/imap messages for the given ids
|
188
|
+
def messages_for_ids(uids)
|
189
|
+
return [] if uids.empty?
|
190
|
+
|
191
|
+
# uid_fetch marks as SEEN, will not be re-fetched for UNSEEN
|
192
|
+
imap.uid_fetch(uids, "RFC822")
|
193
|
+
end
|
194
|
+
end
|
195
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module MailRoom
|
2
|
+
# Coordinate the mailbox watchers
|
3
|
+
# @author Tony Pitale
|
4
|
+
class Coordinator
|
5
|
+
attr_accessor :watchers, :running
|
6
|
+
|
7
|
+
# build watchers for a set of mailboxes
|
8
|
+
# @params mailboxes [Array<MailRoom::Mailbox>] mailboxes to be watched
|
9
|
+
def initialize(mailboxes)
|
10
|
+
self.watchers = []
|
11
|
+
|
12
|
+
mailboxes.each {|box| self.watchers << MailboxWatcher.new(box)}
|
13
|
+
end
|
14
|
+
|
15
|
+
alias :running? :running
|
16
|
+
|
17
|
+
# start each of the watchers to running
|
18
|
+
def run
|
19
|
+
watchers.each(&:run)
|
20
|
+
|
21
|
+
self.running = true
|
22
|
+
|
23
|
+
sleep_while_running
|
24
|
+
ensure
|
25
|
+
quit
|
26
|
+
end
|
27
|
+
|
28
|
+
# quit each of the watchers when we're done running
|
29
|
+
def quit
|
30
|
+
watchers.each(&:quit)
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
# @private
|
35
|
+
def sleep_while_running
|
36
|
+
# do we need to sweep for dead watchers?
|
37
|
+
# or do we let the mailbox rebuild connections
|
38
|
+
while(running?) do; sleep 1; end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|