gitlab-mail_room 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +18 -0
  3. data/.gitlab-ci.yml +27 -0
  4. data/.ruby-version +1 -0
  5. data/.travis.yml +10 -0
  6. data/CHANGELOG.md +125 -0
  7. data/CODE_OF_CONDUCT.md +24 -0
  8. data/Gemfile +4 -0
  9. data/LICENSE.txt +22 -0
  10. data/README.md +361 -0
  11. data/Rakefile +6 -0
  12. data/bin/mail_room +5 -0
  13. data/lib/mail_room.rb +16 -0
  14. data/lib/mail_room/arbitration.rb +16 -0
  15. data/lib/mail_room/arbitration/noop.rb +18 -0
  16. data/lib/mail_room/arbitration/redis.rb +58 -0
  17. data/lib/mail_room/cli.rb +62 -0
  18. data/lib/mail_room/configuration.rb +36 -0
  19. data/lib/mail_room/connection.rb +195 -0
  20. data/lib/mail_room/coordinator.rb +41 -0
  21. data/lib/mail_room/crash_handler.rb +29 -0
  22. data/lib/mail_room/delivery.rb +24 -0
  23. data/lib/mail_room/delivery/letter_opener.rb +34 -0
  24. data/lib/mail_room/delivery/logger.rb +37 -0
  25. data/lib/mail_room/delivery/noop.rb +22 -0
  26. data/lib/mail_room/delivery/postback.rb +72 -0
  27. data/lib/mail_room/delivery/que.rb +63 -0
  28. data/lib/mail_room/delivery/sidekiq.rb +88 -0
  29. data/lib/mail_room/logger/structured.rb +21 -0
  30. data/lib/mail_room/mailbox.rb +182 -0
  31. data/lib/mail_room/mailbox_watcher.rb +62 -0
  32. data/lib/mail_room/version.rb +4 -0
  33. data/logfile.log +1 -0
  34. data/mail_room.gemspec +34 -0
  35. data/spec/fixtures/test_config.yml +16 -0
  36. data/spec/lib/arbitration/redis_spec.rb +146 -0
  37. data/spec/lib/cli_spec.rb +61 -0
  38. data/spec/lib/configuration_spec.rb +29 -0
  39. data/spec/lib/connection_spec.rb +65 -0
  40. data/spec/lib/coordinator_spec.rb +61 -0
  41. data/spec/lib/crash_handler_spec.rb +41 -0
  42. data/spec/lib/delivery/letter_opener_spec.rb +29 -0
  43. data/spec/lib/delivery/logger_spec.rb +46 -0
  44. data/spec/lib/delivery/postback_spec.rb +107 -0
  45. data/spec/lib/delivery/que_spec.rb +45 -0
  46. data/spec/lib/delivery/sidekiq_spec.rb +76 -0
  47. data/spec/lib/logger/structured_spec.rb +55 -0
  48. data/spec/lib/mailbox_spec.rb +132 -0
  49. data/spec/lib/mailbox_watcher_spec.rb +64 -0
  50. data/spec/spec_helper.rb +32 -0
  51. metadata +277 -0
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ require 'rspec/core/rake_task'
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :default => :spec
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'mail_room'
4
+
5
+ MailRoom::CLI.new(ARGV).start
@@ -0,0 +1,16 @@
1
+ require 'net/imap'
2
+ require 'optparse'
3
+ require 'yaml'
4
+
5
+ module MailRoom
6
+ end
7
+
8
+ require "mail_room/version"
9
+ require "mail_room/configuration"
10
+ require "mail_room/mailbox"
11
+ require "mail_room/mailbox_watcher"
12
+ require "mail_room/connection"
13
+ require "mail_room/coordinator"
14
+ require "mail_room/cli"
15
+ require 'mail_room/logger/structured'
16
+ require 'mail_room/crash_handler'
@@ -0,0 +1,16 @@
1
+ module MailRoom
2
+ module Arbitration
3
+ def [](name)
4
+ require_relative("./arbitration/#{name}")
5
+
6
+ case name
7
+ when "redis"
8
+ Arbitration::Redis
9
+ else
10
+ Arbitration::Noop
11
+ end
12
+ end
13
+
14
+ module_function :[]
15
+ end
16
+ end
@@ -0,0 +1,18 @@
1
+ module MailRoom
2
+ module Arbitration
3
+ class Noop
4
+ Options = Class.new do
5
+ def initialize(*)
6
+ super()
7
+ end
8
+ end
9
+
10
+ def initialize(*)
11
+ end
12
+
13
+ def deliver?(*)
14
+ true
15
+ end
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,58 @@
1
+ require "redis"
2
+
3
+ module MailRoom
4
+ module Arbitration
5
+ class Redis
6
+ Options = Struct.new(:redis_url, :namespace, :sentinels) do
7
+ def initialize(mailbox)
8
+ redis_url = mailbox.arbitration_options[:redis_url] || "redis://localhost:6379"
9
+ namespace = mailbox.arbitration_options[:namespace]
10
+ sentinels = mailbox.arbitration_options[:sentinels]
11
+
12
+ super(redis_url, namespace, sentinels)
13
+ end
14
+ end
15
+
16
+ # Expire after 10 minutes so Redis doesn't get filled up with outdated data.
17
+ EXPIRATION = 600
18
+
19
+ attr_accessor :options
20
+
21
+ def initialize(options)
22
+ @options = options
23
+ end
24
+
25
+ def deliver?(uid, expiration = EXPIRATION)
26
+ key = "delivered:#{uid}"
27
+
28
+ # Set the key, but only if it doesn't already exist;
29
+ # the return value is true if successful, false if the key was already set,
30
+ # which is conveniently the correct return value for this method
31
+ # Any subsequent failure in the instance which gets the lock will be dealt
32
+ # with by the expiration, at which time another instance can pick up the
33
+ # message and try again.
34
+ client.set(key, 1, {:nx => true, :ex => expiration})
35
+ end
36
+
37
+ private
38
+
39
+ def client
40
+ @client ||= begin
41
+ sentinels = options.sentinels
42
+ redis_options = { url: options.redis_url }
43
+ redis_options[:sentinels] = sentinels if sentinels
44
+
45
+ redis = ::Redis.new(redis_options)
46
+
47
+ namespace = options.namespace
48
+ if namespace
49
+ require 'redis/namespace'
50
+ ::Redis::Namespace.new(namespace, redis: redis)
51
+ else
52
+ redis
53
+ end
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,62 @@
1
+ module MailRoom
2
+ # The CLI parses ARGV into configuration to start the coordinator with.
3
+ # @author Tony Pitale
4
+ class CLI
5
+ attr_accessor :configuration, :coordinator, :options
6
+
7
+ # Initialize a new CLI instance to handle option parsing from arguments
8
+ # into configuration to start the coordinator running on all mailboxes
9
+ #
10
+ # @param args [Array] `ARGV` passed from `bin/mail_room`
11
+ def initialize(args)
12
+ @options = {}
13
+
14
+ OptionParser.new do |parser|
15
+ parser.banner = [
16
+ "Usage: #{@name} [-c config_file]\n",
17
+ " #{@name} --help\n"
18
+ ].compact.join
19
+
20
+ parser.on('-c', '--config FILE') do |path|
21
+ options[:config_path] = path
22
+ end
23
+
24
+ parser.on('-q', '--quiet') do
25
+ options[:quiet] = true
26
+ end
27
+
28
+ parser.on('--log-exit-as') do |format|
29
+ options[:exit_error_format] = 'json' unless format.nil?
30
+ end
31
+
32
+ # parser.on("-l", "--log FILE") do |path|
33
+ # options[:log_path] = path
34
+ # end
35
+
36
+ parser.on_tail("-?", "--help", "Display this usage information.") do
37
+ puts "#{parser}\n"
38
+ exit
39
+ end
40
+ end.parse!(args)
41
+
42
+ self.configuration = Configuration.new(options)
43
+ self.coordinator = Coordinator.new(configuration.mailboxes)
44
+ end
45
+
46
+ # Start the coordinator running, sets up signal traps
47
+ def start
48
+ Signal.trap(:INT) do
49
+ coordinator.running = false
50
+ end
51
+
52
+ Signal.trap(:TERM) do
53
+ exit
54
+ end
55
+
56
+ coordinator.run
57
+ rescue Exception => e # not just Errors, but includes lower-level Exceptions
58
+ CrashHandler.new(error: e, format: @options[:exit_error_format]).handle
59
+ exit
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,36 @@
1
+ require "erb"
2
+
3
+ module MailRoom
4
+ # Wraps configuration for a set of individual mailboxes with global config
5
+ # @author Tony Pitale
6
+ class Configuration
7
+ attr_accessor :mailboxes, :log_path, :quiet
8
+
9
+ # Initialize a new configuration of mailboxes
10
+ def initialize(options={})
11
+ self.mailboxes = []
12
+ self.quiet = options.fetch(:quiet, false)
13
+
14
+ if options.has_key?(:config_path)
15
+ begin
16
+ erb = ERB.new(File.read(options[:config_path]))
17
+ erb.filename = options[:config_path]
18
+ config_file = YAML.load(erb.result)
19
+
20
+ set_mailboxes(config_file[:mailboxes])
21
+ rescue => e
22
+ raise e unless quiet
23
+ end
24
+ end
25
+ end
26
+
27
+ # Builds individual mailboxes from YAML configuration
28
+ #
29
+ # @param mailboxes_config
30
+ def set_mailboxes(mailboxes_config)
31
+ mailboxes_config.each do |attributes|
32
+ self.mailboxes << Mailbox.new(attributes)
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,195 @@
1
+ module MailRoom
2
+ class Connection
3
+ def initialize(mailbox)
4
+ @mailbox = mailbox
5
+
6
+ # log in and set the mailbox
7
+ reset
8
+ setup
9
+ end
10
+
11
+ def on_new_message(&block)
12
+ @new_message_handler = block
13
+ end
14
+
15
+ # is the connection logged in?
16
+ # @return [Boolean]
17
+ def logged_in?
18
+ @logged_in
19
+ end
20
+
21
+ # is the connection blocked idling?
22
+ # @return [Boolean]
23
+ def idling?
24
+ @idling
25
+ end
26
+
27
+ # is the imap connection closed?
28
+ # @return [Boolean]
29
+ def disconnected?
30
+ @imap.disconnected?
31
+ end
32
+
33
+ # is the connection ready to idle?
34
+ # @return [Boolean]
35
+ def ready_to_idle?
36
+ logged_in? && !idling?
37
+ end
38
+
39
+ def quit
40
+ stop_idling
41
+ reset
42
+ end
43
+
44
+ def wait
45
+ begin
46
+ # in case we missed any between idles
47
+ process_mailbox
48
+
49
+ idle
50
+
51
+ process_mailbox
52
+ rescue Net::IMAP::Error, IOError
53
+ @mailbox.logger.warn({ context: @mailbox.context, action: "Disconnected. Resetting..." })
54
+ reset
55
+ setup
56
+ end
57
+ end
58
+
59
+ private
60
+
61
+ def reset
62
+ @imap = nil
63
+ @logged_in = false
64
+ @idling = false
65
+ end
66
+
67
+ def setup
68
+ @mailbox.logger.info({ context: @mailbox.context, action: "Starting TLS session" })
69
+ start_tls
70
+
71
+ @mailbox.logger.info({ context: @mailbox.context, action: "Logging into mailbox" })
72
+ log_in
73
+
74
+ @mailbox.logger.info({ context: @mailbox.context, action: "Setting mailbox" })
75
+ set_mailbox
76
+ end
77
+
78
+ # build a net/imap connection to google imap
79
+ def imap
80
+ @imap ||= Net::IMAP.new(@mailbox.host, :port => @mailbox.port, :ssl => @mailbox.ssl_options)
81
+ end
82
+
83
+ # start a TLS session
84
+ def start_tls
85
+ imap.starttls if @mailbox.start_tls
86
+ end
87
+
88
+ # send the imap login command to google
89
+ def log_in
90
+ imap.login(@mailbox.email, @mailbox.password)
91
+ @logged_in = true
92
+ end
93
+
94
+ # select the mailbox name we want to use
95
+ def set_mailbox
96
+ imap.select(@mailbox.name) if logged_in?
97
+ end
98
+
99
+ # is the response for a new message?
100
+ # @param response [Net::IMAP::TaggedResponse] the imap response from idle
101
+ # @return [Boolean]
102
+ def message_exists?(response)
103
+ response.respond_to?(:name) && response.name == 'EXISTS'
104
+ end
105
+
106
+ # @private
107
+ def idle_handler
108
+ lambda {|response| imap.idle_done if message_exists?(response)}
109
+ end
110
+
111
+ # maintain an imap idle connection
112
+ def idle
113
+ return unless ready_to_idle?
114
+
115
+ @mailbox.logger.info({ context: @mailbox.context, action: "Idling" })
116
+ @idling = true
117
+
118
+ imap.idle(@mailbox.idle_timeout, &idle_handler)
119
+ ensure
120
+ @idling = false
121
+ end
122
+
123
+ # trigger the idle to finish and wait for the thread to finish
124
+ def stop_idling
125
+ return unless idling?
126
+
127
+ imap.idle_done
128
+
129
+ # idling_thread.join
130
+ # self.idling_thread = nil
131
+ end
132
+
133
+ def process_mailbox
134
+ return unless @new_message_handler
135
+ @mailbox.logger.info({ context: @mailbox.context, action: "Processing started" })
136
+
137
+ msgs = new_messages
138
+
139
+ any_deletions = msgs.
140
+ # deliver each new message, collect success
141
+ map(&@new_message_handler).
142
+ # include messages with success
143
+ zip(msgs).
144
+ # filter failed deliveries, collect message
145
+ select(&:first).map(&:last).
146
+ # scrub delivered messages
147
+ map { |message| scrub(message) }.
148
+ any?
149
+
150
+ imap.expunge if @mailbox.expunge_deleted && any_deletions
151
+ end
152
+
153
+ def scrub(message)
154
+ if @mailbox.delete_after_delivery
155
+ imap.store(message.seqno, "+FLAGS", [Net::IMAP::DELETED])
156
+ true
157
+ end
158
+ end
159
+
160
+ # @private
161
+ # fetch all messages for the new message ids
162
+ def new_messages
163
+ # Both of these calls may results in
164
+ # imap raising an EOFError, we handle
165
+ # this exception in the watcher
166
+ messages_for_ids(new_message_ids)
167
+ end
168
+
169
+ # TODO: label messages?
170
+ # @imap.store(id, "+X-GM-LABELS", [label])
171
+
172
+ # @private
173
+ # search for all new (unseen) message ids
174
+ # @return [Array<Integer>] message ids
175
+ def new_message_ids
176
+ # uid_search still leaves messages UNSEEN
177
+ all_unread = @imap.uid_search(@mailbox.search_command)
178
+
179
+ to_deliver = all_unread.select { |uid| @mailbox.deliver?(uid) }
180
+ @mailbox.logger.info({ context: @mailbox.context, action: "Getting new messages", unread: {count: all_unread.count, ids: all_unread}, to_be_delivered: { count: to_deliver.count, ids: all_unread } })
181
+ to_deliver
182
+ end
183
+
184
+ # @private
185
+ # fetch the email for all given ids in RFC822 format
186
+ # @param ids [Array<Integer>] list of message ids
187
+ # @return [Array<Net::IMAP::FetchData>] the net/imap messages for the given ids
188
+ def messages_for_ids(uids)
189
+ return [] if uids.empty?
190
+
191
+ # uid_fetch marks as SEEN, will not be re-fetched for UNSEEN
192
+ imap.uid_fetch(uids, "RFC822")
193
+ end
194
+ end
195
+ end
@@ -0,0 +1,41 @@
1
+ module MailRoom
2
+ # Coordinate the mailbox watchers
3
+ # @author Tony Pitale
4
+ class Coordinator
5
+ attr_accessor :watchers, :running
6
+
7
+ # build watchers for a set of mailboxes
8
+ # @params mailboxes [Array<MailRoom::Mailbox>] mailboxes to be watched
9
+ def initialize(mailboxes)
10
+ self.watchers = []
11
+
12
+ mailboxes.each {|box| self.watchers << MailboxWatcher.new(box)}
13
+ end
14
+
15
+ alias :running? :running
16
+
17
+ # start each of the watchers to running
18
+ def run
19
+ watchers.each(&:run)
20
+
21
+ self.running = true
22
+
23
+ sleep_while_running
24
+ ensure
25
+ quit
26
+ end
27
+
28
+ # quit each of the watchers when we're done running
29
+ def quit
30
+ watchers.each(&:quit)
31
+ end
32
+
33
+ private
34
+ # @private
35
+ def sleep_while_running
36
+ # do we need to sweep for dead watchers?
37
+ # or do we let the mailbox rebuild connections
38
+ while(running?) do; sleep 1; end
39
+ end
40
+ end
41
+ end