gitlab-mail_room 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +18 -0
  3. data/.gitlab-ci.yml +27 -0
  4. data/.ruby-version +1 -0
  5. data/.travis.yml +10 -0
  6. data/CHANGELOG.md +125 -0
  7. data/CODE_OF_CONDUCT.md +24 -0
  8. data/Gemfile +4 -0
  9. data/LICENSE.txt +22 -0
  10. data/README.md +361 -0
  11. data/Rakefile +6 -0
  12. data/bin/mail_room +5 -0
  13. data/lib/mail_room.rb +16 -0
  14. data/lib/mail_room/arbitration.rb +16 -0
  15. data/lib/mail_room/arbitration/noop.rb +18 -0
  16. data/lib/mail_room/arbitration/redis.rb +58 -0
  17. data/lib/mail_room/cli.rb +62 -0
  18. data/lib/mail_room/configuration.rb +36 -0
  19. data/lib/mail_room/connection.rb +195 -0
  20. data/lib/mail_room/coordinator.rb +41 -0
  21. data/lib/mail_room/crash_handler.rb +29 -0
  22. data/lib/mail_room/delivery.rb +24 -0
  23. data/lib/mail_room/delivery/letter_opener.rb +34 -0
  24. data/lib/mail_room/delivery/logger.rb +37 -0
  25. data/lib/mail_room/delivery/noop.rb +22 -0
  26. data/lib/mail_room/delivery/postback.rb +72 -0
  27. data/lib/mail_room/delivery/que.rb +63 -0
  28. data/lib/mail_room/delivery/sidekiq.rb +88 -0
  29. data/lib/mail_room/logger/structured.rb +21 -0
  30. data/lib/mail_room/mailbox.rb +182 -0
  31. data/lib/mail_room/mailbox_watcher.rb +62 -0
  32. data/lib/mail_room/version.rb +4 -0
  33. data/logfile.log +1 -0
  34. data/mail_room.gemspec +34 -0
  35. data/spec/fixtures/test_config.yml +16 -0
  36. data/spec/lib/arbitration/redis_spec.rb +146 -0
  37. data/spec/lib/cli_spec.rb +61 -0
  38. data/spec/lib/configuration_spec.rb +29 -0
  39. data/spec/lib/connection_spec.rb +65 -0
  40. data/spec/lib/coordinator_spec.rb +61 -0
  41. data/spec/lib/crash_handler_spec.rb +41 -0
  42. data/spec/lib/delivery/letter_opener_spec.rb +29 -0
  43. data/spec/lib/delivery/logger_spec.rb +46 -0
  44. data/spec/lib/delivery/postback_spec.rb +107 -0
  45. data/spec/lib/delivery/que_spec.rb +45 -0
  46. data/spec/lib/delivery/sidekiq_spec.rb +76 -0
  47. data/spec/lib/logger/structured_spec.rb +55 -0
  48. data/spec/lib/mailbox_spec.rb +132 -0
  49. data/spec/lib/mailbox_watcher_spec.rb +64 -0
  50. data/spec/spec_helper.rb +32 -0
  51. metadata +277 -0
@@ -0,0 +1,29 @@
1
+
2
+ module MailRoom
3
+ class CrashHandler
4
+
5
+ attr_reader :error, :format
6
+
7
+ SUPPORTED_FORMATS = %w[json none]
8
+
9
+ def initialize(error:, format:)
10
+ @error = error
11
+ @format = format
12
+ end
13
+
14
+ def handle
15
+ if format == 'json'
16
+ puts json
17
+ return
18
+ end
19
+
20
+ raise error
21
+ end
22
+
23
+ private
24
+
25
+ def json
26
+ { time: Time.now, severity: :fatal, message: error.message, backtrace: error.backtrace }.to_json
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,24 @@
1
+ module MailRoom
2
+ module Delivery
3
+ def [](name)
4
+ require_relative("./delivery/#{name}")
5
+
6
+ case name
7
+ when "postback"
8
+ Delivery::Postback
9
+ when "logger"
10
+ Delivery::Logger
11
+ when "letter_opener"
12
+ Delivery::LetterOpener
13
+ when "sidekiq"
14
+ Delivery::Sidekiq
15
+ when "que"
16
+ Delivery::Que
17
+ else
18
+ Delivery::Noop
19
+ end
20
+ end
21
+
22
+ module_function :[]
23
+ end
24
+ end
@@ -0,0 +1,34 @@
1
+ require 'erb'
2
+ require 'mail'
3
+ require 'letter_opener'
4
+
5
+ module MailRoom
6
+ module Delivery
7
+ # LetterOpener Delivery method
8
+ # @author Tony Pitale
9
+ class LetterOpener
10
+ Options = Struct.new(:location) do
11
+ def initialize(mailbox)
12
+ location = mailbox.location || mailbox.delivery_options[:location]
13
+
14
+ super(location)
15
+ end
16
+ end
17
+
18
+ # Build a new delivery, hold the delivery options
19
+ # @param [MailRoom::Delivery::LetterOpener::Options]
20
+ def initialize(delivery_options)
21
+ @delivery_options = delivery_options
22
+ end
23
+
24
+ # Trigger `LetterOpener` to deliver our message
25
+ # @param message [String] the email message as a string, RFC822 format
26
+ def deliver(message)
27
+ method = ::LetterOpener::DeliveryMethod.new(:location => @delivery_options.location)
28
+ method.deliver!(Mail.read_from_string(message))
29
+
30
+ true
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,37 @@
1
+ require 'logger'
2
+
3
+ module MailRoom
4
+ module Delivery
5
+ # File/STDOUT Logger Delivery method
6
+ # @author Tony Pitale
7
+ class Logger
8
+ Options = Struct.new(:log_path) do
9
+ def initialize(mailbox)
10
+ log_path = mailbox.log_path || mailbox.delivery_options[:log_path]
11
+
12
+ super(log_path)
13
+ end
14
+ end
15
+
16
+ # Build a new delivery, hold the delivery options
17
+ # open a file or stdout for IO depending on the options
18
+ # @param [MailRoom::Delivery::Logger::Options]
19
+ def initialize(delivery_options)
20
+ io = File.open(delivery_options.log_path, 'a') if delivery_options.log_path
21
+ io ||= STDOUT
22
+
23
+ io.sync = true
24
+
25
+ @logger = ::Logger.new(io)
26
+ end
27
+
28
+ # Write the message to our logger
29
+ # @param message [String] the email message as a string, RFC822 format
30
+ def deliver(message)
31
+ @logger.info message
32
+
33
+ true
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,22 @@
1
+ module MailRoom
2
+ module Delivery
3
+ # Noop Delivery method
4
+ # @author Tony Pitale
5
+ class Noop
6
+ Options = Class.new do
7
+ def initialize(*)
8
+ super()
9
+ end
10
+ end
11
+
12
+ # build a new delivery, do nothing
13
+ def initialize(*)
14
+ end
15
+
16
+ # accept the delivery, do nothing
17
+ def deliver(*)
18
+ true
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,72 @@
1
+ require 'faraday'
2
+
3
+ module MailRoom
4
+ module Delivery
5
+ # Postback Delivery method
6
+ # @author Tony Pitale
7
+ class Postback
8
+ Options = Struct.new(:url, :token, :username, :password, :logger, :content_type) do
9
+ def initialize(mailbox)
10
+ url =
11
+ mailbox.delivery_url ||
12
+ mailbox.delivery_options[:delivery_url] ||
13
+ mailbox.delivery_options[:url]
14
+
15
+ token =
16
+ mailbox.delivery_token ||
17
+ mailbox.delivery_options[:delivery_token] ||
18
+ mailbox.delivery_options[:token]
19
+
20
+ username = mailbox.delivery_options[:username]
21
+ password = mailbox.delivery_options[:password]
22
+
23
+ logger = mailbox.logger
24
+
25
+ content_type = mailbox.delivery_options[:content_type]
26
+
27
+ super(url, token, username, password, logger, content_type)
28
+ end
29
+
30
+ def token_auth?
31
+ !self[:token].nil?
32
+ end
33
+
34
+ def basic_auth?
35
+ !self[:username].nil? && !self[:password].nil?
36
+ end
37
+ end
38
+
39
+ # Build a new delivery, hold the delivery options
40
+ # @param [MailRoom::Delivery::Postback::Options]
41
+ def initialize(delivery_options)
42
+ puts delivery_options
43
+ @delivery_options = delivery_options
44
+ end
45
+
46
+ # deliver the message using Faraday to the configured delivery_options url
47
+ # @param message [String] the email message as a string, RFC822 format
48
+ def deliver(message)
49
+ connection = Faraday.new
50
+
51
+ if @delivery_options.token_auth?
52
+ connection.token_auth @delivery_options.token
53
+ elsif @delivery_options.basic_auth?
54
+ connection.basic_auth(
55
+ @delivery_options.username,
56
+ @delivery_options.password
57
+ )
58
+ end
59
+
60
+ connection.post do |request|
61
+ request.url @delivery_options.url
62
+ request.body = message
63
+ # request.options[:timeout] = 3
64
+ request.headers['Content-Type'] = @delivery_options.content_type unless @delivery_options.content_type.nil?
65
+ end
66
+
67
+ @delivery_options.logger.info({ delivery_method: 'Postback', action: 'message pushed', url: @delivery_options.url })
68
+ true
69
+ end
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,63 @@
1
+ require 'pg'
2
+ require 'json'
3
+
4
+ module MailRoom
5
+ module Delivery
6
+ # Que Delivery method
7
+ # @author Tony Pitale
8
+ class Que
9
+ Options = Struct.new(:host, :port, :database, :username, :password, :queue, :priority, :job_class, :logger) do
10
+ def initialize(mailbox)
11
+ host = mailbox.delivery_options[:host] || "localhost"
12
+ port = mailbox.delivery_options[:port] || 5432
13
+ database = mailbox.delivery_options[:database]
14
+ username = mailbox.delivery_options[:username]
15
+ password = mailbox.delivery_options[:password]
16
+
17
+ queue = mailbox.delivery_options[:queue] || ''
18
+ priority = mailbox.delivery_options[:priority] || 100 # lowest priority for Que
19
+ job_class = mailbox.delivery_options[:job_class]
20
+ logger = mailbox.logger
21
+
22
+ super(host, port, database, username, password, queue, priority, job_class, logger)
23
+ end
24
+ end
25
+
26
+ attr_reader :options
27
+
28
+ # Build a new delivery, hold the mailbox configuration
29
+ # @param [MailRoom::Delivery::Que::Options]
30
+ def initialize(options)
31
+ @options = options
32
+ end
33
+
34
+ # deliver the message by pushing it onto the configured Sidekiq queue
35
+ # @param message [String] the email message as a string, RFC822 format
36
+ def deliver(message)
37
+ queue_job(message)
38
+ @options.logger.info({ delivery_method: 'Que', action: 'message pushed' })
39
+ end
40
+
41
+ private
42
+ def connection
43
+ PG.connect(connection_options)
44
+ end
45
+
46
+ def connection_options
47
+ {
48
+ host: options.host,
49
+ port: options.port,
50
+ dbname: options.database,
51
+ user: options.username,
52
+ password: options.password
53
+ }
54
+ end
55
+
56
+ def queue_job(*args)
57
+ sql = "INSERT INTO que_jobs (priority, job_class, queue, args) VALUES ($1, $2, $3, $4)"
58
+
59
+ connection.exec(sql, [options.priority, options.job_class, options.queue, JSON.dump(args)])
60
+ end
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,88 @@
1
+ require "redis"
2
+ require "securerandom"
3
+ require "json"
4
+ require "charlock_holmes"
5
+
6
+ module MailRoom
7
+ module Delivery
8
+ # Sidekiq Delivery method
9
+ # @author Douwe Maan
10
+ class Sidekiq
11
+ Options = Struct.new(:redis_url, :namespace, :sentinels, :queue, :worker, :logger) do
12
+ def initialize(mailbox)
13
+ redis_url = mailbox.delivery_options[:redis_url] || "redis://localhost:6379"
14
+ namespace = mailbox.delivery_options[:namespace]
15
+ sentinels = mailbox.delivery_options[:sentinels]
16
+ queue = mailbox.delivery_options[:queue] || "default"
17
+ worker = mailbox.delivery_options[:worker]
18
+ logger = mailbox.logger
19
+
20
+ super(redis_url, namespace, sentinels, queue, worker, logger)
21
+ end
22
+ end
23
+
24
+ attr_accessor :options
25
+
26
+ # Build a new delivery, hold the mailbox configuration
27
+ # @param [MailRoom::Delivery::Sidekiq::Options]
28
+ def initialize(options)
29
+ @options = options
30
+ end
31
+
32
+ # deliver the message by pushing it onto the configured Sidekiq queue
33
+ # @param message [String] the email message as a string, RFC822 format
34
+ def deliver(message)
35
+ item = item_for(message)
36
+
37
+ client.lpush("queue:#{options.queue}", JSON.generate(item))
38
+
39
+ @options.logger.info({ delivery_method: 'Sidekiq', action: 'message pushed' })
40
+ true
41
+ end
42
+
43
+ private
44
+
45
+ def client
46
+ @client ||= begin
47
+ sentinels = options.sentinels
48
+ redis_options = { url: options.redis_url }
49
+ redis_options[:sentinels] = sentinels if sentinels
50
+
51
+ redis = ::Redis.new(redis_options)
52
+
53
+ namespace = options.namespace
54
+ if namespace
55
+ require 'redis/namespace'
56
+ Redis::Namespace.new(namespace, redis: redis)
57
+ else
58
+ redis
59
+ end
60
+ end
61
+ end
62
+
63
+ def item_for(message)
64
+ {
65
+ 'class' => options.worker,
66
+ 'args' => [utf8_encode_message(message)],
67
+ 'queue' => options.queue,
68
+ 'jid' => SecureRandom.hex(12),
69
+ 'retry' => false,
70
+ 'enqueued_at' => Time.now.to_f
71
+ }
72
+ end
73
+
74
+ def utf8_encode_message(message)
75
+ message = message.dup
76
+
77
+ message.force_encoding("UTF-8")
78
+ return message if message.valid_encoding?
79
+
80
+ detection = CharlockHolmes::EncodingDetector.detect(message)
81
+ return message unless detection && detection[:encoding]
82
+
83
+ # Convert non-UTF-8 body UTF-8 so it can be dumped as JSON.
84
+ CharlockHolmes::Converter.convert(message, detection[:encoding], 'UTF-8')
85
+ end
86
+ end
87
+ end
88
+ end
@@ -0,0 +1,21 @@
1
+ require 'logger'
2
+ require 'json'
3
+
4
+ module MailRoom
5
+ module Logger
6
+ class Structured < ::Logger
7
+
8
+ def format_message(severity, timestamp, progname, message)
9
+ raise ArgumentError.new("Message must be a Hash") unless message.is_a? Hash
10
+
11
+ data = {}
12
+ data[:severity] = severity
13
+ data[:time] = timestamp || Time.now.to_s
14
+ # only accept a Hash
15
+ data.merge!(message)
16
+
17
+ data.to_json + "\n"
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,182 @@
1
+ require "mail_room/delivery"
2
+ require "mail_room/arbitration"
3
+
4
+ module MailRoom
5
+ # Mailbox Configuration fields
6
+ MAILBOX_FIELDS = [
7
+ :email,
8
+ :password,
9
+ :host,
10
+ :port,
11
+ :ssl,
12
+ :start_tls,
13
+ :idle_timeout,
14
+ :search_command,
15
+ :name,
16
+ :delete_after_delivery,
17
+ :expunge_deleted,
18
+ :delivery_klass,
19
+ :delivery_method, # :noop, :logger, :postback, :letter_opener
20
+ :log_path, # for logger
21
+ :delivery_url, # for postback
22
+ :delivery_token, # for postback
23
+ :content_type, # for postback
24
+ :location, # for letter_opener
25
+ :delivery_options,
26
+ :arbitration_method,
27
+ :arbitration_options,
28
+ :logger
29
+ ]
30
+
31
+ ConfigurationError = Class.new(RuntimeError)
32
+ IdleTimeoutTooLarge = Class.new(RuntimeError)
33
+
34
+ # Holds configuration for each of the email accounts we wish to monitor
35
+ # and deliver email to when new emails arrive over imap
36
+ Mailbox = Struct.new(*MAILBOX_FIELDS) do
37
+ # Keep it to 29 minutes or less
38
+ # The IMAP serve will close the connection after 30 minutes of inactivity
39
+ # (which sending IDLE and then nothing technically is), so we re-idle every
40
+ # 29 minutes, as suggested by the spec: https://tools.ietf.org/html/rfc2177
41
+ IMAP_IDLE_TIMEOUT = 29 * 60 # 29 minutes in in seconds
42
+
43
+ REQUIRED_CONFIGURATION = [:name, :email, :password, :host, :port]
44
+
45
+ # Default attributes for the mailbox configuration
46
+ DEFAULTS = {
47
+ :search_command => 'UNSEEN',
48
+ :delivery_method => 'postback',
49
+ :host => 'imap.gmail.com',
50
+ :port => 993,
51
+ :ssl => true,
52
+ :start_tls => false,
53
+ :idle_timeout => IMAP_IDLE_TIMEOUT,
54
+ :delete_after_delivery => false,
55
+ :expunge_deleted => false,
56
+ :delivery_options => {},
57
+ :arbitration_method => 'noop',
58
+ :arbitration_options => {},
59
+ :logger => {}
60
+ }
61
+
62
+ # Store the configuration and require the appropriate delivery method
63
+ # @param attributes [Hash] configuration options
64
+ def initialize(attributes={})
65
+ super(*DEFAULTS.merge(attributes).values_at(*members))
66
+
67
+ validate!
68
+ end
69
+
70
+ def logger
71
+ @logger ||=
72
+ case self[:logger]
73
+ when Logger
74
+ self[:logger]
75
+ else
76
+ self[:logger] ||= {}
77
+ MailRoom::Logger::Structured.new(normalize_log_path(self[:logger][:log_path]))
78
+ end
79
+ end
80
+
81
+ def delivery_klass
82
+ self[:delivery_klass] ||= Delivery[delivery_method]
83
+ end
84
+
85
+ def arbitration_klass
86
+ Arbitration[arbitration_method]
87
+ end
88
+
89
+ def delivery
90
+ @delivery ||= delivery_klass.new(parsed_delivery_options)
91
+ end
92
+
93
+ def arbitrator
94
+ @arbitrator ||= arbitration_klass.new(parsed_arbitration_options)
95
+ end
96
+
97
+ def deliver?(uid)
98
+ logger.info({context: context, uid: uid, action: "asking arbiter to deliver", arbitrator: arbitrator.class.name})
99
+
100
+ arbitrator.deliver?(uid)
101
+ end
102
+
103
+ # deliver the imap email message
104
+ # @param message [Net::IMAP::FetchData]
105
+ def deliver(message)
106
+ body = message.attr['RFC822']
107
+ return true unless body
108
+
109
+ logger.info({context: context, uid: message.attr['UID'], action: "sending to deliverer", deliverer: delivery.class.name, byte_size: message.attr['RFC822.SIZE']})
110
+ delivery.deliver(body)
111
+ end
112
+
113
+ # true, false, or ssl options hash
114
+ def ssl_options
115
+ replace_verify_mode(ssl)
116
+ end
117
+
118
+ def context
119
+ { email: self.email, name: self.name }
120
+ end
121
+
122
+ def validate!
123
+ if self[:idle_timeout] > IMAP_IDLE_TIMEOUT
124
+ raise IdleTimeoutTooLarge,
125
+ "Please use an idle timeout smaller than #{29*60} to prevent " \
126
+ "IMAP server disconnects"
127
+ end
128
+
129
+ REQUIRED_CONFIGURATION.each do |k|
130
+ if self[k].nil?
131
+ raise ConfigurationError,
132
+ "Field :#{k} is required in Mailbox: #{inspect}"
133
+ end
134
+ end
135
+ end
136
+
137
+ private
138
+
139
+ def parsed_arbitration_options
140
+ arbitration_klass::Options.new(self)
141
+ end
142
+
143
+ def parsed_delivery_options
144
+ delivery_klass::Options.new(self)
145
+ end
146
+
147
+ def replace_verify_mode(options)
148
+ return options unless options.is_a?(Hash)
149
+ return options unless options.has_key?(:verify_mode)
150
+
151
+ options[:verify_mode] = lookup_verify_mode(options[:verify_mode])
152
+
153
+ options
154
+ end
155
+
156
+ def lookup_verify_mode(verify_mode)
157
+ case verify_mode.to_sym
158
+ when :none
159
+ OpenSSL::SSL::VERIFY_NONE
160
+ when :peer
161
+ OpenSSL::SSL::VERIFY_PEER
162
+ when :client_once
163
+ OpenSSL::SSL::VERIFY_CLIENT_ONCE
164
+ when :fail_if_no_peer_cert
165
+ OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
166
+ end
167
+ end
168
+
169
+ def normalize_log_path(log_path)
170
+ case log_path
171
+ when nil, ""
172
+ nil
173
+ when :stdout, "STDOUT"
174
+ STDOUT
175
+ when :stderr, "STDERR"
176
+ STDERR
177
+ else
178
+ log_path
179
+ end
180
+ end
181
+ end
182
+ end