backup_zh 4.0.3.1 → 4.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +15 -7
- data/lib/backup.rb +6 -0
- data/lib/backup/cli.rb +10 -0
- data/lib/backup/config/dsl.rb +3 -3
- data/lib/backup/database/mysql.rb +17 -6
- data/lib/backup/database/postgresql.rb +2 -2
- data/lib/backup/database/sqlite.rb +57 -0
- data/lib/backup/encryptor/open_ssl.rb +7 -2
- data/lib/backup/model.rb +26 -1
- data/lib/backup/notifier/base.rb +30 -0
- data/lib/backup/notifier/campfire.rb +1 -7
- data/lib/backup/notifier/command.rb +99 -0
- data/lib/backup/notifier/datadog.rb +107 -0
- data/lib/backup/notifier/flowdock.rb +6 -5
- data/lib/backup/notifier/hipchat.rb +27 -8
- data/lib/backup/notifier/http_post.rb +2 -7
- data/lib/backup/notifier/mail.rb +19 -10
- data/lib/backup/notifier/nagios.rb +3 -8
- data/lib/backup/notifier/pagerduty.rb +81 -0
- data/lib/backup/notifier/prowl.rb +8 -9
- data/lib/backup/notifier/pushover.rb +1 -7
- data/lib/backup/notifier/ses.rb +88 -0
- data/lib/backup/notifier/slack.rb +7 -17
- data/lib/backup/notifier/twitter.rb +1 -7
- data/lib/backup/notifier/zabbix.rb +1 -6
- data/lib/backup/package.rb +4 -0
- data/lib/backup/packager.rb +8 -2
- data/lib/backup/storage/base.rb +15 -3
- data/lib/backup/storage/cycler.rb +24 -14
- data/lib/backup/storage/ftp.rb +15 -1
- data/lib/backup/storage/s3.rb +2 -1
- data/lib/backup/syncer/base.rb +2 -2
- data/lib/backup/version.rb +1 -1
- data/templates/cli/config +2 -2
- data/templates/cli/databases/sqlite +11 -0
- data/templates/cli/model +1 -1
- data/templates/cli/notifiers/command +32 -0
- data/templates/cli/notifiers/datadog +57 -0
- data/templates/cli/notifiers/hipchat +1 -0
- data/templates/cli/notifiers/mail +3 -0
- data/templates/cli/notifiers/pagerduty +12 -0
- data/templates/cli/notifiers/ses +15 -0
- data/templates/cli/notifiers/slack +3 -4
- data/templates/cli/storages/dropbox +1 -0
- data/templates/cli/storages/ftp +1 -0
- data/templates/cli/storages/local +1 -0
- data/templates/cli/storages/ninefold +1 -0
- data/templates/cli/storages/s3 +2 -0
- data/templates/cli/storages/scp +1 -0
- data/templates/cli/storages/sftp +1 -0
- data/templates/general/links +3 -3
- metadata +53 -136
@@ -0,0 +1,107 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'dogapi'
|
3
|
+
|
4
|
+
module Backup
|
5
|
+
module Notifier
|
6
|
+
class DataDog < Base
|
7
|
+
|
8
|
+
##
|
9
|
+
# The DataDog API key
|
10
|
+
attr_accessor :api_key
|
11
|
+
|
12
|
+
##
|
13
|
+
# The title of the event
|
14
|
+
attr_accessor :title
|
15
|
+
|
16
|
+
attr_deprecate :text,
|
17
|
+
:version => '4.2',
|
18
|
+
:message => 'Please use the `message` attribute. For more information '\
|
19
|
+
'see https://github.com/backup/backup/pull/698'
|
20
|
+
|
21
|
+
##
|
22
|
+
# The timestamp for the event
|
23
|
+
attr_accessor :date_happened
|
24
|
+
|
25
|
+
##
|
26
|
+
# The priority of the event (low/normal)
|
27
|
+
attr_accessor :priority
|
28
|
+
|
29
|
+
##
|
30
|
+
# The host that generated the event
|
31
|
+
attr_accessor :host
|
32
|
+
|
33
|
+
##
|
34
|
+
# The tags for this host (should be an array)
|
35
|
+
attr_accessor :tags
|
36
|
+
|
37
|
+
##
|
38
|
+
# The alert_type of the event (error/warning/info/success)
|
39
|
+
attr_accessor :alert_type
|
40
|
+
|
41
|
+
##
|
42
|
+
# The aggregation_key for the event
|
43
|
+
attr_accessor :aggregation_key
|
44
|
+
|
45
|
+
##
|
46
|
+
# The source_type for the event (nagios, hudson, jenkins, user, my apps, feed, chef, puppet, git, bitbucket, fabric, capistrano)
|
47
|
+
attr_accessor :source_type_name
|
48
|
+
|
49
|
+
def initialize(model, &block)
|
50
|
+
super
|
51
|
+
instance_eval(&block) if block_given?
|
52
|
+
@title ||= "Backup #{ model.label }"
|
53
|
+
end
|
54
|
+
|
55
|
+
private
|
56
|
+
|
57
|
+
##
|
58
|
+
# Notify the user of the backup operation results.
|
59
|
+
#
|
60
|
+
# `status` indicates one of the following:
|
61
|
+
#
|
62
|
+
# `:success`
|
63
|
+
# : The backup completed successfully.
|
64
|
+
# : Notification will be sent if `on_success` is `true`.
|
65
|
+
#
|
66
|
+
# `:warning`
|
67
|
+
# : The backup completed successfully, but warnings were logged.
|
68
|
+
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
69
|
+
#
|
70
|
+
# `:failure`
|
71
|
+
# : The backup operation failed.
|
72
|
+
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
73
|
+
#
|
74
|
+
def notify!(status)
|
75
|
+
msg = message.call(model, :status => status_data_for(status))
|
76
|
+
|
77
|
+
hash = { alert_type: default_alert_type(status) }
|
78
|
+
hash.store(:msg_title, @title)
|
79
|
+
hash.store(:date_happened, @date_happened) if @date_happened
|
80
|
+
hash.store(:priority, @priority) if @priority
|
81
|
+
hash.store(:host, @host) if @host
|
82
|
+
hash.store(:tags, @tags) if @tags
|
83
|
+
hash.store(:aggregation_key, @aggregation_key) if @aggregation_key
|
84
|
+
hash.store(:source_type_name, @source_type_name) if @source_type_name
|
85
|
+
hash.store(:alert_type, @alert_type) if @alert_type
|
86
|
+
send_event(msg, hash)
|
87
|
+
end
|
88
|
+
|
89
|
+
# Dogapi::Client will raise an error if unsuccessful.
|
90
|
+
def send_event(msg, hash)
|
91
|
+
client = Dogapi::Client.new(@api_key)
|
92
|
+
event = Dogapi::Event.new(msg, hash)
|
93
|
+
client.emit_event(event)
|
94
|
+
end
|
95
|
+
|
96
|
+
# set alert type
|
97
|
+
def default_alert_type(status)
|
98
|
+
case status
|
99
|
+
when :success then 'success'
|
100
|
+
when :warning then 'warning'
|
101
|
+
when :failure then 'error'
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
@@ -61,15 +61,16 @@ module Backup
|
|
61
61
|
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
62
62
|
#
|
63
63
|
def notify!(status)
|
64
|
-
@tags
|
65
|
-
message
|
66
|
-
send_message(message)
|
64
|
+
@tags += default_tags(status)
|
65
|
+
send_message(message.call(model, status: status_data_for(status)))
|
67
66
|
end
|
68
67
|
|
69
68
|
# Flowdock::Client will raise an error if unsuccessful.
|
70
69
|
def send_message(msg)
|
71
|
-
client = Flowdock::Flow.new(
|
72
|
-
|
70
|
+
client = Flowdock::Flow.new(
|
71
|
+
:api_token => token, :source => source,
|
72
|
+
:from => {:name => from_name, :address => from_email }
|
73
|
+
)
|
73
74
|
|
74
75
|
client.push_to_team_inbox(:subject => subject,
|
75
76
|
:content => msg,
|
@@ -9,6 +9,11 @@ module Backup
|
|
9
9
|
# The Hipchat API token
|
10
10
|
attr_accessor :token
|
11
11
|
|
12
|
+
##
|
13
|
+
# The Hipchat API version
|
14
|
+
# Either 'v1' or 'v2' (default is 'v1')
|
15
|
+
attr_accessor :api_version
|
16
|
+
|
12
17
|
##
|
13
18
|
# Who the notification should appear from
|
14
19
|
attr_accessor :from
|
@@ -45,6 +50,7 @@ module Backup
|
|
45
50
|
@success_color ||= 'yellow'
|
46
51
|
@warning_color ||= 'yellow'
|
47
52
|
@failure_color ||= 'yellow'
|
53
|
+
@api_version ||= 'v1'
|
48
54
|
end
|
49
55
|
|
50
56
|
private
|
@@ -67,18 +73,18 @@ module Backup
|
|
67
73
|
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
68
74
|
#
|
69
75
|
def notify!(status)
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
76
|
+
status_data = status_data_for(status)
|
77
|
+
msg = message.call(model, :status => status_data)
|
78
|
+
send_message(msg, status_data[:color])
|
79
|
+
end
|
80
|
+
|
81
|
+
def client_options
|
82
|
+
{ api_version: @api_version }
|
77
83
|
end
|
78
84
|
|
79
85
|
# Hipchat::Client will raise an error if unsuccessful.
|
80
86
|
def send_message(msg, color)
|
81
|
-
client = HipChat::Client.new(token)
|
87
|
+
client = HipChat::Client.new(token, client_options)
|
82
88
|
rooms_to_notify.each do |room|
|
83
89
|
client[room].send(from, msg, :color => color, :notify => notify_users)
|
84
90
|
end
|
@@ -88,6 +94,19 @@ module Backup
|
|
88
94
|
Array(rooms_notified).map {|r| r.split(',').map(&:strip) }.flatten
|
89
95
|
end
|
90
96
|
|
97
|
+
def status_data_for(status)
|
98
|
+
data = super(status)
|
99
|
+
data[:color] = status_color_for(status)
|
100
|
+
data
|
101
|
+
end
|
102
|
+
|
103
|
+
def status_color_for(status)
|
104
|
+
{
|
105
|
+
:success => success_color,
|
106
|
+
:warning => warning_color,
|
107
|
+
:failure => failure_color
|
108
|
+
}[status]
|
109
|
+
end
|
91
110
|
end
|
92
111
|
end
|
93
112
|
end
|
@@ -95,18 +95,13 @@ module Backup
|
|
95
95
|
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
96
96
|
#
|
97
97
|
def notify!(status)
|
98
|
-
|
99
|
-
when :success then '[Backup::Success]'
|
100
|
-
when :failure then '[Backup::Failure]'
|
101
|
-
when :warning then '[Backup::Warning]'
|
102
|
-
end
|
103
|
-
message = "#{ tag } #{ model.label } (#{ model.trigger })"
|
98
|
+
msg = message.call(model, :status => status_data_for(status))
|
104
99
|
|
105
100
|
opts = {
|
106
101
|
:headers => { 'User-Agent' => "Backup/#{ VERSION }" }.
|
107
102
|
merge(headers).reject {|k,v| v.nil? }.
|
108
103
|
merge('Content-Type' => 'application/x-www-form-urlencoded'),
|
109
|
-
:body => URI.encode_www_form({ 'message' =>
|
104
|
+
:body => URI.encode_www_form({ 'message' => msg }.
|
110
105
|
merge(params).reject {|k,v| v.nil? }.
|
111
106
|
merge('status' => status.to_s)),
|
112
107
|
:expects => success_codes # raise error if unsuccessful
|
data/lib/backup/notifier/mail.rb
CHANGED
@@ -37,6 +37,18 @@ module Backup
|
|
37
37
|
# Receiver Email Address
|
38
38
|
attr_accessor :to
|
39
39
|
|
40
|
+
##
|
41
|
+
# CC receiver Email Address
|
42
|
+
attr_accessor :cc
|
43
|
+
|
44
|
+
##
|
45
|
+
# BCC receiver Email Address
|
46
|
+
attr_accessor :bcc
|
47
|
+
|
48
|
+
##
|
49
|
+
# Set reply to email address
|
50
|
+
attr_accessor :reply_to
|
51
|
+
|
40
52
|
##
|
41
53
|
# SMTP Server Address
|
42
54
|
attr_accessor :address
|
@@ -148,14 +160,8 @@ module Backup
|
|
148
160
|
# : backup log, if `on_failure` is `true`.
|
149
161
|
#
|
150
162
|
def notify!(status)
|
151
|
-
tag = case status
|
152
|
-
when :success then '[Backup::Success]'
|
153
|
-
when :warning then '[Backup::Warning]'
|
154
|
-
when :failure then '[Backup::Failure]'
|
155
|
-
end
|
156
|
-
|
157
163
|
email = new_email
|
158
|
-
email.subject =
|
164
|
+
email.subject = message.call(model, :status => status_data_for(status))
|
159
165
|
|
160
166
|
send_log = send_log_on.include?(status)
|
161
167
|
template = Backup::Template.new({ :model => model, :send_log => send_log })
|
@@ -214,8 +220,11 @@ module Backup
|
|
214
220
|
end
|
215
221
|
|
216
222
|
email = ::Mail.new
|
217
|
-
email.to
|
218
|
-
email.from
|
223
|
+
email.to = to
|
224
|
+
email.from = from
|
225
|
+
email.cc = cc
|
226
|
+
email.bcc = bcc
|
227
|
+
email.reply_to = reply_to
|
219
228
|
email
|
220
229
|
end
|
221
230
|
|
@@ -224,7 +233,7 @@ module Backup
|
|
224
233
|
end
|
225
234
|
|
226
235
|
# Patch mail v2.5.4 Exim delivery method
|
227
|
-
# https://github.com/
|
236
|
+
# https://github.com/backup/backup/issues/446
|
228
237
|
# https://github.com/mikel/mail/pull/546
|
229
238
|
module Mail
|
230
239
|
class Exim
|
@@ -14,8 +14,8 @@ module Backup
|
|
14
14
|
|
15
15
|
##
|
16
16
|
# Nagios nrpe configuration file.
|
17
|
-
attr_accessor :send_nsca_cfg
|
18
|
-
|
17
|
+
attr_accessor :send_nsca_cfg
|
18
|
+
|
19
19
|
##
|
20
20
|
# Name of the Nagios service for the backup check.
|
21
21
|
attr_accessor :service_name
|
@@ -55,12 +55,7 @@ module Backup
|
|
55
55
|
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
56
56
|
#
|
57
57
|
def notify!(status)
|
58
|
-
message
|
59
|
-
when :success then 'Completed Successfully'
|
60
|
-
when :warning then 'Completed Successfully (with Warnings)'
|
61
|
-
when :failure then 'Failed'
|
62
|
-
end
|
63
|
-
send_message("#{ message } in #{ model.duration }")
|
58
|
+
send_message(message.call(model, :status => status_data_for(status)))
|
64
59
|
end
|
65
60
|
|
66
61
|
def send_message(message)
|
@@ -0,0 +1,81 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'pagerduty'
|
3
|
+
|
4
|
+
module Backup
|
5
|
+
module Notifier
|
6
|
+
class PagerDuty < Base
|
7
|
+
|
8
|
+
##
|
9
|
+
# PagerDuty Service API Key. Should be a 32 character hex string.
|
10
|
+
attr_accessor :service_key
|
11
|
+
|
12
|
+
##
|
13
|
+
# Determines if a backup with a warning should resolve an incident rather
|
14
|
+
# than trigger one.
|
15
|
+
#
|
16
|
+
# Defaults to false.
|
17
|
+
attr_accessor :resolve_on_warning
|
18
|
+
|
19
|
+
def initialize(mode, &block)
|
20
|
+
super
|
21
|
+
instance_eval(&block) if block_given?
|
22
|
+
|
23
|
+
@resolve_on_warning ||= false
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
##
|
29
|
+
# Trigger or resolve a PagerDuty incident for this model
|
30
|
+
#
|
31
|
+
# `status` indicates one of the following:
|
32
|
+
#
|
33
|
+
# `:success`
|
34
|
+
# : The backup completed successfully.
|
35
|
+
# : The incident will be resolved if `on_success` is `true`.
|
36
|
+
#
|
37
|
+
# `:warning`
|
38
|
+
# : The backup completed successfully, but warnings were logged.
|
39
|
+
# : An incident will be triggered if `on_warning` or `on_success` is `true`.
|
40
|
+
#
|
41
|
+
# `:failure`
|
42
|
+
# : The backup operation failed.
|
43
|
+
# : An incident will be triggered if `on_failure` is `true`.
|
44
|
+
#
|
45
|
+
def notify!(status)
|
46
|
+
incident_description = "Backup - #{model.label}"
|
47
|
+
incident_key = "backup/#{model.trigger}"
|
48
|
+
incident_details = {
|
49
|
+
:incident_key => incident_key,
|
50
|
+
:details => {
|
51
|
+
:trigger => model.trigger,
|
52
|
+
:label => model.label,
|
53
|
+
:started_at => model.started_at,
|
54
|
+
:finished_at => model.finished_at,
|
55
|
+
:duration => model.duration,
|
56
|
+
:status => status,
|
57
|
+
:exception => model.exception
|
58
|
+
}
|
59
|
+
}
|
60
|
+
|
61
|
+
event_type = case status
|
62
|
+
when :success then :resolve
|
63
|
+
when :warning then resolve_on_warning ? :resolve : :trigger
|
64
|
+
when :failure then :trigger
|
65
|
+
end
|
66
|
+
|
67
|
+
case event_type
|
68
|
+
when :trigger
|
69
|
+
pagerduty.trigger(incident_description, incident_details)
|
70
|
+
when :resolve
|
71
|
+
incident = pagerduty.get_incident(incident_key)
|
72
|
+
incident.resolve(incident_description, incident_details)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def pagerduty
|
77
|
+
@pagerduty ||= Pagerduty.new(service_key)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -16,6 +16,9 @@ module Backup
|
|
16
16
|
attr_accessor :api_key
|
17
17
|
|
18
18
|
def initialize(model, &block)
|
19
|
+
@message = lambda do |model, data|
|
20
|
+
"#{ model.label } (#{ model.trigger })"
|
21
|
+
end
|
19
22
|
super
|
20
23
|
instance_eval(&block) if block_given?
|
21
24
|
end
|
@@ -40,21 +43,17 @@ module Backup
|
|
40
43
|
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
41
44
|
#
|
42
45
|
def notify!(status)
|
43
|
-
|
44
|
-
when :success then '[Backup::Success]'
|
45
|
-
when :warning then '[Backup::Warning]'
|
46
|
-
when :failure then '[Backup::Failure]'
|
47
|
-
end
|
48
|
-
send_message(tag)
|
46
|
+
send_message(status)
|
49
47
|
end
|
50
48
|
|
51
|
-
def send_message(
|
49
|
+
def send_message(status)
|
52
50
|
uri = 'https://api.prowlapp.com/publicapi/add'
|
51
|
+
status_data = status_data_for(status)
|
53
52
|
data = {
|
54
53
|
:application => application,
|
55
54
|
:apikey => api_key,
|
56
|
-
:event => message,
|
57
|
-
:description =>
|
55
|
+
:event => status_data[:message],
|
56
|
+
:description => message.call(model, :status => status_data)
|
58
57
|
}
|
59
58
|
options = {
|
60
59
|
:headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
|
@@ -51,13 +51,7 @@ module Backup
|
|
51
51
|
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
52
52
|
#
|
53
53
|
def notify!(status)
|
54
|
-
|
55
|
-
when :success then '[Backup::Success]'
|
56
|
-
when :failure then '[Backup::Failure]'
|
57
|
-
when :warning then '[Backup::Warning]'
|
58
|
-
end
|
59
|
-
message = "#{ tag } #{ model.label } (#{ model.trigger })"
|
60
|
-
send_message(message)
|
54
|
+
send_message(message.call(model, :status => status_data_for(status)))
|
61
55
|
end
|
62
56
|
|
63
57
|
def send_message(message)
|
@@ -0,0 +1,88 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'aws/ses'
|
3
|
+
|
4
|
+
module Backup
|
5
|
+
module Notifier
|
6
|
+
class Ses < Base
|
7
|
+
|
8
|
+
##
|
9
|
+
# Amazon Simple Email Service (SES) Credentials
|
10
|
+
attr_accessor :access_key_id, :secret_access_key
|
11
|
+
|
12
|
+
##
|
13
|
+
# SES Region
|
14
|
+
attr_accessor :region
|
15
|
+
|
16
|
+
##
|
17
|
+
# Sender Email Address
|
18
|
+
attr_accessor :from
|
19
|
+
|
20
|
+
##
|
21
|
+
# Receiver Email Address
|
22
|
+
attr_accessor :to
|
23
|
+
|
24
|
+
def initialize(model, &block)
|
25
|
+
super
|
26
|
+
instance_eval(&block) if block_given?
|
27
|
+
|
28
|
+
@region ||= 'eu-west-1'
|
29
|
+
@send_log_on ||= [:warning, :failure]
|
30
|
+
end
|
31
|
+
|
32
|
+
##
|
33
|
+
# Array of statuses for which the log file should be attached.
|
34
|
+
#
|
35
|
+
# Available statuses are: `:success`, `:warning` and `:failure`.
|
36
|
+
# Default: [:warning, :failure]
|
37
|
+
attr_accessor :send_log_on
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
def client
|
42
|
+
AWS::SES::Base.new(
|
43
|
+
:access_key_id => access_key_id,
|
44
|
+
:secret_access_key => secret_access_key,
|
45
|
+
:server => "email.#{region}.amazonaws.com"
|
46
|
+
)
|
47
|
+
end
|
48
|
+
|
49
|
+
##
|
50
|
+
# Notify the user of the backup operation results.
|
51
|
+
#
|
52
|
+
# `status` indicates one of the following:
|
53
|
+
#
|
54
|
+
# `:success`
|
55
|
+
# : The backup completed successfully.
|
56
|
+
# : Notification will be sent if `on_success` is `true`.
|
57
|
+
#
|
58
|
+
# `:warning`
|
59
|
+
# : The backup completed successfully, but warnings were logged.
|
60
|
+
# : Notification will be sent, including a copy of the current
|
61
|
+
# : backup log, if `on_warning` or `on_success` is `true`.
|
62
|
+
#
|
63
|
+
# `:failure`
|
64
|
+
# : The backup operation failed.
|
65
|
+
# : Notification will be sent, including a copy of the current
|
66
|
+
# : backup log, if `on_failure` is `true`.
|
67
|
+
#
|
68
|
+
def notify!(status)
|
69
|
+
email = ::Mail.new(:to => to, :from => from)
|
70
|
+
email.subject = message.call(model, :status => status_data_for(status))
|
71
|
+
|
72
|
+
send_log = send_log_on.include?(status)
|
73
|
+
template = Backup::Template.new({ :model => model, :send_log => send_log })
|
74
|
+
email.body = template.result('notifier/mail/%s.erb' % status.to_s)
|
75
|
+
|
76
|
+
if send_log
|
77
|
+
email.convert_to_multipart
|
78
|
+
email.attachments["#{ model.time }.#{ model.trigger }.log"] = {
|
79
|
+
:mime_type => 'text/plain;',
|
80
|
+
:content => Logger.messages.map(&:formatted_lines).flatten.join("\n")
|
81
|
+
}
|
82
|
+
end
|
83
|
+
|
84
|
+
client.send_raw_email(email)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|