sp-job 0.1.17 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +1 -1
- data/README.md +40 -17
- data/VERSION +1 -1
- data/bin/console +1 -1
- data/bin/unique-file +55 -0
- data/lib/sp-job.rb +9 -8
- data/lib/sp/job/back_burner.rb +301 -42
- data/lib/sp/job/broker.rb +59 -21
- data/lib/sp/job/common.rb +288 -156
- data/lib/sp/job/job_db_adapter.rb +63 -0
- data/lib/sp/job/{engine.rb → jwt.rb} +14 -12
- data/lib/sp/job/mail_queue.rb +60 -0
- data/lib/sp/job/pg_connection.rb +68 -76
- data/lib/sp/job/unique_file.rb +73 -0
- data/lib/sp/job/uploaded_image_converter.rb +35 -16
- data/lib/sp/job/worker.rb +1 -8
- data/lib/sp/job/worker_thread.rb +57 -0
- data/lib/tasks/configure.rake +66 -16
- data/sp-job.gemspec +10 -8
- metadata +23 -17
data/lib/sp/job/broker.rb
CHANGED
@@ -19,13 +19,6 @@ module SP
|
|
19
19
|
#
|
20
20
|
class I18N
|
21
21
|
|
22
|
-
private
|
23
|
-
|
24
|
-
@key
|
25
|
-
@args
|
26
|
-
|
27
|
-
public
|
28
|
-
|
29
22
|
attr_accessor :key
|
30
23
|
attr_accessor :args
|
31
24
|
|
@@ -98,14 +91,6 @@ module SP
|
|
98
91
|
#
|
99
92
|
class OAuth2
|
100
93
|
|
101
|
-
private
|
102
|
-
|
103
|
-
@service_id = nil
|
104
|
-
@client = nil
|
105
|
-
@redis = nil
|
106
|
-
|
107
|
-
public
|
108
|
-
|
109
94
|
def initialize (service_id:, config:, redis: nil)
|
110
95
|
@service_id = service_id
|
111
96
|
@client = ::SP::Job::BrokerOAuth2Client.new(
|
@@ -131,7 +116,7 @@ module SP
|
|
131
116
|
ac_response = @client.get_authorization_code(
|
132
117
|
a_redirect_uri = nil,
|
133
118
|
a_scope = scope
|
134
|
-
)
|
119
|
+
)
|
135
120
|
# got a valid 'authorization code'?
|
136
121
|
if ac_response[:oauth2].has_key?(:code)
|
137
122
|
# got fields?
|
@@ -142,7 +127,7 @@ module SP
|
|
142
127
|
array << k.to_s
|
143
128
|
array << v
|
144
129
|
end
|
145
|
-
|
130
|
+
@redis.hmset("#{@service_id}:oauth:authorization_code:#{ac_response[:oauth2][:code]}",
|
146
131
|
array,
|
147
132
|
'patched_by', 'toconline-session'
|
148
133
|
)
|
@@ -212,7 +197,7 @@ module SP
|
|
212
197
|
a_scope = nil # keep current scope
|
213
198
|
)
|
214
199
|
if at_response[:oauth2].has_key?(:error)
|
215
|
-
|
200
|
+
raise ::SP::Job::Broker::InternalError.new(i18n: nil, internal: at_response[:oauth2][:error])
|
216
201
|
end
|
217
202
|
# prepare redis arguments: field value, [field value, ...]
|
218
203
|
array = []
|
@@ -294,6 +279,58 @@ module SP
|
|
294
279
|
}
|
295
280
|
end
|
296
281
|
|
282
|
+
#
|
283
|
+
# Obtain an 'access' and a 'refresh' token.
|
284
|
+
#
|
285
|
+
# @param args check the authorize method o OAuth2 class
|
286
|
+
# @return hash with response content type and status code
|
287
|
+
#
|
288
|
+
def authorize (args)
|
289
|
+
call do
|
290
|
+
finalized(response: oauth2.authorize(args))
|
291
|
+
end
|
292
|
+
@output
|
293
|
+
end
|
294
|
+
|
295
|
+
#
|
296
|
+
# Refresh an access token.
|
297
|
+
#
|
298
|
+
# @param args check the refresh method o OAuth2 class
|
299
|
+
# @return hash with response content type and status code
|
300
|
+
#
|
301
|
+
def refresh (args)
|
302
|
+
call do
|
303
|
+
finalized(response: oauth2.refresh(args))
|
304
|
+
end
|
305
|
+
@output
|
306
|
+
end
|
307
|
+
|
308
|
+
#
|
309
|
+
# Patch a pair of tokens, by generating new ones
|
310
|
+
#
|
311
|
+
# @param args check the patch methods o OAuth2 class
|
312
|
+
# @return hash with response content type and status code
|
313
|
+
#
|
314
|
+
def patch (args)
|
315
|
+
call do
|
316
|
+
finalized(response: oauth2.patch(args))
|
317
|
+
end
|
318
|
+
@output
|
319
|
+
end
|
320
|
+
|
321
|
+
#
|
322
|
+
# Remove a pair of tokens from redis.
|
323
|
+
#
|
324
|
+
# @param args check the dispose method o OAuth2 class
|
325
|
+
# @return hash with response content type and status code
|
326
|
+
#
|
327
|
+
def dispose (args)
|
328
|
+
call do
|
329
|
+
finalized(response: oauth2.dispose(args))
|
330
|
+
end
|
331
|
+
@output
|
332
|
+
end
|
333
|
+
|
297
334
|
#
|
298
335
|
# Finalize the job response.
|
299
336
|
#
|
@@ -301,7 +338,7 @@ module SP
|
|
301
338
|
# @param content_type
|
302
339
|
# @param status_code
|
303
340
|
#
|
304
|
-
# @return
|
341
|
+
# @return hash with response content type and status code
|
305
342
|
#
|
306
343
|
def finalized (response:, content_type: 'application/json', status_code: 200)
|
307
344
|
@output[:response] = response
|
@@ -312,11 +349,12 @@ module SP
|
|
312
349
|
|
313
350
|
#
|
314
351
|
# Perform an OAuth2 request, catch errors
|
315
|
-
# and
|
352
|
+
# and convert them to a common result hash.
|
316
353
|
#
|
317
354
|
# @param callback
|
318
355
|
#
|
319
356
|
def call(*callback)
|
357
|
+
@output = {}
|
320
358
|
begin
|
321
359
|
@output = yield
|
322
360
|
rescue ::SP::Job::BrokerOAuth2Client::InvalidEmailOrPassword => invalid_password
|
@@ -357,7 +395,7 @@ module SP
|
|
357
395
|
@output[:status_code] = broker_error.code
|
358
396
|
@output[:content_type], @output[:response] = broker_error.content_type_and_body()
|
359
397
|
rescue Exception => e
|
360
|
-
internal_error = InternalError.new(i18n: nil, internal:
|
398
|
+
internal_error = InternalError.new(i18n: nil, internal: e.message)
|
361
399
|
@output[:status_code] = internal_error.code
|
362
400
|
@output[:content_type], @output[:response] = internal_error.content_type_and_body()
|
363
401
|
end
|
data/lib/sp/job/common.rb
CHANGED
@@ -24,6 +24,84 @@ module SP
|
|
24
24
|
module Job
|
25
25
|
module Common
|
26
26
|
|
27
|
+
ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
|
28
|
+
|
29
|
+
def thread_data
|
30
|
+
$thread_data[Thread.current]
|
31
|
+
end
|
32
|
+
|
33
|
+
def http (oauth_client_id:, oauth_client_secret:)
|
34
|
+
|
35
|
+
$http_oauth_clients ||= {}
|
36
|
+
$http_oauth_clients[oauth_client_id] ||= ::SP::Job::BrokerHTTPClient.new(
|
37
|
+
a_session = ::SP::Job::BrokerHTTPClient::Session.new(
|
38
|
+
a_access_token = nil,
|
39
|
+
a_refresh_token = nil,
|
40
|
+
a_scope = $config[:api][:oauth][:scope]
|
41
|
+
),
|
42
|
+
a_oauth2_client = ::SP::Job::BrokerOAuth2Client.new(
|
43
|
+
protocol: $config[:api][:oauth][:protocol],
|
44
|
+
host: $config[:api][:oauth][:host],
|
45
|
+
port: $config[:api][:oauth][:port],
|
46
|
+
client_id: oauth_client_id,
|
47
|
+
client_secret: oauth_client_secret,
|
48
|
+
redirect_uri: $config[:api][:oauth][:redirect_uri],
|
49
|
+
scope: $config[:api][:oauth][:scope],
|
50
|
+
options: {}
|
51
|
+
),
|
52
|
+
a_refreshed_callback = method(:refreshed_callback),
|
53
|
+
a_auto_renew_refresh_token = true
|
54
|
+
)
|
55
|
+
$http_oauth_clients[oauth_client_id]
|
56
|
+
end
|
57
|
+
|
58
|
+
#
|
59
|
+
# Called by BrokerHTTPClient when a new session was created
|
60
|
+
# or an older one was refreshed.
|
61
|
+
#
|
62
|
+
def refreshed_callback(a_session)
|
63
|
+
logger.task('#', "Session #{a_session.is_new ? 'created' : 'refreshed' }, access_token=#{a_session.access_token}, refresh_token=#{a_session.refresh_token}")
|
64
|
+
end
|
65
|
+
|
66
|
+
def db
|
67
|
+
$pg
|
68
|
+
end
|
69
|
+
|
70
|
+
def redis
|
71
|
+
$redis
|
72
|
+
end
|
73
|
+
|
74
|
+
def config
|
75
|
+
$config
|
76
|
+
end
|
77
|
+
|
78
|
+
#
|
79
|
+
# Returns the object you should use to perform JSON api requests
|
80
|
+
#
|
81
|
+
# jsonapi.get! (resource, params)
|
82
|
+
# jsonapi.post! (resource, params)
|
83
|
+
# jsonapi.patch! (resource, params)
|
84
|
+
# jsonapi.delete! (resource)
|
85
|
+
#
|
86
|
+
def jsonapi
|
87
|
+
thread_data.jsonapi.adapter
|
88
|
+
end
|
89
|
+
|
90
|
+
#
|
91
|
+
# You should not use this method ... unless ... you REALLY need to overide the JSON:API
|
92
|
+
# parameters defined by the JOB object
|
93
|
+
#
|
94
|
+
def set_jsonapi_parameters (params)
|
95
|
+
thread_data.jsonapi.set_jsonapi_parameters(SP::Duh::JSONAPI::ParametersNotPicky.new(params))
|
96
|
+
end
|
97
|
+
|
98
|
+
# You should not use this method ... unless ... you REALLY need to overide the JSON:API
|
99
|
+
# parameters defined by the JOB object
|
100
|
+
#
|
101
|
+
def get_jsonapi_parameters
|
102
|
+
HashWithIndifferentAccess.new(JSON.parse(thread_data.jsonapi.parameters.to_json))
|
103
|
+
end
|
104
|
+
|
27
105
|
def logger
|
28
106
|
Backburner.configuration.logger
|
29
107
|
end
|
@@ -37,7 +115,42 @@ module SP
|
|
37
115
|
]
|
38
116
|
end
|
39
117
|
|
118
|
+
def get_random_folder
|
119
|
+
ALPHABET[rand(26)] + ALPHABET[rand(26)]
|
120
|
+
end
|
121
|
+
|
122
|
+
def send_to_upload_server (src_file:, id:, extension:, entity: 'company')
|
123
|
+
folder = get_random_folder
|
124
|
+
remote_path = File.join(entity, id_to_path(id.to_i), folder)
|
125
|
+
if config[:uploads][:local] == true
|
126
|
+
destination_file = ::SP::Job::Unique::File.create(File.join(config[:uploads][:path], remote_path), extension)
|
127
|
+
FileUtils.cp(src_file, destination_file)
|
128
|
+
else
|
129
|
+
uploads_server = config[:uploads][:server]
|
130
|
+
destination_file = %x[ssh #{uploads_server} unique-file -p #{File.join(config[:uploads][:path], remote_path)} -e #{extension}].strip
|
131
|
+
if $?.exit_status == 0
|
132
|
+
%x[scp #{src_file} #{uploads_server}:#{remote_file}]
|
133
|
+
raise_error(message: 'i18n_upload_to_server_failed') if $?.exit_status != 0
|
134
|
+
else
|
135
|
+
raise_error(message: 'i18n_upload_to_server_failed')
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
return entity[0] + folder + destination_file[-(6+extension.length)..-1]
|
140
|
+
end
|
141
|
+
|
40
142
|
def submit_job (args)
|
143
|
+
if $redis_mutex.nil?
|
144
|
+
rv = _submit_job(args)
|
145
|
+
else
|
146
|
+
$redis_mutex.synchronize {
|
147
|
+
rv = _submit_job(args)
|
148
|
+
}
|
149
|
+
end
|
150
|
+
rv
|
151
|
+
end
|
152
|
+
|
153
|
+
def _submit_job (args)
|
41
154
|
job = args[:job]
|
42
155
|
tube = args[:tube] || $args[:program_name]
|
43
156
|
raise 'missing job argument' unless args[:job]
|
@@ -53,157 +166,215 @@ module SP
|
|
53
166
|
$redis.expire(redis_key, validity)
|
54
167
|
end
|
55
168
|
$beaneater.tubes[tube].put job.to_json, ttr: ttr
|
169
|
+
"#{tube}:#{job[:id]}"
|
56
170
|
end
|
57
171
|
|
58
|
-
def
|
172
|
+
def prepare_job (job)
|
173
|
+
logger.debug "Preparing job id #{job[:id]}".green
|
59
174
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
$connected = true
|
64
|
-
end
|
65
|
-
|
66
|
-
$job_status = {
|
67
|
-
action: 'response',
|
175
|
+
td = thread_data
|
176
|
+
td.current_job = job
|
177
|
+
td.job_status = {
|
68
178
|
content_type: 'application/json',
|
69
|
-
progress:
|
179
|
+
progress: [
|
180
|
+
{
|
181
|
+
message: nil,
|
182
|
+
value: 0
|
183
|
+
}
|
184
|
+
]
|
70
185
|
}
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
$validity = job[:validity].nil? ? 300 : job[:validity].to_i
|
186
|
+
td.report_time_stamp = 0
|
187
|
+
td.exception_reported = false
|
188
|
+
td.job_id = job[:id]
|
189
|
+
td.publish_key = $config[:service_id] + ':' + (job[:tube] || $args[:program_name]) + ':' + job[:id]
|
190
|
+
td.job_key = $config[:service_id] + ':jobs:' + (job[:tube] || $args[:program_name]) + ':' + job[:id]
|
77
191
|
if $config[:options] && $config[:options][:jsonapi] == true
|
78
192
|
raise "Job didn't specify the mandatory field prefix!" if job[:prefix].blank?
|
79
|
-
|
80
|
-
|
81
|
-
init_params[:user_id] = job[:user_id] unless job[:user_id].blank?
|
82
|
-
init_params[:company_id] = job[:company_id] unless job[:company_id].blank?
|
83
|
-
init_params[:company_schema] = job[:company_schema] unless job[:company_schema].blank?
|
84
|
-
init_params[:sharded_schema] = job[:sharded_schema] unless job[:sharded_schema].blank?
|
85
|
-
init_params[:accounting_prefix] = job[:accounting_prefix] unless job[:accounting_prefix].blank?
|
86
|
-
init_params[:accounting_schema] = job[:accounting_schema] unless job[:accounting_schema].blank?
|
87
|
-
|
88
|
-
$jsonapi.set_jsonapi_parameters(SP::Duh::JSONAPI::Parameters.new(init_params))
|
193
|
+
td.jsonapi.set_url(job[:prefix])
|
194
|
+
td.set_jsonapi_parameters(SP::Duh::JSONAPI::ParametersNotPicky.new(job))
|
89
195
|
end
|
90
196
|
|
91
197
|
# Make sure the job is still allowed to run by checking if the key exists in redis
|
92
|
-
unless $redis.exists(
|
93
|
-
logger.warn "Job validity has expired: job ignored"
|
198
|
+
unless $redis.exists(td.job_key )
|
199
|
+
logger.warn "Job validity has expired: job ignored".yellow
|
94
200
|
return false
|
95
201
|
end
|
96
202
|
return true
|
97
203
|
end
|
98
204
|
|
99
|
-
#
|
100
|
-
# Optionally after the jobs runs sucessfully clean the "job" key in redis
|
101
|
-
#
|
102
|
-
def after_perform_cleanup (job)
|
103
|
-
if false # TODO check key namings with americo $job key and redis key
|
104
|
-
return if $redis.nil?
|
105
|
-
return if $job_key.nil?
|
106
|
-
$redis.del $job_key
|
107
|
-
end
|
108
|
-
end
|
109
|
-
|
110
205
|
def update_progress (args)
|
111
|
-
|
206
|
+
td = thread_data
|
112
207
|
status = args[:status]
|
113
208
|
progress = args[:progress]
|
114
|
-
|
115
|
-
p_index = args[:index]
|
116
|
-
response = args[:response]
|
209
|
+
p_index = args[:index] || 0
|
117
210
|
|
118
211
|
if args.has_key? :message
|
119
212
|
message_args = Hash.new
|
120
213
|
args.each do |key, value|
|
121
|
-
next if [:step, :progress, :message, :status, :barrier, :index, :response].include? key
|
214
|
+
next if [:step, :progress, :message, :status, :barrier, :index, :response, :action, :content_type, :status_code, :link].include? key
|
122
215
|
message_args[key] = value
|
123
216
|
end
|
124
217
|
message = [ args[:message], message_args ]
|
125
218
|
else
|
126
219
|
message = nil
|
127
220
|
end
|
128
|
-
$job_status[:progress] = progress.to_f.round(2) unless progress.nil?
|
129
|
-
$job_status[:progress] = ($job_status[:progress] + step.to_f).round(2) unless step.nil?
|
130
|
-
$job_status[:message] = message unless message.nil?
|
131
|
-
$job_status[:index] = p_index unless p_index.nil?
|
132
|
-
$job_status[:status] = status.nil? ? 'in-progress' : status
|
133
|
-
$job_status[:response] = response unless response.nil?
|
134
|
-
if args.has_key? :link
|
135
|
-
$job_status[:link] = args[:link]
|
136
|
-
end
|
137
221
|
|
138
|
-
|
139
|
-
|
140
|
-
|
222
|
+
# update job status
|
223
|
+
if p_index >= td.job_status[:progress].size
|
224
|
+
(1 + p_index - td.job_status[:progress].size).times do
|
225
|
+
td.job_status[:progress] << { message: nil, value: 0 }
|
141
226
|
end
|
142
227
|
end
|
228
|
+
unless message.nil?
|
229
|
+
td.job_status[:progress][p_index][:message] = message
|
230
|
+
end
|
231
|
+
unless progress.nil?
|
232
|
+
td.job_status[:progress][p_index][:value] = progress.to_f.round(2)
|
233
|
+
end
|
234
|
+
td.job_status[:status] = status.nil? ? 'in-progress' : status
|
235
|
+
td.job_status[:link] = args[:link] if args[:link]
|
236
|
+
td.job_status[:status_code] = args[:status_code] if args[:status_code]
|
237
|
+
if args.has_key? :response
|
238
|
+
td.job_status[:response] = args[:response]
|
239
|
+
td.job_status[:content_type] = args[:content_type]
|
240
|
+
td.job_status[:action] = args[:action]
|
241
|
+
end
|
242
|
+
|
243
|
+
# Create notification that will be published
|
244
|
+
td.job_notification = {}
|
245
|
+
td.job_notification[:progress] = progress.to_f.round(2) unless progress.nil?
|
246
|
+
td.job_notification[:message] = message unless message.nil?
|
247
|
+
td.job_notification[:index] = p_index unless p_index.nil?
|
248
|
+
td.job_notification[:status] = status.nil? ? 'in-progress' : status
|
249
|
+
td.job_notification[:link] = args[:link] if args[:link]
|
250
|
+
td.job_notification[:status_code] = args[:status_code] if args[:status_code]
|
251
|
+
if args.has_key? :response
|
252
|
+
td.job_notification[:response] = args[:response]
|
253
|
+
td.job_notification[:content_type] = args[:content_type]
|
254
|
+
td.job_notification[:action] = args[:action]
|
255
|
+
end
|
143
256
|
|
144
|
-
if
|
257
|
+
if ['completed', 'error', 'follow-up', 'cancelled'].include?(status) || (Time.now.to_f - td.report_time_stamp) > $min_progress || args[:barrier]
|
145
258
|
update_progress_on_redis
|
146
259
|
end
|
147
260
|
end
|
148
261
|
|
149
|
-
def
|
150
|
-
|
262
|
+
def send_response (args)
|
263
|
+
td = thread_data
|
264
|
+
args[:status] ||= 'completed'
|
265
|
+
args[:action] ||= 'response'
|
266
|
+
args[:content_type] ||= 'application/json'
|
267
|
+
args[:response] ||= {}
|
268
|
+
args[:status_code] ||= 200
|
151
269
|
update_progress(args)
|
152
|
-
|
153
|
-
raise args[:message]
|
154
|
-
end
|
155
|
-
|
156
|
-
def update_progress_on_redis
|
157
|
-
$redis.pipelined do
|
158
|
-
redis_str = $job_status.to_json
|
159
|
-
$redis.publish $publish_key, redis_str
|
160
|
-
$redis.hset $job_key, 'status', redis_str
|
161
|
-
$redis.expire $job_key, $validity
|
162
|
-
end
|
163
|
-
$report_time_stamp = Time.now.to_f
|
270
|
+
td.job_id = nil
|
164
271
|
end
|
165
272
|
|
166
|
-
def
|
167
|
-
|
168
|
-
|
273
|
+
def error_handler(args)
|
274
|
+
td = thread_data
|
275
|
+
args[:status] ||= 'error'
|
276
|
+
args[:action] ||= 'response'
|
277
|
+
args[:content_type] ||= 'application/json'
|
278
|
+
args[:status_code] ||= 500
|
279
|
+
update_progress(args)
|
280
|
+
logger.error(args)
|
281
|
+
td.exception_reported = true
|
282
|
+
td.job_id = nil
|
169
283
|
end
|
170
284
|
|
171
|
-
def
|
172
|
-
|
173
|
-
|
285
|
+
def report_error (args)
|
286
|
+
td = thread_data
|
287
|
+
error_handler(args)
|
288
|
+
raise ::SP::Job::JobAborted.new(args: args, job: td.current_job)
|
174
289
|
end
|
175
290
|
|
176
|
-
def
|
177
|
-
|
178
|
-
|
291
|
+
def raise_error (args)
|
292
|
+
td = thread_data
|
293
|
+
error_handler(args)
|
294
|
+
raise ::SP::Job::JobException.new(args: args, job: td.current_job)
|
179
295
|
end
|
180
296
|
|
181
|
-
def
|
182
|
-
|
183
|
-
$
|
297
|
+
def send_text_response (response)
|
298
|
+
td = thread_data
|
299
|
+
$redis.pipelined do
|
300
|
+
$redis.publish td.publish_key, response
|
301
|
+
$redis.hset td.job_key, 'status', response
|
302
|
+
end
|
184
303
|
end
|
185
304
|
|
186
|
-
def
|
187
|
-
|
305
|
+
def update_progress_on_redis
|
306
|
+
td = thread_data
|
307
|
+
if $redis_mutex.nil?
|
308
|
+
if $transient_job
|
309
|
+
$redis.publish td.publish_key, td.job_notification.to_json
|
310
|
+
else
|
311
|
+
$redis.pipelined do
|
312
|
+
$redis.publish td.publish_key, td.job_notification.to_json
|
313
|
+
$redis.hset td.job_key, 'status', td.job_status.to_json
|
314
|
+
end
|
315
|
+
end
|
316
|
+
else
|
317
|
+
$redis_mutex.synchronize {
|
318
|
+
if $transient_job
|
319
|
+
$redis.publish td.publish_key, td.job_notification.to_json
|
320
|
+
else
|
321
|
+
$redis.pipelined do
|
322
|
+
$redis.publish td.publish_key, td.job_notification.to_json
|
323
|
+
$redis.hset td.job_key, 'status', td.job_status.to_json
|
324
|
+
end
|
325
|
+
end
|
326
|
+
}
|
327
|
+
end
|
328
|
+
td.report_time_stamp = Time.now.to_f
|
188
329
|
end
|
189
330
|
|
190
331
|
def expand_mail_body (template)
|
191
|
-
if
|
192
|
-
|
332
|
+
if template.class == Hash
|
333
|
+
template_path = template[:path]
|
334
|
+
erb_binding = OpenStruct.new(template[:args]).instance_eval { binding }
|
335
|
+
else
|
336
|
+
template_path = template
|
337
|
+
erb_binding = binding
|
338
|
+
end
|
339
|
+
|
340
|
+
if File.extname(template_path) == ''
|
341
|
+
template_path += '.erb'
|
193
342
|
end
|
194
|
-
|
195
|
-
|
343
|
+
|
344
|
+
if template_path[0] == '/'
|
345
|
+
erb_template = File.read(template_path)
|
196
346
|
else
|
197
|
-
erb_template = File.read(File.join(File.expand_path(File.dirname($PROGRAM_NAME)),
|
347
|
+
erb_template = File.read(File.join(File.expand_path(File.dirname($PROGRAM_NAME)), template_path))
|
198
348
|
end
|
199
|
-
|
349
|
+
|
350
|
+
ERB.new(erb_template).result(erb_binding)
|
200
351
|
end
|
201
352
|
|
202
353
|
def send_email (args)
|
203
|
-
|
354
|
+
|
355
|
+
if args.has_key?(:body) && args[:body] != nil
|
356
|
+
email_body = args[:body]
|
357
|
+
elsif args.has_key?(:template) && args[:template] != nil
|
204
358
|
email_body = expand_mail_body args[:template]
|
205
|
-
|
359
|
+
end
|
360
|
+
|
361
|
+
submit_job(
|
362
|
+
tube: 'mail-queue',
|
363
|
+
job: {
|
364
|
+
to: args[:to],
|
365
|
+
subject: args[:subject],
|
366
|
+
reply_to: args[:reply_to],
|
367
|
+
body: email_body
|
368
|
+
}
|
369
|
+
)
|
370
|
+
end
|
371
|
+
|
372
|
+
def synchronous_send_email (args)
|
373
|
+
|
374
|
+
if args.has_key?(:body) && args[:body] != nil
|
206
375
|
email_body = args[:body]
|
376
|
+
elsif args.has_key?(:template) && args[:template] != nil
|
377
|
+
email_body = expand_mail_body args[:template]
|
207
378
|
end
|
208
379
|
|
209
380
|
document = Roadie::Document.new email_body
|
@@ -213,6 +384,7 @@ module SP
|
|
213
384
|
from $config[:mail][:from]
|
214
385
|
to args[:to]
|
215
386
|
subject args[:subject]
|
387
|
+
reply_to (args[:reply_to] || $config[:mail][:from])
|
216
388
|
|
217
389
|
html_part do
|
218
390
|
content_type 'text/html; charset=UTF-8'
|
@@ -220,76 +392,36 @@ module SP
|
|
220
392
|
end
|
221
393
|
end
|
222
394
|
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
end
|
236
|
-
|
237
|
-
def database_connect
|
238
|
-
# any connection to close?
|
239
|
-
if ! $jsonapi.nil?
|
240
|
-
$jsonapi.close
|
241
|
-
$jsonapi = nil
|
242
|
-
end
|
243
|
-
if nil != $pg
|
244
|
-
$pg.disconnect()
|
245
|
-
$pg = nil
|
246
|
-
end
|
247
|
-
# establish new connection?
|
248
|
-
if $config[:postgres] && $config[:postgres][:conn_str]
|
249
|
-
$pg = ::SP::Job::PGConnection.new(owner: 'back_burner', config: $config[:postgres])
|
250
|
-
$pg.connect()
|
251
|
-
if $config[:options][:jsonapi] == true
|
252
|
-
$jsonapi = SP::Duh::JSONAPI::Service.new($pg.connection, ($jsonapi.nil? ? nil : $jsonapi.url))
|
395
|
+
if args.has_key?(:attachments) && args[:attachments] != nil
|
396
|
+
args[:attachments].each do |attach|
|
397
|
+
attach_uri = URI.escape("#{attach[:protocol]}://#{attach[:host]}:#{attach[:port]}/#{attach[:path]}/#{attach[:file]}")
|
398
|
+
attach_http_call = Curl::Easy.http_get(attach_uri)
|
399
|
+
if attach_http_call.response_code == 200
|
400
|
+
attributes = {}
|
401
|
+
attach_http_call.header_str.scan(/(\w+)="([^"]*)"/).each do |group|
|
402
|
+
attributes[group[0].to_sym] = group[1]
|
403
|
+
end
|
404
|
+
|
405
|
+
m.attachments[attributes[:filename].force_encoding('UTF-8')] = { mime_type: attach_http_call.content_type, content: attach_http_call.body_str }
|
406
|
+
end
|
253
407
|
end
|
254
408
|
end
|
255
|
-
end
|
256
409
|
|
257
|
-
|
258
|
-
min = $config[:postgres][:min_queries_per_conn]
|
259
|
-
max = $config[:postgres][:max_queries_per_conn]
|
260
|
-
if (!max.nil? && max > 0) || (!min.nil? && min > 0)
|
261
|
-
$db_life_span = 0
|
262
|
-
$check_db_life_span = true
|
263
|
-
new_min, new_max = [min, max].minmax
|
264
|
-
new_min = new_min if new_min <= 0
|
265
|
-
if new_min + new_min > 0
|
266
|
-
$db_treshold = (new_min + (new_min - new_min) * rand).to_i
|
267
|
-
else
|
268
|
-
$db_treshold = new_min.to_i
|
269
|
-
end
|
270
|
-
end
|
410
|
+
m.deliver!
|
271
411
|
end
|
272
412
|
|
273
|
-
def
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
end
|
280
|
-
end
|
413
|
+
def pg_server_error(e)
|
414
|
+
raise e if e.is_a?(::SP::Job::JobCancelled)
|
415
|
+
base_exception = e
|
416
|
+
begin
|
417
|
+
base_exception = base_exception.cause
|
418
|
+
end while base_exception.respond_to?(:cause) && !base_exception.cause.blank?
|
281
419
|
|
282
|
-
|
283
|
-
case e
|
284
|
-
when PG::UnableToSend, PG::AdminShutdown, PG::ConnectionBad
|
285
|
-
logger.fatal "Lost connection to database exiting now"
|
286
|
-
exit
|
287
|
-
when Redis::CannotConnectError
|
288
|
-
logger.fatal "Can't connect to redis exiting now"
|
289
|
-
exit
|
290
|
-
end
|
420
|
+
return base_exception.is_a?(PG::ServerError) ? e.cause.result.error_field(PG::PG_DIAG_MESSAGE_PRIMARY) : e.message
|
291
421
|
end
|
292
422
|
|
423
|
+
def get_percentage(total: 1, count: 0) ; (count * 100 / total).to_i ; end
|
424
|
+
|
293
425
|
end # Module Common
|
294
426
|
end # Module Job
|
295
|
-
end # Module SP
|
427
|
+
end # Module SP
|