sp-job 0.1.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,295 @@
1
+ #
2
+ # Copyright (c) 2011-2016 Cloudware S.A. All rights reserved.
3
+ #
4
+ # This file is part of sp-job.
5
+ #
6
+ # And this is the mix-in we'll apply to Job execution classes
7
+ #
8
+ # sp-job is free software: you can redistribute it and/or modify
9
+ # it under the terms of the GNU Affero General Public License as published by
10
+ # the Free Software Foundation, either version 3 of the License, or
11
+ # (at your option) any later version.
12
+ #
13
+ # sp-job is distributed in the hope that it will be useful,
14
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
15
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16
+ # GNU General Public License for more details.
17
+ #
18
+ # You should have received a copy of the GNU Affero General Public License
19
+ # along with sp-job. If not, see <http://www.gnu.org/licenses/>.
20
+ #
21
+ # encoding: utf-8
22
+ #
23
+ module SP
24
+ module Job
25
+ module Common
26
+
27
+ def logger
28
+ Backburner.configuration.logger
29
+ end
30
+
31
+ def id_to_path (id)
32
+ "%03d/%03d/%03d/%03d" % [
33
+ (id % 1000000000000) / 1000000000,
34
+ (id % 1000000000) / 1000000 ,
35
+ (id % 1000000) / 1000 ,
36
+ (id % 1000)
37
+ ]
38
+ end
39
+
40
+ def submit_job (args)
41
+ job = args[:job]
42
+ tube = args[:tube] || $args[:program_name]
43
+ raise 'missing job argument' unless args[:job]
44
+
45
+ validity = args[:validity] || 180
46
+ ttr = args[:ttr] || 60
47
+ job[:id] = ($redis.incr "#{$config[:service_id]}:jobs:sequential_id").to_s
48
+ job[:tube] = tube
49
+ job[:validity] = validity
50
+ redis_key = "#{$config[:service_id]}:jobs:#{tube}:#{job[:id]}"
51
+ $redis.pipelined do
52
+ $redis.hset(redis_key, 'status', '{"status":"queued"}')
53
+ $redis.expire(redis_key, validity)
54
+ end
55
+ $beaneater.tubes[tube].put job.to_json, ttr: ttr
56
+ end
57
+
58
+ def before_perform_init (job)
59
+
60
+ if $connected == false
61
+ database_connect
62
+ $redis.get "#{$config}:jobs:sequential_id"
63
+ $connected = true
64
+ end
65
+
66
+ $job_status = {
67
+ action: 'response',
68
+ content_type: 'application/json',
69
+ progress: 0
70
+ }
71
+ $report_time_stamp = 0
72
+ $job_status[:progress] = 0
73
+ $exception_reported = false
74
+ $publish_key = $config[:service_id] + ':' + (job[:tube] || $args[:program_name]) + ':' + job[:id]
75
+ $job_key = $config[:service_id] + ':jobs:' + (job[:tube] || $args[:program_name]) + ':' + job[:id]
76
+ $validity = job[:validity].nil? ? 300 : job[:validity].to_i
77
+ if $config[:options] && $config[:options][:jsonapi] == true
78
+ raise "Job didn't specify the mandatory field prefix!" if job[:prefix].blank?
79
+ $jsonapi.set_url(job[:prefix])
80
+ init_params = {}
81
+ init_params[:user_id] = job[:user_id] unless job[:user_id].blank?
82
+ init_params[:company_id] = job[:company_id] unless job[:company_id].blank?
83
+ init_params[:company_schema] = job[:company_schema] unless job[:company_schema].blank?
84
+ init_params[:sharded_schema] = job[:sharded_schema] unless job[:sharded_schema].blank?
85
+ init_params[:accounting_prefix] = job[:accounting_prefix] unless job[:accounting_prefix].blank?
86
+ init_params[:accounting_schema] = job[:accounting_schema] unless job[:accounting_schema].blank?
87
+
88
+ $jsonapi.set_jsonapi_parameters(SP::Duh::JSONAPI::Parameters.new(init_params))
89
+ end
90
+
91
+ # Make sure the job is still allowed to run by checking if the key exists in redis
92
+ unless $redis.exists($job_key )
93
+ logger.warn "Job validity has expired: job ignored"
94
+ return false
95
+ end
96
+ return true
97
+ end
98
+
99
+ #
100
+ # Optionally after the jobs runs sucessfully clean the "job" key in redis
101
+ #
102
+ def after_perform_cleanup (job)
103
+ if false # TODO check key namings with americo $job key and redis key
104
+ return if $redis.nil?
105
+ return if $job_key.nil?
106
+ $redis.del $job_key
107
+ end
108
+ end
109
+
110
+ def update_progress (args)
111
+ step = args[:step]
112
+ status = args[:status]
113
+ progress = args[:progress]
114
+ barrier = args[:barrier]
115
+ p_index = args[:index]
116
+ response = args[:response]
117
+
118
+ if args.has_key? :message
119
+ message_args = Hash.new
120
+ args.each do |key, value|
121
+ next if [:step, :progress, :message, :status, :barrier, :index, :response].include? key
122
+ message_args[key] = value
123
+ end
124
+ message = [ args[:message], message_args ]
125
+ else
126
+ message = nil
127
+ end
128
+ $job_status[:progress] = progress.to_f.round(2) unless progress.nil?
129
+ $job_status[:progress] = ($job_status[:progress] + step.to_f).round(2) unless step.nil?
130
+ $job_status[:message] = message unless message.nil?
131
+ $job_status[:index] = p_index unless p_index.nil?
132
+ $job_status[:status] = status.nil? ? 'in-progress' : status
133
+ $job_status[:response] = response unless response.nil?
134
+ if args.has_key? :link
135
+ $job_status[:link] = args[:link]
136
+ end
137
+
138
+ if args.has_key? :extra
139
+ args[:extra].each do |key, value|
140
+ $job_status[key] = value
141
+ end
142
+ end
143
+
144
+ if status == 'completed' || status == 'error' || (Time.now.to_f - $report_time_stamp) > $min_progress || barrier
145
+ update_progress_on_redis
146
+ end
147
+ end
148
+
149
+ def raise_error (args)
150
+ args[:status] = 'error'
151
+ update_progress(args)
152
+ $exception_reported = true
153
+ raise args[:message]
154
+ end
155
+
156
+ def update_progress_on_redis
157
+ $redis.pipelined do
158
+ redis_str = $job_status.to_json
159
+ $redis.publish $publish_key, redis_str
160
+ $redis.hset $job_key, 'status', redis_str
161
+ $redis.expire $job_key, $validity
162
+ end
163
+ $report_time_stamp = Time.now.to_f
164
+ end
165
+
166
+ def get_jsonapi!(path, params)
167
+ check_db_life_span()
168
+ $jsonapi.adapter.get!(path, params)
169
+ end
170
+
171
+ def post_jsonapi!(path, params)
172
+ check_db_life_span()
173
+ $jsonapi.adapter.post!(path, params)
174
+ end
175
+
176
+ def patch_jsonapi!(path, params)
177
+ check_db_life_span()
178
+ $jsonapi.adapter.patch!(path, params)
179
+ end
180
+
181
+ def delete_jsonapi!(path)
182
+ check_db_life_span()
183
+ $jsonapi.adapter.delete!(path)
184
+ end
185
+
186
+ def db_exec (query)
187
+ $pg.query(query: query)
188
+ end
189
+
190
+ def expand_mail_body (template)
191
+ if File.extname(template) == ''
192
+ template += '.erb'
193
+ end
194
+ if template[0] == '/'
195
+ erb_template = File.read(template)
196
+ else
197
+ erb_template = File.read(File.join(File.expand_path(File.dirname($PROGRAM_NAME)), template))
198
+ end
199
+ ERB.new(erb_template).result(binding)
200
+ end
201
+
202
+ def send_email (args)
203
+ if args.has_key?(:template)
204
+ email_body = expand_mail_body args[:template]
205
+ else
206
+ email_body = args[:body]
207
+ end
208
+
209
+ document = Roadie::Document.new email_body
210
+ email_body = document.transform
211
+
212
+ m = Mail.new do
213
+ from $config[:mail][:from]
214
+ to args[:to]
215
+ subject args[:subject]
216
+
217
+ html_part do
218
+ content_type 'text/html; charset=UTF-8'
219
+ body email_body
220
+ end
221
+ end
222
+
223
+ begin
224
+ m.deliver!
225
+ # ap m.to_s
226
+ return OpenStruct.new(status: true)
227
+ rescue Net::OpenTimeout => e
228
+ ap ["OpenTimeout", e]
229
+ return OpenStruct.new(status: false, message: e.message)
230
+ rescue Exception => e
231
+ ap e
232
+ return OpenStruct.new(status: false, message: e.message)
233
+ end
234
+
235
+ end
236
+
237
+ def database_connect
238
+ # any connection to close?
239
+ if ! $jsonapi.nil?
240
+ $jsonapi.close
241
+ $jsonapi = nil
242
+ end
243
+ if nil != $pg
244
+ $pg.disconnect()
245
+ $pg = nil
246
+ end
247
+ # establish new connection?
248
+ if $config[:postgres] && $config[:postgres][:conn_str]
249
+ $pg = ::SP::Job::PGConnection.new(owner: 'back_burner', config: $config[:postgres])
250
+ $pg.connect()
251
+ if $config[:options][:jsonapi] == true
252
+ $jsonapi = SP::Duh::JSONAPI::Service.new($pg.connection, ($jsonapi.nil? ? nil : $jsonapi.url))
253
+ end
254
+ end
255
+ end
256
+
257
+ def define_db_life_span_treshhold
258
+ min = $config[:postgres][:min_queries_per_conn]
259
+ max = $config[:postgres][:max_queries_per_conn]
260
+ if (!max.nil? && max > 0) || (!min.nil? && min > 0)
261
+ $db_life_span = 0
262
+ $check_db_life_span = true
263
+ new_min, new_max = [min, max].minmax
264
+ new_min = new_min if new_min <= 0
265
+ if new_min + new_min > 0
266
+ $db_treshold = (new_min + (new_min - new_min) * rand).to_i
267
+ else
268
+ $db_treshold = new_min.to_i
269
+ end
270
+ end
271
+ end
272
+
273
+ def check_db_life_span
274
+ return unless $check_db_life_span
275
+ $db_life_span += 1
276
+ if $db_life_span > $db_treshold
277
+ # Reset pg connection
278
+ database_connect()
279
+ end
280
+ end
281
+
282
+ def catch_fatal_exceptions (e)
283
+ case e
284
+ when PG::UnableToSend, PG::AdminShutdown, PG::ConnectionBad
285
+ logger.fatal "Lost connection to database exiting now"
286
+ exit
287
+ when Redis::CannotConnectError
288
+ logger.fatal "Can't connect to redis exiting now"
289
+ exit
290
+ end
291
+ end
292
+
293
+ end # Module Common
294
+ end # Module Job
295
+ end # Module SP
@@ -0,0 +1,34 @@
1
+ #
2
+ # Copyright (c) 2011-2016 Cloudware S.A. All rights reserved.
3
+ #
4
+ # This file is part of sp-job.
5
+ #
6
+ # sp-job is free software: you can redistribute it and/or modify
7
+ # it under the terms of the GNU Affero General Public License as published by
8
+ # the Free Software Foundation, either version 3 of the License, or
9
+ # (at your option) any later version.
10
+ #
11
+ # sp-job is distributed in the hope that it will be useful,
12
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
13
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
+ # GNU General Public License for more details.
15
+ #
16
+ # You should have received a copy of the GNU Affero General Public License
17
+ # along with sp-job. If not, see <http://www.gnu.org/licenses/>.
18
+ #
19
+ # encoding: utf-8
20
+ #
21
+
22
+ module SP
23
+ module Job
24
+ class Engine < ::Rails::Engine
25
+ isolate_namespace SP::Job
26
+
27
+ initializer :append_migrations do |app|
28
+ unless app.root.to_s.match root.to_s
29
+ app.config.paths["db/migrate"] += config.paths["db/migrate"].expanded
30
+ end
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,94 @@
1
+ #!/usr/bin/env ruby
2
+ #
3
+ # encoding: utf-8
4
+ #
5
+ # Copyright (c) 2017 Cloudware S.A. Allrights reserved
6
+ #
7
+ # Helper to obtain tokens to access toconline API's.
8
+ #
9
+
10
+ module SP
11
+ module Job
12
+
13
+ #
14
+ # Helper class to transform an error into json api format.
15
+ #
16
+ class JSONAPIError < StandardError
17
+
18
+ # {
19
+ # "errors": [
20
+ # {
21
+ # "status": nullptr,
22
+ # "code": nullptr,
23
+ # "detail": nullptr,
24
+ # "meta": {
25
+ # "internal-error": nullptr
26
+ # }
27
+ # }
28
+ # ]
29
+ # }
30
+
31
+ private
32
+
33
+ @error
34
+
35
+ public
36
+
37
+ def initialize (code: 500, detail: nil, internal: nil)
38
+ @errors = [
39
+ {
40
+ :code => code,
41
+ :detail => detail
42
+ }
43
+ ]
44
+ # 4xx
45
+ case code
46
+ when 400
47
+ @errors[0][:status] = "Bad Request"
48
+ when 401
49
+ @errors[0][:status] = "Unauthorized"
50
+ when 403
51
+ @errors[0][:status] = "Forbidden"
52
+ when 404
53
+ @errors[0][:status] = "Not Found"
54
+ when 405
55
+ @errors[0][:status] = "Method Not Allowed"
56
+ when 406
57
+ @errors[0][:status] = "Not Acceptable"
58
+ when 408
59
+ @errors[0][:status] = "Request Timeout"
60
+ # 5xx
61
+ when 501
62
+ @errors[0][:status] = "Not Implemented"
63
+ else
64
+ # other
65
+ @errors[0][:status] = "Internal Server Error"
66
+ end
67
+ @errors[0][:status] = "#{code} #{@errors[0][:status]}"
68
+ if nil != internal
69
+ @errors[0][:meta] = { :'internal-error' => internal }
70
+ end
71
+ end
72
+
73
+ def code ()
74
+ return @errors[0][:code]
75
+ end
76
+
77
+ def content_type ()
78
+ "application/vnd.api+json;charset=utf-8"
79
+ end
80
+
81
+ def body ()
82
+ {
83
+ :errors => @errors
84
+ }
85
+ end
86
+
87
+ def content_type_and_body ()
88
+ [ content_type(), body() ]
89
+ end
90
+
91
+ end
92
+
93
+ end # Job module
94
+ end # SP module
@@ -0,0 +1,179 @@
1
+ #
2
+ # Copyright (c) 2011-2016 Cloudware S.A. All rights reserved.
3
+ #
4
+ # This file is part of sp-job.
5
+ #
6
+ # sp-job is free software: you can redistribute it and/or modify
7
+ # it under the terms of the GNU Affero General Public License as published by
8
+ # the Free Software Foundation, either version 3 of the License, or
9
+ # (at your option) any later version.
10
+ #
11
+ # sp-job is distributed in the hope that it will be useful,
12
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
13
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
+ # GNU General Public License for more details.
15
+ #
16
+ # You should have received a copy of the GNU Affero General Public License
17
+ # along with sp-job. If not, see <http://www.gnu.org/licenses/>.
18
+ #
19
+ # encoding: utf-8
20
+ #
21
+
22
+ require 'pg'
23
+ require 'openssl'
24
+
25
+ module SP
26
+ module Job
27
+
28
+ class PGConnection
29
+
30
+ private
31
+
32
+ #
33
+ # Private Data
34
+ #
35
+ @owner = nil
36
+ @config = nil
37
+ @connection = nil
38
+ @treshold = -1
39
+ @counter = 0
40
+ @statements = []
41
+
42
+ public
43
+
44
+ #
45
+ # Public Attributes
46
+ #
47
+ attr_accessor :connection
48
+
49
+ #
50
+ # Prepare database connection configuration.
51
+ #
52
+ # @param owner
53
+ # @param config
54
+ #
55
+ def initialize (owner:, config:)
56
+ @owner = owner
57
+ @config = config
58
+ @connection = nil
59
+ @treshold = -1
60
+ @counter = 0
61
+ @statements = []
62
+ min = @config[:min_queries_per_conn]
63
+ max = @config[:max_queries_per_conn]
64
+ if (!max.nil? && max > 0) || (!min.nil? && min > 0)
65
+ @counter = 0
66
+ new_min, new_max = [min, max].minmax
67
+ new_min = new_min if new_min <= 0
68
+ if new_min + new_min > 0
69
+ @treshold = (new_min + (new_min - new_min) * rand).to_i
70
+ else
71
+ @treshold = new_min.to_i
72
+ end
73
+ end
74
+ end
75
+
76
+ #
77
+ # Establish a new database connection.
78
+ # Previous one ( if any ) will be closed first.
79
+ #
80
+ def connect ()
81
+ disconnect()
82
+ @connection = PG.connect(@config[:conn_str])
83
+ end
84
+
85
+ #
86
+ # Close currenly open database connection.
87
+ #
88
+ def disconnect ()
89
+ if @connection.nil?
90
+ return
91
+ end
92
+ while @statements.count > 0 do
93
+ @connection.exec("DEALLOCATE #{@statements.pop()}")
94
+ end
95
+ @connection.close
96
+ @connection = nil
97
+ @counter = 0
98
+ end
99
+
100
+ #
101
+ # Prepare an SQL statement.
102
+ #
103
+ # @param query
104
+ #
105
+ # @return Statement id.
106
+ #
107
+ def prepare_statement (query:)
108
+ if nil == @connection
109
+ connect()
110
+ end
111
+ id = "#{@owner}_#{Digest::MD5.hexdigest(query)}"
112
+ if @statements.include? id
113
+ return id
114
+ else
115
+ @statements << id
116
+ @connection.prepare(@statements.last, query)
117
+ return @statements.last
118
+ end
119
+ end
120
+
121
+ #
122
+ # Execute a previously prepared SQL statement.
123
+ #
124
+ # @param id
125
+ # @param args
126
+ #
127
+ # @return PG result
128
+ #
129
+ def execute_statement (id:, args:)
130
+ check_life_span()
131
+ @connection.exec_prepared(id, args)
132
+ end
133
+
134
+ #
135
+ # Destroy a previously prepared SQL statement.
136
+ #
137
+ # @param id
138
+ # @param args
139
+ #
140
+ def dealloc_statement (id:)
141
+ if nil == id
142
+ while @statements.count > 0 do
143
+ @connection.exec("DEALLOCATE #{@statements.pop()}")
144
+ end
145
+ else
146
+ @statements.delete!(id)
147
+ @connection.exec("DEALLOCATE #{id}")
148
+ end
149
+ end
150
+
151
+ #
152
+ # Execute a query,
153
+ #
154
+ # @param query
155
+ #
156
+ def query (query:)
157
+ unless query.nil?
158
+ check_life_span()
159
+ @connection.exec(query)
160
+ end
161
+ end
162
+
163
+ private
164
+
165
+ #
166
+ # Check connection life span
167
+ #
168
+ def check_life_span ()
169
+ return unless @treshold > 0
170
+ @counter += 1
171
+ if @counter > @treshold
172
+ connect()
173
+ end
174
+ end
175
+
176
+ end # end class 'PGConnection'
177
+
178
+ end # module 'Job'
179
+ end # module 'SP'