fanforce-plugin-worker 1.6.0.rc1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +22 -0
- data/.ruby-version +1 -0
- data/.yardopts +5 -0
- data/.yardopts.rb +1 -0
- data/Gemfile +12 -0
- data/README.md +4 -0
- data/Rakefile +22 -0
- data/fanforce-plugin-worker.gemspec +29 -0
- data/lib/fanforce/plugin_worker/errors.rb +200 -0
- data/lib/fanforce/plugin_worker/runner.rb +116 -0
- data/lib/fanforce/plugin_worker/utils.rb +13 -0
- data/lib/fanforce/plugin_worker/version.rb +5 -0
- data/lib/fanforce/plugin_worker/worker.rb +52 -0
- data/lib/fanforce/plugin_worker.rb +24 -0
- data/test/controllers/enqueue_test.rb +27 -0
- data/test/controllers/error_test.rb +32 -0
- data/test/test_helper.rb +23 -0
- metadata +176 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: aa556539bb5a1b75c95a9e5d41909054789f7545
|
4
|
+
data.tar.gz: 961e75393aa8c57fecfdb18da9190ed06a1f075e
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: f5c3440807534ef96d1ee683c60a61c214905c3d70fba61b02acd3b8fbdf1aa1b0c70300f6d8c6667aac59098cba752871fc4977b11d18a680e5c912ed61a6fe
|
7
|
+
data.tar.gz: 12e47a5f1d2fd563046b8750cec01272b02a55a13fed59e7860996f366bb2d03a3555aa8290d8210f7dbd00dd32b6c67c070289d60c5fef0df1be058468d8b8f
|
data/.gitignore
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
*.gem
|
2
|
+
*.rbc
|
3
|
+
.bundle
|
4
|
+
.config
|
5
|
+
coverage
|
6
|
+
InstalledFiles
|
7
|
+
lib/bundler/man
|
8
|
+
pkg
|
9
|
+
rdoc
|
10
|
+
spec/reports
|
11
|
+
test/tmp
|
12
|
+
test/version_tmp
|
13
|
+
tmp
|
14
|
+
|
15
|
+
# YARD artifacts
|
16
|
+
.yardoc
|
17
|
+
_yardoc
|
18
|
+
doc/
|
19
|
+
.DS_Store
|
20
|
+
.idea/
|
21
|
+
.idea/*
|
22
|
+
Gemfile.lock
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.0.0-p576
|
data/.yardopts
ADDED
data/.yardopts.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require 'redcarpet/compat'
|
data/Gemfile
ADDED
data/README.md
ADDED
data/Rakefile
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'bundler/gem_tasks'
|
2
|
+
require 'rake/testtask'
|
3
|
+
require 'fileutils'
|
4
|
+
include FileUtils
|
5
|
+
|
6
|
+
task :run_test do
|
7
|
+
Rake::Task['test'].execute
|
8
|
+
puts '---------------------------------------------------------------------'
|
9
|
+
puts 'TESTS PASSED... READY TO BUILD...'
|
10
|
+
puts '---------------------------------------------------------------------'
|
11
|
+
end
|
12
|
+
task :build => :run_test
|
13
|
+
task :default => :build
|
14
|
+
|
15
|
+
########################################################################
|
16
|
+
|
17
|
+
Rake::TestTask.new do |t|
|
18
|
+
t.libs.push 'lib'
|
19
|
+
t.libs.push 'test'
|
20
|
+
t.pattern = 'test/**/*_test.rb'
|
21
|
+
t.verbose = false
|
22
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'fanforce/plugin_worker/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |gem|
|
7
|
+
gem.name = 'fanforce-plugin-worker'
|
8
|
+
gem.version = Fanforce::PluginWorker::VERSION
|
9
|
+
gem.date = Time.now.utc.strftime('%Y-%m-%d')
|
10
|
+
gem.summary = %q{Worker library to help Fanforce plugins with background processing}
|
11
|
+
|
12
|
+
gem.authors = ['Caleb Clark']
|
13
|
+
gem.email = ['cclark@fanforce.com']
|
14
|
+
gem.homepage = 'http://github.com/fanforce/gem-plugin-worker'
|
15
|
+
|
16
|
+
gem.files = `git ls-files`.split($/)
|
17
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
18
|
+
gem.require_paths = ['lib']
|
19
|
+
|
20
|
+
gem.add_runtime_dependency 'iron_mq', '~> 5.0.1'
|
21
|
+
gem.add_runtime_dependency 'iron_cache', '~> 1.4.2'
|
22
|
+
gem.add_runtime_dependency 'uuid', '~> 2.3.7'
|
23
|
+
gem.add_runtime_dependency 'activesupport', '~> 4.1.6'
|
24
|
+
gem.add_runtime_dependency 'redis', '~> 3.1.0'
|
25
|
+
gem.add_runtime_dependency 'multi_json', '>= 1.7.2'
|
26
|
+
|
27
|
+
gem.add_runtime_dependency 'fanforce-base', '~> 1.6.0'
|
28
|
+
gem.add_runtime_dependency 'fanforce-api', '~> 1.6.0'
|
29
|
+
end
|
@@ -0,0 +1,200 @@
|
|
1
|
+
require 'redis'
|
2
|
+
require 'uuid'
|
3
|
+
|
4
|
+
class Fanforce::PluginWorker::Errors
|
5
|
+
include Fanforce::PluginWorker::Utils
|
6
|
+
|
7
|
+
@@redis = nil
|
8
|
+
|
9
|
+
def self.redis
|
10
|
+
@@redis ||= Redis.new(url: Fanforce::PluginWorker.redis_url_errorlog)
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.find(queue_id, error_id)
|
14
|
+
Fanforce::PluginWorker::Error.new(queue_id, error_id)
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.list(queue_id, from_num=0, to_num=100, reverse=false)
|
18
|
+
error_ids = redis.lrange("ERRORS:#{queue_id}", from_num, to_num)
|
19
|
+
error_ids.reverse! if reverse
|
20
|
+
Fanforce::PluginWorker::ErrorList.new(queue_id, error_ids)
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.list_summaries(queue_id, from_num=0, to_num=100, reverse=false)
|
24
|
+
error_ids = redis.lrange("ERRORS:#{queue_id}", from_num, to_num)
|
25
|
+
error_ids.reverse! if reverse
|
26
|
+
Fanforce::PluginWorker::ErrorList.new(queue_id, error_ids).summaries
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.delete_all(queue_id, error_ids)
|
30
|
+
Fanforce::PluginWorker::ErrorList.new(queue_id, error_ids).delete
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.retry_all(queue_id, error_ids)
|
34
|
+
Fanforce::PluginWorker::ErrorList.new(queue_id, error_ids).retry
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.add(queue_id, e, job_data, worker_env)
|
38
|
+
error_id = UUID.generate
|
39
|
+
error = {
|
40
|
+
error_id: error_id,
|
41
|
+
class: Fanforce::PluginWorker,
|
42
|
+
exception: e.class.name,
|
43
|
+
http_code: (e.code if e.respond_to?(:code)),
|
44
|
+
message: e.message,
|
45
|
+
backtrace: e.backtrace.to_json,
|
46
|
+
errored_at: Time.now,
|
47
|
+
queue_id: queue_id,
|
48
|
+
raw_json: job_data[:params].to_json,
|
49
|
+
env_vars: worker_env.to_json,
|
50
|
+
retries: job_data[:retries],
|
51
|
+
curl_command: (e.curl_command if e.respond_to?(:curl_command)),
|
52
|
+
iron_token: Fanforce::PluginWorker.iron_token,
|
53
|
+
iron_project_id: Fanforce::PluginWorker.iron_project_id
|
54
|
+
}
|
55
|
+
redis.multi do
|
56
|
+
redis.rpush("ERRORS:#{queue_id}", error_id)
|
57
|
+
redis.hmset("ERRORS:#{queue_id}:#{error_id}", error.flatten)
|
58
|
+
end
|
59
|
+
rescue => e
|
60
|
+
log.fatal '-----------------------------------------------------'
|
61
|
+
log.fatal 'PluginWorker ERRORED WHILE RECOVERING FROM JOB ERROR:'
|
62
|
+
log.fatal '-----------------------------------------------------'
|
63
|
+
error.each {|k,v| log.fatal "#{k}: #{v}" }
|
64
|
+
log.fatal '-----------------------------------------------------'
|
65
|
+
raise
|
66
|
+
end
|
67
|
+
|
68
|
+
end
|
69
|
+
|
70
|
+
###################################################################################################################
|
71
|
+
|
72
|
+
class Fanforce::PluginWorker::Error
|
73
|
+
|
74
|
+
def initialize(queue_id, error_id)
|
75
|
+
@queue_id = queue_id
|
76
|
+
@error_id = error_id
|
77
|
+
end
|
78
|
+
|
79
|
+
def delete
|
80
|
+
self.class.delete(@queue_id, @error_id)
|
81
|
+
return nil
|
82
|
+
end
|
83
|
+
|
84
|
+
def details
|
85
|
+
v = self.class.get_details(@queue_id, @error_id)
|
86
|
+
self.class.format_details(v)
|
87
|
+
end
|
88
|
+
|
89
|
+
def summary
|
90
|
+
v = self.class.get_summary(@queue_id, @error_id)
|
91
|
+
self.class.format_summary(v)
|
92
|
+
end
|
93
|
+
|
94
|
+
def retry
|
95
|
+
v = Error.get_all(@queue_id, @error_id)
|
96
|
+
Error.retry(@queue_id, v)
|
97
|
+
end
|
98
|
+
|
99
|
+
###################################################################################################################
|
100
|
+
|
101
|
+
SUMMARY_FIELDS = [:error_id, :exception, :message, :errored_at, :raw_json, :retries]
|
102
|
+
DETAIL_FIELDS = [:error_id, :exception, :http_code, :message, :backtrace, :errored_at, :raw_json, :env_vars, :retries, :curl_command]
|
103
|
+
|
104
|
+
def self.redis; Fanforce::PluginWorker::Errors.redis end
|
105
|
+
|
106
|
+
def self.delete(queue_id, error_id)
|
107
|
+
redis.lrem("ERRORS:#{queue_id}", 0, error_id)
|
108
|
+
redis.del("ERRORS:#{queue_id}:#{error_id}")
|
109
|
+
end
|
110
|
+
|
111
|
+
def self.get_summary(queue_id, error_id)
|
112
|
+
redis.hmget("ERRORS:#{queue_id}:#{error_id}", *SUMMARY_FIELDS)
|
113
|
+
end
|
114
|
+
|
115
|
+
def self.format_summary(summary)
|
116
|
+
format(Hash[SUMMARY_FIELDS.zip(summary)])
|
117
|
+
end
|
118
|
+
|
119
|
+
def self.format(error)
|
120
|
+
error.each do |k,v|
|
121
|
+
error[k] = case k
|
122
|
+
when :backtrace then MultiJson.load(v) rescue []
|
123
|
+
when :env_vars then MultiJson.load(v) rescue {}
|
124
|
+
when :retries then v.to_i
|
125
|
+
else v
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
def self.get_details(queue_id, error_id)
|
131
|
+
redis.hmget("ERRORS:#{queue_id}:#{error_id}", *DETAIL_FIELDS)
|
132
|
+
end
|
133
|
+
|
134
|
+
def self.format_details(details)
|
135
|
+
format(Hash[DETAIL_FIELDS.zip(details)])
|
136
|
+
end
|
137
|
+
|
138
|
+
def self.get_all(queue_id, error_id)
|
139
|
+
redis.hgetall("ERRORS:#{queue_id}:#{error_id}")
|
140
|
+
end
|
141
|
+
|
142
|
+
def self.retry(queue_id, raw_error)
|
143
|
+
error = format(raw_error.symbolize_keys)
|
144
|
+
payload = MultiJson.load(error[:raw_json], symbolize_keys: true) rescue {}
|
145
|
+
queue_id = queue_id.gsub(":#{ENV['DEVELOPER_EMAIL']}", '') if ENV['DEVELOPER_EMAIL']
|
146
|
+
Fanforce::PluginWorker.set_config(iron_token: error[:iron_token], iron_project_id: error[:iron_project_id])
|
147
|
+
Fanforce::PluginWorker.enqueue(queue_id, payload, :retries => error[:retries] + 1)
|
148
|
+
delete(queue_id, error[:error_id])
|
149
|
+
end
|
150
|
+
|
151
|
+
end
|
152
|
+
|
153
|
+
###################################################################################################################
|
154
|
+
|
155
|
+
class Fanforce::PluginWorker::ErrorList
|
156
|
+
Error = Fanforce::PluginWorker::Error
|
157
|
+
|
158
|
+
def initialize(queue_id, error_ids)
|
159
|
+
@queue_id = queue_id
|
160
|
+
@error_ids = error_ids
|
161
|
+
end
|
162
|
+
|
163
|
+
def redis; Fanforce::PluginWorker::Errors.redis end
|
164
|
+
|
165
|
+
def summaries
|
166
|
+
redis_responses = []
|
167
|
+
redis.multi do
|
168
|
+
@error_ids.each do |error_id|
|
169
|
+
redis_responses << Error.get_summary(@queue_id, error_id)
|
170
|
+
end
|
171
|
+
end
|
172
|
+
redis_responses.map do |redis_response|
|
173
|
+
Error.format_summary(redis_response.value)
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
def retry
|
178
|
+
redis_responses = []
|
179
|
+
redis.multi do
|
180
|
+
@error_ids.each do |error_id|
|
181
|
+
redis_responses << Error.get_all(@queue_id, error_id)
|
182
|
+
end
|
183
|
+
end
|
184
|
+
redis.multi do
|
185
|
+
redis_responses.map do |redis_response|
|
186
|
+
Error.retry(@queue_id, redis_response.value)
|
187
|
+
end
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
def delete
|
192
|
+
redis.multi do
|
193
|
+
@error_ids.each do |error_id|
|
194
|
+
Error.delete(@queue_id, error_id)
|
195
|
+
end
|
196
|
+
end
|
197
|
+
return nil
|
198
|
+
end
|
199
|
+
|
200
|
+
end
|
@@ -0,0 +1,116 @@
|
|
1
|
+
require 'iron_mq'
|
2
|
+
require 'fanforce/api'
|
3
|
+
require 'timeout'
|
4
|
+
|
5
|
+
class Fanforce::PluginWorker::Runner
|
6
|
+
include Fanforce::PluginWorker::Utils
|
7
|
+
|
8
|
+
MAX_EXECUTION_TIME = 3300
|
9
|
+
class Timeout < RuntimeError; end
|
10
|
+
|
11
|
+
def initialize(worker_data, min_execution_time=300, &code_block)
|
12
|
+
raise "min_execution_time was set to #{min_execution_time}, which is #{min_execution_time - MAX_EXECUTION_TIME} seconds too long" if min_execution_time > MAX_EXECUTION_TIME
|
13
|
+
log.debug '------------------------------------------------------------------------------------'
|
14
|
+
log.debug 'LOADING WORKER ENV'
|
15
|
+
|
16
|
+
@queue_id = worker_data['queue_id'] || (raise 'worker_data must contain queue_id')
|
17
|
+
@worker_env = worker_data['env_vars'] || {}
|
18
|
+
|
19
|
+
@min_execution_time = min_execution_time
|
20
|
+
@code_block = code_block
|
21
|
+
|
22
|
+
load_env_from_server
|
23
|
+
load_env_from_worker(@worker_env)
|
24
|
+
ENV.each {|k,v| log.debug "#{k} = #{v}" }
|
25
|
+
|
26
|
+
load_jobs
|
27
|
+
end
|
28
|
+
|
29
|
+
def load_jobs
|
30
|
+
log.debug '------------------------------------------------------------------------------------'
|
31
|
+
log.debug 'PROCESSING JOBS...'
|
32
|
+
log.debug '------------------------------------------------------------------------------------'
|
33
|
+
job_num = 0
|
34
|
+
job_data = nil
|
35
|
+
while job_has_enough_time_to_run and (job = Fanforce::PluginWorker.iron_mq.queue(iron_queue_id(@queue_id)).get(timeout: 3600)) do
|
36
|
+
log.debug "- JOB #{job_num+=1}: #{job.body}"
|
37
|
+
timeout(worker_time_remaining, Timeout) do
|
38
|
+
job_data = nil
|
39
|
+
job_data = MultiJson.load(job.body, symbolize_keys: true)
|
40
|
+
run_job(job, job_data, &@code_block)
|
41
|
+
end
|
42
|
+
delete_job
|
43
|
+
log.debug '------------------------------------------------------------------------------------'
|
44
|
+
end
|
45
|
+
delete_job
|
46
|
+
log.debug 'WINDING DOWN WORKER!'
|
47
|
+
|
48
|
+
rescue Exception => e
|
49
|
+
handle_job_loading_error(e, job, job_data)
|
50
|
+
end
|
51
|
+
|
52
|
+
def load_env_from_server
|
53
|
+
if File.exists?('.developmentenv.rb')
|
54
|
+
require '.developmentenv'
|
55
|
+
elsif File.exists?('.stagingenv.rb')
|
56
|
+
require '.stagingenv'
|
57
|
+
elsif File.exists?('.productionenv.rb')
|
58
|
+
require '.productionenv'
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def load_env_from_worker(vars)
|
63
|
+
vars.each {|k,v| ENV[k.to_s]=v }
|
64
|
+
end
|
65
|
+
|
66
|
+
def run_job(job, job_data, &code_block)
|
67
|
+
@current_job = job
|
68
|
+
@current_params = job_data[:params]
|
69
|
+
@current_retries = job_data[:retries]
|
70
|
+
|
71
|
+
code_block.call(job_data[:params].clone, retries: job_data[:retries], queue_id: @queue_id)
|
72
|
+
delete_job(job)
|
73
|
+
|
74
|
+
rescue Exception => e
|
75
|
+
handle_job_error(e, job, job_data)
|
76
|
+
end
|
77
|
+
|
78
|
+
def handle_job_loading_error(e, job, job_data)
|
79
|
+
raise($!, "#{$!}: THERE IS NO JOB", $!.backtrace) if job.nil?
|
80
|
+
|
81
|
+
delete_job(job)
|
82
|
+
log.debug 'REMOVED JOB FROM QUEUE, BUT COULD NOT SAVE TO ERROR CACHE...'
|
83
|
+
raise($!, "#{$!}: #{job_data.to_json}", $!.backtrace)
|
84
|
+
end
|
85
|
+
|
86
|
+
def handle_job_error(e, job, job_data)
|
87
|
+
raise($!, "#{$!}: THERE IS NO JOB", $!.backtrace) if job.nil?
|
88
|
+
|
89
|
+
delete_job(job)
|
90
|
+
require_relative 'errors'
|
91
|
+
log.debug 'REMOVED JOB FROM QUEUE, AND SAVING TO ERROR CACHE...'
|
92
|
+
Fanforce::PluginWorker::Errors.add(@queue_id, e, job_data, @worker_env)
|
93
|
+
end
|
94
|
+
|
95
|
+
def worker_time_remaining
|
96
|
+
time_since_load = Time.now - Fanforce::PluginWorker::LOADED_AT
|
97
|
+
MAX_EXECUTION_TIME - time_since_load
|
98
|
+
end
|
99
|
+
|
100
|
+
def job_has_enough_time_to_run
|
101
|
+
time_since_load = Time.now - Fanforce::PluginWorker::LOADED_AT
|
102
|
+
return false if time_since_load > MAX_EXECUTION_TIME
|
103
|
+
return false if worker_time_remaining < @min_execution_time
|
104
|
+
return true
|
105
|
+
end
|
106
|
+
|
107
|
+
def delete_job(job=nil)
|
108
|
+
return if job.nil? and @current_job.nil?
|
109
|
+
(job || @current_job).delete
|
110
|
+
rescue Exception => e
|
111
|
+
log.debug "Job could not be deleted: #{e.message}"
|
112
|
+
ensure
|
113
|
+
@current_job = nil
|
114
|
+
end
|
115
|
+
|
116
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module Fanforce::PluginWorker::Utils
|
2
|
+
extend self
|
3
|
+
def self.included(base) base.extend(self) end
|
4
|
+
|
5
|
+
def log
|
6
|
+
Fanforce::PluginWorker.log
|
7
|
+
end
|
8
|
+
|
9
|
+
def iron_queue_id(iron_queue_id)
|
10
|
+
(ENV['RACK_ENV'] == 'development' && ENV['DEVELOPER_EMAIL']) ? "#{iron_queue_id}:#{ENV['DEVELOPER_EMAIL']}" : iron_queue_id
|
11
|
+
end
|
12
|
+
|
13
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
class Fanforce::PluginWorker
|
2
|
+
|
3
|
+
LOADED_AT = Time.now
|
4
|
+
|
5
|
+
@@iron = {}
|
6
|
+
@@redis = {}
|
7
|
+
@@log = Logger.new($stdout)
|
8
|
+
@@log.datetime_format = '%H:%M:%S'
|
9
|
+
@@log.formatter = proc do |severity, datetime, prog_name, msg| "#{datetime}: #{msg}\n" end
|
10
|
+
|
11
|
+
def self.iron_token
|
12
|
+
@@iron[:token] || ENV['IRON_TOKEN']
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.iron_project_id
|
16
|
+
@@iron[:project_id] || ENV['IRON_PROJECT_ID']
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.iron_mq
|
20
|
+
require 'iron_mq'
|
21
|
+
@@iron[:mq] ||= IronMQ::Client.new(:token => @@iron[:token], :project_id => @@iron[:project_id])
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.redis_url_errorlog
|
25
|
+
@@redis[:url_errorlog] || ENV['REDIS_URL_ERRORLOG'] || (raise 'No REDIS_URL_ERRORLOG found in ENV')
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.set_config(obj)
|
29
|
+
@@iron[:token] = obj[:iron_token] if obj[:iron_token]
|
30
|
+
@@iron[:project_id] = obj[:iron_project_id] if obj[:iron_project_id]
|
31
|
+
@@redis[:url_errorlog] = obj[:redis_url_errorlog] if obj[:redis_url_errorlog]
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.log
|
35
|
+
@@log
|
36
|
+
end
|
37
|
+
|
38
|
+
##########################################################################################
|
39
|
+
|
40
|
+
def self.enqueue(queue_id, params, options={})
|
41
|
+
raise 'Params being sent to the queue must be a Hash' if !params.is_a?(Hash)
|
42
|
+
|
43
|
+
queue_id = Utils.iron_queue_id(queue_id)
|
44
|
+
retries = (options[:retries].present?) ? options.delete(:retries) : 0
|
45
|
+
iron_mq.queue(queue_id).post({params: params, retries: retries}.to_json, options)
|
46
|
+
end
|
47
|
+
|
48
|
+
def self.run(worker_data, min_execution_time=300, &code_block)
|
49
|
+
Runner.new(worker_data, min_execution_time, &code_block)
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'logger'
|
2
|
+
require 'multi_json'
|
3
|
+
require 'active_support/all'
|
4
|
+
|
5
|
+
require_relative 'plugin_worker/version'
|
6
|
+
require_relative 'plugin_worker/utils'
|
7
|
+
require_relative 'plugin_worker/worker'
|
8
|
+
require_relative 'plugin_worker/errors'
|
9
|
+
require_relative 'plugin_worker/runner'
|
10
|
+
|
11
|
+
########################################################################################################################
|
12
|
+
|
13
|
+
# The following is run if file is required from an IronWorker
|
14
|
+
if ENV['HOME'] == '/mnt/task'
|
15
|
+
require 'fanforce/plugin_factory/load_iron_worker'
|
16
|
+
Fanforce::Plugin.load(nil)
|
17
|
+
|
18
|
+
Plugin = Fanforce::Plugin
|
19
|
+
PluginWorker = Fanforce::PluginWorker
|
20
|
+
|
21
|
+
puts 'FINISHED LOADING PLUGIN'
|
22
|
+
else
|
23
|
+
puts "UNKNOWN CALLER: #{caller[0].split(':')[0]}"
|
24
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
describe Fanforce::PluginWorker do
|
4
|
+
|
5
|
+
before do
|
6
|
+
clean_dbs
|
7
|
+
end
|
8
|
+
|
9
|
+
it 'should enqueue a job' do
|
10
|
+
Fanforce::PluginWorker.enqueue('test', {name: 'caleb'})
|
11
|
+
assert Fanforce::PluginWorker.iron_mq.queue('test').size == 1
|
12
|
+
end
|
13
|
+
|
14
|
+
it 'should run an enqueued job' do
|
15
|
+
Fanforce::PluginWorker.enqueue('test', {name: 'caleb'})
|
16
|
+
processed_job = false
|
17
|
+
|
18
|
+
Fanforce::PluginWorker.run({'queue_id' => 'test'}) do |params|
|
19
|
+
processed_job = true
|
20
|
+
assert params[:name] == 'caleb'
|
21
|
+
end
|
22
|
+
|
23
|
+
assert processed_job == true
|
24
|
+
assert Fanforce::PluginWorker.iron_mq.queue('test').size == 0
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
describe Fanforce::PluginWorker::Errors do
|
4
|
+
|
5
|
+
before do
|
6
|
+
clean_dbs
|
7
|
+
end
|
8
|
+
|
9
|
+
it 'should save error to redis' do
|
10
|
+
Fanforce::PluginWorker.enqueue('test', {name: 'caleb'})
|
11
|
+
ran_job = false
|
12
|
+
|
13
|
+
Fanforce::PluginWorker.run({'queue_id' => 'test'}) do |params|
|
14
|
+
ran_job = true
|
15
|
+
raise 'test'
|
16
|
+
end
|
17
|
+
assert ran_job == true
|
18
|
+
assert Fanforce::PluginWorker::Errors.list_summaries('test').size == 1
|
19
|
+
assert Fanforce::PluginWorker.iron_mq.queue('test').size == 0
|
20
|
+
end
|
21
|
+
|
22
|
+
it 'should correctly retry error' do
|
23
|
+
Fanforce::PluginWorker.enqueue('test', {name: 'caleb'})
|
24
|
+
|
25
|
+
Fanforce::PluginWorker.run({'queue_id' => 'test'}) do |params|
|
26
|
+
raise 'test'
|
27
|
+
end
|
28
|
+
assert Fanforce::PluginWorker::Errors.list('test').retry
|
29
|
+
assert Fanforce::PluginWorker.iron_mq.queue('test').size == 1
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
data/test/test_helper.rb
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'minitest/autorun'
|
3
|
+
require 'minitest/spec'
|
4
|
+
require 'rack/test'
|
5
|
+
|
6
|
+
ENV['RACK_ENV'] = 'test'
|
7
|
+
|
8
|
+
require 'fanforce/plugin_worker'
|
9
|
+
|
10
|
+
# Clean databases before each test case
|
11
|
+
def clean_dbs
|
12
|
+
# ToDo: Find out why old token/project_id are giving 401 errors
|
13
|
+
Fanforce::PluginWorker.set_config(
|
14
|
+
iron_token: 'TF0YNz8na2H_reJ7_EoRnQAeKHM',
|
15
|
+
iron_project_id: '53725dfe0db3ef0005000001',
|
16
|
+
redis_url_errorlog: 'redis://localhost:6379/5'
|
17
|
+
)
|
18
|
+
Fanforce::PluginWorker.iron_mq.queues.all.each do |queue|
|
19
|
+
queue.delete_queue
|
20
|
+
end
|
21
|
+
Fanforce::PluginWorker::Errors.redis.flushdb
|
22
|
+
Fanforce::PluginWorker.log.level = Logger::FATAL
|
23
|
+
end
|
metadata
ADDED
@@ -0,0 +1,176 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fanforce-plugin-worker
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.6.0.rc1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Caleb Clark
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-02-11 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: iron_mq
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 5.0.1
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 5.0.1
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: iron_cache
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ~>
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 1.4.2
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ~>
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 1.4.2
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: uuid
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ~>
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 2.3.7
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ~>
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 2.3.7
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: activesupport
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ~>
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 4.1.6
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ~>
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 4.1.6
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: redis
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ~>
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: 3.1.0
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ~>
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: 3.1.0
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: multi_json
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - '>='
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: 1.7.2
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - '>='
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: 1.7.2
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: fanforce-base
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ~>
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: 1.6.0
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ~>
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: 1.6.0
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: fanforce-api
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ~>
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: 1.6.0
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ~>
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: 1.6.0
|
125
|
+
description:
|
126
|
+
email:
|
127
|
+
- cclark@fanforce.com
|
128
|
+
executables: []
|
129
|
+
extensions: []
|
130
|
+
extra_rdoc_files: []
|
131
|
+
files:
|
132
|
+
- .gitignore
|
133
|
+
- .ruby-version
|
134
|
+
- .yardopts
|
135
|
+
- .yardopts.rb
|
136
|
+
- Gemfile
|
137
|
+
- README.md
|
138
|
+
- Rakefile
|
139
|
+
- fanforce-plugin-worker.gemspec
|
140
|
+
- lib/fanforce/plugin_worker.rb
|
141
|
+
- lib/fanforce/plugin_worker/errors.rb
|
142
|
+
- lib/fanforce/plugin_worker/runner.rb
|
143
|
+
- lib/fanforce/plugin_worker/utils.rb
|
144
|
+
- lib/fanforce/plugin_worker/version.rb
|
145
|
+
- lib/fanforce/plugin_worker/worker.rb
|
146
|
+
- test/controllers/enqueue_test.rb
|
147
|
+
- test/controllers/error_test.rb
|
148
|
+
- test/test_helper.rb
|
149
|
+
homepage: http://github.com/fanforce/gem-plugin-worker
|
150
|
+
licenses: []
|
151
|
+
metadata: {}
|
152
|
+
post_install_message:
|
153
|
+
rdoc_options: []
|
154
|
+
require_paths:
|
155
|
+
- lib
|
156
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
157
|
+
requirements:
|
158
|
+
- - '>='
|
159
|
+
- !ruby/object:Gem::Version
|
160
|
+
version: '0'
|
161
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
162
|
+
requirements:
|
163
|
+
- - '>'
|
164
|
+
- !ruby/object:Gem::Version
|
165
|
+
version: 1.3.1
|
166
|
+
requirements: []
|
167
|
+
rubyforge_project:
|
168
|
+
rubygems_version: 2.0.14
|
169
|
+
signing_key:
|
170
|
+
specification_version: 4
|
171
|
+
summary: Worker library to help Fanforce plugins with background processing
|
172
|
+
test_files:
|
173
|
+
- test/controllers/enqueue_test.rb
|
174
|
+
- test/controllers/error_test.rb
|
175
|
+
- test/test_helper.rb
|
176
|
+
has_rdoc:
|