heimdall-worker 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.version +1 -0
- data/lib/heimdall/heimdall_base.rb +152 -0
- data/lib/heimdall/heimdall_database.rb +95 -0
- data/lib/heimdall/heimdall_model.rb +33 -0
- data/lib/heimdall/heimdall_model_schedule.rb +81 -0
- data/lib/heimdall/heimdall_model_task.rb +131 -0
- data/lib/heimdall/heimdall_proxy.rb +43 -0
- data/lib/heimdall/heimdall_que.rb +100 -0
- data/lib/heimdall/heimdall_start.rb +91 -0
- data/lib/heimdall/heimdall_web.rb +126 -0
- data/lib/heimdall/heimdall_worker.rb +49 -0
- data/lib/heimdall-worker.rb +28 -0
- metadata +54 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 0b7a93b686ebbac0443797bd3202095c6f081a00020c04db22b665ce486f84b5
|
4
|
+
data.tar.gz: 61cd38475818a4ab93b9f3d85b83373c67671f08701f7e92148ed1ba15815016
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 8a8f4a06751167445de726e0f2b4c6f7843b4878e6131499965b1718a09a143433d9eb9ac46b006a58101a9f84d1fedc160796ecc4a49a0d37c7317488df470f
|
7
|
+
data.tar.gz: 521be25ae29004026ecdbdce3effe77530110f9d4fce058be43037c7455e4a0f9a9b857606ac433f2d43aaa293ce390b8d3fb3ec2d8b0387fe2a6ed33a101933
|
data/.version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
@@ -0,0 +1,152 @@
|
|
1
|
+
module Heimdall
|
2
|
+
EVENT_HOOKS ||= {}
|
3
|
+
|
4
|
+
extend self
|
5
|
+
|
6
|
+
def run
|
7
|
+
start
|
8
|
+
run_runner
|
9
|
+
end
|
10
|
+
|
11
|
+
def run_runner
|
12
|
+
return if CONFIG['running']
|
13
|
+
|
14
|
+
CONFIG.running = true
|
15
|
+
|
16
|
+
CONFIG.active_jobs = ObjectSpace
|
17
|
+
.each_object(Class)
|
18
|
+
.select { |klass| klass < ::Heimdall::Worker }
|
19
|
+
|
20
|
+
log 'HEIMDALL started with %s jobs (%s)' % [CONFIG.active_jobs.length, CONFIG.active_jobs.join(', ')], true
|
21
|
+
|
22
|
+
# will run forever in background and get new jobs every CONFIG.sleep
|
23
|
+
Thread.new do
|
24
|
+
while CONFIG.running
|
25
|
+
while que.has_free? && get_next
|
26
|
+
# add all new jobs to a que with minimal sleep
|
27
|
+
sleep 0.01
|
28
|
+
end
|
29
|
+
|
30
|
+
sleep CONFIG.sleep_lookup
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
Schedule.run
|
35
|
+
end
|
36
|
+
|
37
|
+
def call env
|
38
|
+
::Heimdall::Web.call env
|
39
|
+
end
|
40
|
+
|
41
|
+
def que
|
42
|
+
CONFIG.que
|
43
|
+
end
|
44
|
+
|
45
|
+
def config
|
46
|
+
CONFIG
|
47
|
+
end
|
48
|
+
|
49
|
+
def stop
|
50
|
+
log 'HEIMDALL is stopping'
|
51
|
+
|
52
|
+
CONFIG.running = false
|
53
|
+
|
54
|
+
while !poll.done?
|
55
|
+
sleep 0.3
|
56
|
+
end
|
57
|
+
|
58
|
+
log 'HEIMDALL STOPED'
|
59
|
+
end
|
60
|
+
|
61
|
+
def log text, screen = false
|
62
|
+
return if CONFIG.silent
|
63
|
+
puts text if screen
|
64
|
+
CONFIG.logger.info text
|
65
|
+
end
|
66
|
+
|
67
|
+
def logger
|
68
|
+
CONFIG.logger
|
69
|
+
end
|
70
|
+
|
71
|
+
def tasks filter = {}, limit = 100
|
72
|
+
Task
|
73
|
+
.order(Sequel.lit('id desc'))
|
74
|
+
.limit(limit)
|
75
|
+
.all
|
76
|
+
end
|
77
|
+
|
78
|
+
def server
|
79
|
+
CONFIG[:server] || raise('Heimdall not started')
|
80
|
+
end
|
81
|
+
|
82
|
+
def db
|
83
|
+
CONFIG[:db]
|
84
|
+
end
|
85
|
+
|
86
|
+
# add job to job server
|
87
|
+
def add job_klass = nil, opts = {}
|
88
|
+
if block_given?
|
89
|
+
Thread.current[:_heimdall_batch] = [Time.now.to_f, Digest::SHA1.hexdigest(rand.to_s)[0, 10]].join('').sub('.', '')
|
90
|
+
|
91
|
+
begin
|
92
|
+
yield self
|
93
|
+
ensure
|
94
|
+
Thread.current[:_heimdall_batch] = nil
|
95
|
+
end
|
96
|
+
else
|
97
|
+
if Thread.current[:_heimdall_batch]
|
98
|
+
opts = opts.merge _batch: Thread.current[:_heimdall_batch]
|
99
|
+
end
|
100
|
+
|
101
|
+
if job_klass.class == Symbol
|
102
|
+
job_klass = "#{job_klass}_job".classify.constantize
|
103
|
+
end
|
104
|
+
|
105
|
+
server.add(job_klass, opts)
|
106
|
+
|
107
|
+
true
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def restart job_id
|
112
|
+
Task[job_id.to_i].update status_sid: 'a', remaining_runs: 1, scheduled_at: Time.at(1)
|
113
|
+
end
|
114
|
+
|
115
|
+
# run next job from job server
|
116
|
+
def get_next
|
117
|
+
if (task = server.pop)
|
118
|
+
que.add do
|
119
|
+
begin
|
120
|
+
job_class = task.job_class.new
|
121
|
+
|
122
|
+
task.logger 'STARTED'
|
123
|
+
|
124
|
+
Timeout::timeout job_class.cattr.timeout.to_i do
|
125
|
+
job_class.call task.opts.to_hwia
|
126
|
+
task.set_done job_class.log
|
127
|
+
end
|
128
|
+
rescue => error
|
129
|
+
log_data = ["#{error.class}: #{error.message}"]
|
130
|
+
|
131
|
+
if job_class.cattr.backtrace
|
132
|
+
log_data.push error.backtrace.map{|el| " #{el}"}.join($/)
|
133
|
+
log_data.push ''
|
134
|
+
end
|
135
|
+
|
136
|
+
task.set_fail log_data.join("\n")
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
# Heimdall.job_on :fail do |task, status| ...
|
143
|
+
def job_on name, &block
|
144
|
+
allowed = [:all, :done, :start, :error, :fail]
|
145
|
+
|
146
|
+
unless allowed.include?(name)
|
147
|
+
raise ArgumentError, 'Event kind %s does not exist' % name
|
148
|
+
end
|
149
|
+
|
150
|
+
EVENT_HOOKS[name] = block
|
151
|
+
end
|
152
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
# Talks to the database
|
2
|
+
|
3
|
+
module Heimdall
|
4
|
+
class Database
|
5
|
+
STATUS ||= {
|
6
|
+
a: 'Added',
|
7
|
+
r: 'Running',
|
8
|
+
d: 'Done',
|
9
|
+
f: 'Failed',
|
10
|
+
}.to_hwia
|
11
|
+
|
12
|
+
def db
|
13
|
+
CONFIG[:db]
|
14
|
+
end
|
15
|
+
|
16
|
+
def add job_class, opts = {}
|
17
|
+
opts = opts.dup
|
18
|
+
uid = nil
|
19
|
+
|
20
|
+
job_cattr = job_class.cattr
|
21
|
+
|
22
|
+
# job unique ID, unique accross all jobs. Genareate complicated SHA1 on many keys?
|
23
|
+
# used to define UID on jobs that last long
|
24
|
+
# example: PDF generation that last a long time. user hits page refresh multiple times, only one job is added
|
25
|
+
# [@uique_id_string, @time_to_be_valid_in_seconds or 0 for inf]
|
26
|
+
if (ttl_seconds = (opts.delete(:_unique_for) || job_cattr.unique_for))
|
27
|
+
unless ttl_seconds.is_numeric?
|
28
|
+
raise ArgumentError, '_ttl argument is not numneric'
|
29
|
+
end
|
30
|
+
|
31
|
+
uid = Digest::SHA1.hexdigest((job_class.to_s + opts.to_s).chars.sort.join)
|
32
|
+
|
33
|
+
if db[:tasks].where(uid: uid).where(Sequel.lit('created_at > ?', Time.now - ttl_seconds.to_i.seconds)).first
|
34
|
+
Heimdall.logger.info '%s skipped add by TTL check' % job_class
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
batch = opts.delete :_batch
|
39
|
+
retries = job_class.cattr.retries.to_i
|
40
|
+
retries = 1 if retries < 1
|
41
|
+
|
42
|
+
scheduled_at = opts.delete(:_scheduled_at).to_i
|
43
|
+
|
44
|
+
task = Task.create({
|
45
|
+
created_at: Time.now,
|
46
|
+
scheduled_at: Time.at(scheduled_at),
|
47
|
+
job: job_class.to_s,
|
48
|
+
opts: opts.to_json,
|
49
|
+
uid: uid,
|
50
|
+
batch: batch,
|
51
|
+
remaining_runs: retries,
|
52
|
+
total_runs: 0,
|
53
|
+
status_sid: 'a'
|
54
|
+
})
|
55
|
+
|
56
|
+
task.logger 'ADDED - %s' % opts.to_json
|
57
|
+
|
58
|
+
task
|
59
|
+
end
|
60
|
+
|
61
|
+
def pop
|
62
|
+
if (task = available_base.first)
|
63
|
+
task.total_runs += 1
|
64
|
+
task.update({
|
65
|
+
status_sid: 'r',
|
66
|
+
started_at: Time.now,
|
67
|
+
restart_at: Time.now + task.job_class.cattr.timeout.to_i.seconds,
|
68
|
+
finished_at: nil,
|
69
|
+
total_runs: task.total_runs
|
70
|
+
})
|
71
|
+
|
72
|
+
task
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def available_ids
|
77
|
+
available_base.select(:id).to_a.map{|el| el[:id] }
|
78
|
+
end
|
79
|
+
|
80
|
+
private
|
81
|
+
|
82
|
+
def available_base
|
83
|
+
Task
|
84
|
+
.order(Sequel.lit('RANDOM()'))
|
85
|
+
.xwhere(%{
|
86
|
+
remaining_runs>0
|
87
|
+
and (
|
88
|
+
(scheduled_at<:time and status_sid in :list)
|
89
|
+
or
|
90
|
+
(status_sid = 'r' and restart_at<:time)
|
91
|
+
)
|
92
|
+
}, time: Time.now, list: %w[a e])
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module Heimdall
|
2
|
+
PER_PAGE = 50
|
3
|
+
|
4
|
+
Model = Class.new(Sequel::Model(Heimdall::CONFIG[:db]))
|
5
|
+
|
6
|
+
def Model.first_or_new filter
|
7
|
+
object = where(filter).first || new(filter)
|
8
|
+
yield object if block_given? && !object.id
|
9
|
+
object
|
10
|
+
end
|
11
|
+
|
12
|
+
Model.dataset_module do
|
13
|
+
def page num, size = nil
|
14
|
+
num = num.to_i
|
15
|
+
num = 1 if !num || num < 1
|
16
|
+
|
17
|
+
size ||= PER_PAGE
|
18
|
+
self.limit(size).offset((num - 1) * size).all
|
19
|
+
end
|
20
|
+
|
21
|
+
def xwhere *args
|
22
|
+
self.where Sequel.lit *args
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
Model.class_eval do
|
27
|
+
def update opts
|
28
|
+
# do not call callbacks
|
29
|
+
this.update opts
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
@@ -0,0 +1,81 @@
|
|
1
|
+
# Class to handle scheduled and repeted jobs
|
2
|
+
|
3
|
+
# https://crontab.guru/every-1-minute
|
4
|
+
|
5
|
+
module Heimdall
|
6
|
+
class Schedule < Model
|
7
|
+
class << self
|
8
|
+
# sets schedules and fills in memory objects
|
9
|
+
def set_schedules
|
10
|
+
@active_schedules = []
|
11
|
+
|
12
|
+
jobs = CONFIG.active_jobs.select{|el| el.cattr.every || el.cattr.cron }
|
13
|
+
|
14
|
+
for job in Schedule.all
|
15
|
+
job.delete unless jobs.map(&:to_s).include?(job)
|
16
|
+
end
|
17
|
+
|
18
|
+
for job_class in jobs
|
19
|
+
if job_class.cattr.cron
|
20
|
+
CronParser.new(job_class.cattr.cron)
|
21
|
+
end
|
22
|
+
|
23
|
+
row = self.first_or_new name: job_class.to_s
|
24
|
+
row.every = job_class.cattr.every
|
25
|
+
row.cron = job_class.cattr.cron
|
26
|
+
row.next_run_at ||= Time.now - 1.day
|
27
|
+
row.save
|
28
|
+
@active_schedules.push row
|
29
|
+
end
|
30
|
+
|
31
|
+
run
|
32
|
+
end
|
33
|
+
|
34
|
+
# run schedules in a loop
|
35
|
+
def run
|
36
|
+
return if @running
|
37
|
+
@running = true
|
38
|
+
|
39
|
+
require 'parse-cron'
|
40
|
+
|
41
|
+
set_schedules
|
42
|
+
|
43
|
+
if CONFIG[:disable_cron]
|
44
|
+
puts 'Heimdall - CRON disabled in config'
|
45
|
+
else
|
46
|
+
Thread.new do
|
47
|
+
loop do
|
48
|
+
for schedule in @active_schedules
|
49
|
+
if schedule.next_run_at < Time.now
|
50
|
+
schedule.set_next_run
|
51
|
+
schedule.job.call
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
sleep 1
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
###
|
63
|
+
|
64
|
+
def job
|
65
|
+
name.constantize
|
66
|
+
end
|
67
|
+
|
68
|
+
def set_next_run
|
69
|
+
if self[:cron].present?
|
70
|
+
self[:next_run_at] = CronParser.new(cron).next(Time.now)
|
71
|
+
elsif self[:every].present?
|
72
|
+
self[:next_run_at] = Time.now + every.to_i.seconds
|
73
|
+
end
|
74
|
+
|
75
|
+
self[:last_run_at] = Time.now
|
76
|
+
|
77
|
+
save
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
@@ -0,0 +1,131 @@
|
|
1
|
+
module Heimdall
|
2
|
+
class Task < Model
|
3
|
+
def name
|
4
|
+
if job == 'HeimdallProxyWorker'
|
5
|
+
begin
|
6
|
+
parts = Marshal.load Base64.decode64 opts['b64']
|
7
|
+
'%s#%s (ProxyWorker)' % parts
|
8
|
+
rescue
|
9
|
+
'HeimdallProxyWorker (?)'
|
10
|
+
end
|
11
|
+
else
|
12
|
+
job
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def status
|
17
|
+
Heimdall::Database::STATUS[self[:status_sid]] || 'Unknown'
|
18
|
+
end
|
19
|
+
|
20
|
+
def job_class
|
21
|
+
self[:job].constantize
|
22
|
+
end
|
23
|
+
|
24
|
+
def has_error?
|
25
|
+
self[:status_sid] == 'e'
|
26
|
+
end
|
27
|
+
|
28
|
+
def opts
|
29
|
+
if self[:opts].class == Hash
|
30
|
+
self[:opts]
|
31
|
+
else
|
32
|
+
@opts ||= JSON.load self[:opts]
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def full_status
|
37
|
+
@colorize = false
|
38
|
+
full_text_status
|
39
|
+
end
|
40
|
+
|
41
|
+
def full_color_status
|
42
|
+
@colorize = true
|
43
|
+
full_text_status
|
44
|
+
end
|
45
|
+
|
46
|
+
def full_text_status
|
47
|
+
if status_sid == 'd'
|
48
|
+
colorize 'Finished', :green
|
49
|
+
elsif status_sid == 'e'
|
50
|
+
colorize "Failed after #{self[:total_runs]} tries", :red
|
51
|
+
elsif status_sid == 'a'
|
52
|
+
colorize 'In que', :gray
|
53
|
+
elsif status_sid == 'r'
|
54
|
+
diff = (Time.now - self[:started_at]).to_i
|
55
|
+
'Running for %s sec' % diff
|
56
|
+
else
|
57
|
+
'Unknow: %s' % status_sid
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def run_time
|
62
|
+
if self[:started_at] && self[:finished_at]
|
63
|
+
'%s sec' % (self[:finished_at] - self[:started_at]).to_i
|
64
|
+
else
|
65
|
+
'-'
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def next_run
|
70
|
+
if ['d', 'r'].include?(status_sid) || (status_sid == 'a' && remaining_runs == 0)
|
71
|
+
'-'
|
72
|
+
else
|
73
|
+
if scheduled_at > Time.now
|
74
|
+
Time.ago scheduled_at
|
75
|
+
else
|
76
|
+
'now'
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def set_done log_data
|
82
|
+
self.update status_sid: 'd', log: log_data, remaining_runs: 0, finished_at: Time.now
|
83
|
+
logger 'DONE'
|
84
|
+
end
|
85
|
+
|
86
|
+
def set_fail log_data
|
87
|
+
self.update status_sid: 'e',
|
88
|
+
log: log_data,
|
89
|
+
finished_at: Time.now,
|
90
|
+
scheduled_at: Time.now + Heimdall::CONFIG.repeat_after,
|
91
|
+
remaining_runs: remaining_runs - 1
|
92
|
+
|
93
|
+
logger :error
|
94
|
+
end
|
95
|
+
|
96
|
+
def logger msg
|
97
|
+
return if Heimdall::CONFIG.silent
|
98
|
+
|
99
|
+
status = msg.to_s.downcase.include?('error') ? :error : :info
|
100
|
+
message = "#{self.class}[#{self.id}] (#{self.job}) - #{msg}"
|
101
|
+
Heimdall.logger.send status, message
|
102
|
+
end
|
103
|
+
|
104
|
+
def export
|
105
|
+
data = to_h
|
106
|
+
data['log'] = data['log'].to_s.split($/)
|
107
|
+
data['info'] = {
|
108
|
+
name: name,
|
109
|
+
status: full_text_status,
|
110
|
+
run_time: run_time,
|
111
|
+
has_error: has_error?
|
112
|
+
}
|
113
|
+
data
|
114
|
+
end
|
115
|
+
|
116
|
+
def to_json
|
117
|
+
JSON.pretty_generate export
|
118
|
+
end
|
119
|
+
|
120
|
+
private
|
121
|
+
|
122
|
+
def colorize text, color
|
123
|
+
if @colorize
|
124
|
+
%[<span style="color: #{color};">#{text}</span>]
|
125
|
+
else
|
126
|
+
text
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# Provides proxy access to heimdall, for easy job offloading
|
2
|
+
|
3
|
+
# TODO: at and in -> @object.heimdall(at: Time.now + 5.hours).do_something
|
4
|
+
|
5
|
+
# makes @foo.bar(:baz)
|
6
|
+
# work via @foo.heimdall.bar(:baz)
|
7
|
+
# delayed
|
8
|
+
|
9
|
+
class Object
|
10
|
+
def heimdall
|
11
|
+
Heimdall::ObjectProxy.new self
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
module Heimdall
|
16
|
+
class ObjectProxy
|
17
|
+
def initialize object
|
18
|
+
@object = object
|
19
|
+
end
|
20
|
+
|
21
|
+
def method_missing name, *args
|
22
|
+
dump = Marshal.dump [@object, name, args]
|
23
|
+
dump = Base64.encode64 dump
|
24
|
+
dump = dump.sub %r{=*\s*$}, ''
|
25
|
+
Heimdall.add HeimdallProxyWorker, b64: dump
|
26
|
+
true
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
class HeimdallProxyWorker < Heimdall::Worker
|
32
|
+
retries Heimdall::CONFIG[:retries]
|
33
|
+
timeout 3.minutes
|
34
|
+
|
35
|
+
def call opts
|
36
|
+
object, name, args = Marshal.load Base64.decode64 opts[:b64]
|
37
|
+
response = object.send name, *args
|
38
|
+
|
39
|
+
if [Integer, Symbol, String].include?(response.class)
|
40
|
+
log(response.to_s).to_s[0, 1_000]
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,100 @@
|
|
1
|
+
# runs x parralel ques
|
2
|
+
|
3
|
+
module Heimdall
|
4
|
+
class Que
|
5
|
+
attr_reader :active, :size
|
6
|
+
|
7
|
+
def initialize size = 5
|
8
|
+
@size = size
|
9
|
+
@active = []
|
10
|
+
end
|
11
|
+
|
12
|
+
# add to block, it is possible to add
|
13
|
+
def add &block
|
14
|
+
in_ques do |i|
|
15
|
+
if is_free?(i)
|
16
|
+
@active[i] = Thread.new(&block)
|
17
|
+
return true
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
false
|
22
|
+
end
|
23
|
+
|
24
|
+
# does it have a free slot?
|
25
|
+
def has_free?
|
26
|
+
in_ques do |i|
|
27
|
+
return true if is_free?(i)
|
28
|
+
end
|
29
|
+
|
30
|
+
false
|
31
|
+
end
|
32
|
+
|
33
|
+
# are the all jobs done?
|
34
|
+
def done?
|
35
|
+
in_ques do |i|
|
36
|
+
return false unless is_free?(i)
|
37
|
+
end
|
38
|
+
|
39
|
+
true
|
40
|
+
end
|
41
|
+
|
42
|
+
def status_text i
|
43
|
+
if @active[i]
|
44
|
+
status = @active[i].status
|
45
|
+
if status.class == String
|
46
|
+
'Active (%s)' % status
|
47
|
+
else
|
48
|
+
'Free'
|
49
|
+
end
|
50
|
+
else
|
51
|
+
'Free'
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
private
|
56
|
+
|
57
|
+
# loop trough active ques
|
58
|
+
def in_ques
|
59
|
+
0.upto @size - 1 do |i|
|
60
|
+
yield i
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
# Thread statuses - "sleep", "run", "aborting", false, nil
|
65
|
+
def is_free? num
|
66
|
+
current = @active[num]
|
67
|
+
!current || ["aborting", false, nil].include?(current.status)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
# func = proc do
|
73
|
+
# num = rand*10
|
74
|
+
# sleep num
|
75
|
+
# puts 'num %s' % num
|
76
|
+
# end
|
77
|
+
|
78
|
+
# jobs = []
|
79
|
+
# 1.upto(10) do
|
80
|
+
# jobs.push func
|
81
|
+
# end
|
82
|
+
|
83
|
+
# que = Heimdall::Que.new
|
84
|
+
|
85
|
+
# while jobs.first
|
86
|
+
# if que.has_free?
|
87
|
+
# que.add &jobs.shift
|
88
|
+
# puts 'added'
|
89
|
+
# else
|
90
|
+
# puts 'waiting'
|
91
|
+
# sleep 1
|
92
|
+
# end
|
93
|
+
# end
|
94
|
+
|
95
|
+
# while !que.done?
|
96
|
+
# puts 'no still done'
|
97
|
+
# sleep 1
|
98
|
+
# end
|
99
|
+
|
100
|
+
# puts 'done'
|
@@ -0,0 +1,91 @@
|
|
1
|
+
module Heimdall
|
2
|
+
CONFIG ||= {}.to_hwia
|
3
|
+
|
4
|
+
extend self
|
5
|
+
|
6
|
+
def start
|
7
|
+
return if CONFIG['db']
|
8
|
+
|
9
|
+
CONFIG.db = Sequel.connect ENV.fetch('HEIMDALL_DB')
|
10
|
+
|
11
|
+
# init tables unless exist
|
12
|
+
unless CONFIG.db.tables.include?(:tasks)
|
13
|
+
CONFIG.db.create_table :tasks do
|
14
|
+
primary_key :id
|
15
|
+
Time :created_at
|
16
|
+
Time :scheduled_at
|
17
|
+
Time :started_at
|
18
|
+
Time :finished_at
|
19
|
+
Time :restart_at
|
20
|
+
Integer :total_runs
|
21
|
+
Integer :remaining_runs
|
22
|
+
String :batch
|
23
|
+
String :uid
|
24
|
+
String :job
|
25
|
+
String :opts
|
26
|
+
String :log
|
27
|
+
String :status_sid, limit: 1
|
28
|
+
end
|
29
|
+
|
30
|
+
CONFIG.db.add_index :tasks, :scheduled_at
|
31
|
+
CONFIG.db.add_index :tasks, :restart_at
|
32
|
+
CONFIG.db.add_index :tasks, :remaining_runs
|
33
|
+
CONFIG.db.add_index :tasks, :uid
|
34
|
+
CONFIG.db.add_index :tasks, :batch
|
35
|
+
CONFIG.db.add_index :tasks, :status_sid
|
36
|
+
end
|
37
|
+
|
38
|
+
unless CONFIG.db.tables.include?(:schedules)
|
39
|
+
CONFIG.db.create_table :schedules do
|
40
|
+
primary_key :id
|
41
|
+
String :name
|
42
|
+
String :every
|
43
|
+
String :cron
|
44
|
+
Time :next_run_at
|
45
|
+
Time :last_run_at
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
# allow custom locations
|
50
|
+
log_location = ENV['HEIMDALL_LOG'] || './log/heimdall.log'
|
51
|
+
log_location = STDOUT if log_location.upcase == 'STDOUT'
|
52
|
+
|
53
|
+
# log errors to screen and file
|
54
|
+
error_log_locations = [STDOUT]
|
55
|
+
|
56
|
+
if log_location != STDOUT
|
57
|
+
error_log_locations.push log_location.sub('.log', '_sql_errors.log')
|
58
|
+
end
|
59
|
+
|
60
|
+
for el in error_log_locations
|
61
|
+
logger = Logger.new el, 1, 1_240_000
|
62
|
+
logger.level = :error
|
63
|
+
CONFIG.db.loggers << logger
|
64
|
+
end
|
65
|
+
|
66
|
+
# models can be loaded only when we have DB connection (Sequel gem req)
|
67
|
+
require_relative 'heimdall_model'
|
68
|
+
require_relative 'heimdall_model_schedule'
|
69
|
+
require_relative 'heimdall_model_task'
|
70
|
+
|
71
|
+
unless CONFIG[:server]
|
72
|
+
# how many seconds to wait before getting new jobs
|
73
|
+
CONFIG[:sleep_lookup] ||= 1
|
74
|
+
|
75
|
+
# default number of retries
|
76
|
+
CONFIG[:retries] ||= 2
|
77
|
+
|
78
|
+
# repeat failed job after
|
79
|
+
CONFIG[:repeat_after] ||= 1.minute
|
80
|
+
|
81
|
+
CONFIG[:logger_file] ||= ENV['HEIMDALL_LOG'] || './log/heimdall.log'
|
82
|
+
CONFIG[:logger_file] = STDOUT if CONFIG[:logger_file].upcase == 'STDOUT'
|
83
|
+
CONFIG[:logger] ||= Logger.new(CONFIG.logger_file, 3, 10_240_000)
|
84
|
+
CONFIG[:silent] ||= false
|
85
|
+
CONFIG[:que_size] ||= 5
|
86
|
+
|
87
|
+
CONFIG.que = Heimdall::Que.new CONFIG.que_size
|
88
|
+
CONFIG.server = Heimdall::Database.new
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
@@ -0,0 +1,126 @@
|
|
1
|
+
require "bundler/setup"
|
2
|
+
|
3
|
+
|
4
|
+
module Heimdall
|
5
|
+
class Web < Sinatra::Base
|
6
|
+
if ENV['RACK_ENV'] == 'development'
|
7
|
+
require "sinatra/reloader"
|
8
|
+
register Sinatra::Reloader
|
9
|
+
end
|
10
|
+
|
11
|
+
helpers do
|
12
|
+
def info_buton name, count, param_name = nil, param_value = nil
|
13
|
+
klass = count > 0 ? 'bg-primary' : 'bg-success'
|
14
|
+
if param_value
|
15
|
+
%[<a class="badge #{klass}" href="?#{param_name}=#{param_value}">#{name} ⋅ #{count}</a>]
|
16
|
+
else
|
17
|
+
%[<span class="badge #{klass}">#{name} ⋅ #{count}</span>]
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def action name
|
22
|
+
path = request.path.split '/'
|
23
|
+
path[2] = name
|
24
|
+
path.join('/')
|
25
|
+
end
|
26
|
+
|
27
|
+
def paginate
|
28
|
+
page = (params[:page] || 1).to_i
|
29
|
+
has_prev = page > 1
|
30
|
+
has_next = @tasks.length > 49
|
31
|
+
|
32
|
+
HtmlTag.div class: 'btn-group' do |n|
|
33
|
+
n.a has_prev ? '<' : '⋅', class: 'btn btn-secondary', href: has_prev ? "/heimdall?page=#{page - 1}" : nil
|
34
|
+
n.button [page, @total].join(' / '), class: 'btn btn-secondary', onclick: "if (page = prompt('Jump to page:', #{page})){ Page.load('/heimdall?page='+page) }"
|
35
|
+
n.a has_next ? '>' : '-', class: 'btn btn-secondary', href: has_next ? "/heimdall?page=#{page + 1}" : nil
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
###
|
41
|
+
|
42
|
+
get '/' do
|
43
|
+
'Heimdall root. You probably want to go to <a href="/heimdall">/heimdall</a>'
|
44
|
+
end
|
45
|
+
|
46
|
+
get '/heimdall' do
|
47
|
+
@tasks = Heimdall::Task.order(Sequel.lit('id desc'))
|
48
|
+
@tasks = @tasks.where(status_sid: params[:s]) if params[:s]
|
49
|
+
@tasks = @tasks.xwhere('lower(job) like ?', "%#{params[:q].downcase}%") if params[:q]
|
50
|
+
# rr @tasks.sql
|
51
|
+
@tasks = @tasks.page(params[:page])
|
52
|
+
@total = (Heimdall::Task.count / Heimdall::PER_PAGE) + 1
|
53
|
+
|
54
|
+
base = Heimdall::Task.where(Sequel.lit('created_at>?', Time.now - 1.day))
|
55
|
+
@count_in_que = base.xwhere('remaining_runs>0').count
|
56
|
+
@count_running = base.where(status_sid: 'r').count
|
57
|
+
@count_done = base.where(status_sid: 'd').count
|
58
|
+
@count_failed = base.where(status_sid: 'e').count
|
59
|
+
|
60
|
+
@title = 'Tasks'
|
61
|
+
|
62
|
+
haml :tasks, layout: true
|
63
|
+
end
|
64
|
+
|
65
|
+
get '/heimdall/public/*' do
|
66
|
+
file = Pathname.new File.dirname(__FILE__) + '/public/%s' % params[:splat].first
|
67
|
+
|
68
|
+
if file.exist?
|
69
|
+
case params[:splat].first.split('.').last.to_sym
|
70
|
+
when :js
|
71
|
+
content_type :js
|
72
|
+
when :png
|
73
|
+
content_type :png
|
74
|
+
end
|
75
|
+
|
76
|
+
file.read
|
77
|
+
else
|
78
|
+
status 404
|
79
|
+
'File not found'
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
get '/heimdall/info/:id' do
|
84
|
+
task = Heimdall::Task[params[:id].to_i]
|
85
|
+
|
86
|
+
if task
|
87
|
+
JSON.pretty_generate task.export
|
88
|
+
else
|
89
|
+
status 404
|
90
|
+
{ error: 'not found' }.to_json
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
get '/heimdall/task/:id' do
|
95
|
+
@task = Heimdall::Task[params[:id].to_i]
|
96
|
+
|
97
|
+
if @task
|
98
|
+
@title = 'Task %s' % @task.id
|
99
|
+
haml :task, layout: true
|
100
|
+
else
|
101
|
+
status 404
|
102
|
+
haml 'Task not found', layout: true
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
post '/heimdall/restart/:id' do
|
107
|
+
Heimdall.restart params[:id].to_i
|
108
|
+
|
109
|
+
'restarted'
|
110
|
+
end
|
111
|
+
|
112
|
+
get '/heimdall/schedules' do
|
113
|
+
@schedules = Heimdall::Schedule.order(Sequel.lit('id desc'))
|
114
|
+
@title = 'Schedules'
|
115
|
+
|
116
|
+
haml :schedules, layout: true
|
117
|
+
end
|
118
|
+
|
119
|
+
get '/heimdall/ques' do
|
120
|
+
@que = Heimdall.que
|
121
|
+
@title = 'Ques'
|
122
|
+
|
123
|
+
haml :ques, layout: true
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
# class TestWorker < Heimdall::Worker
|
2
|
+
# timeout 10
|
3
|
+
#
|
4
|
+
# def call opts
|
5
|
+
# ...
|
6
|
+
# end
|
7
|
+
# end
|
8
|
+
#
|
9
|
+
# to add to que
|
10
|
+
#
|
11
|
+
# Heimdall.add :test, num: 1233
|
12
|
+
# or
|
13
|
+
# TestWorker.call num: 1233
|
14
|
+
|
15
|
+
module Heimdall
|
16
|
+
class Worker
|
17
|
+
cattr :retries, 3
|
18
|
+
cattr :retry_after, 60
|
19
|
+
cattr :timeout, nil
|
20
|
+
cattr :every, nil
|
21
|
+
cattr :cron, nil
|
22
|
+
cattr :backtrace, true
|
23
|
+
cattr :unique_for, nil
|
24
|
+
|
25
|
+
###
|
26
|
+
|
27
|
+
class << self
|
28
|
+
def call opts = {}
|
29
|
+
Heimdall.add self, opts.to_hwia
|
30
|
+
end
|
31
|
+
alias :perform :call
|
32
|
+
end
|
33
|
+
|
34
|
+
###
|
35
|
+
|
36
|
+
def initialize
|
37
|
+
@_log = []
|
38
|
+
end
|
39
|
+
|
40
|
+
def log text = nil
|
41
|
+
if text
|
42
|
+
@_log.push text
|
43
|
+
text
|
44
|
+
else
|
45
|
+
@_log.join $/
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# With start Heimdall you can add jobs
|
2
|
+
# Heimdall.start
|
3
|
+
|
4
|
+
# With run you can add and have running server that will process jobs
|
5
|
+
# Heimdall.run
|
6
|
+
|
7
|
+
# to stop, you should not need that
|
8
|
+
# Heimdall.stop
|
9
|
+
|
10
|
+
# Heimdall.add :test, num: 123
|
11
|
+
|
12
|
+
###
|
13
|
+
|
14
|
+
unless ENV['HEIMDALL_DB']
|
15
|
+
puts 'You are using Heimdall (https://....) but you have not defined HEIMDALL_DB'
|
16
|
+
puts 'Examples:'
|
17
|
+
puts 'HEIMDALL_DB=sqlite://./db/heimdall.sqlite'
|
18
|
+
puts 'HEIMDALL_DB=postgres://localhost:5432/heimdall'
|
19
|
+
exit
|
20
|
+
end
|
21
|
+
|
22
|
+
require_relative 'heimdall/heimdall_start'
|
23
|
+
require_relative 'heimdall/heimdall_base'
|
24
|
+
require_relative 'heimdall/heimdall_que'
|
25
|
+
require_relative 'heimdall/heimdall_database'
|
26
|
+
require_relative 'heimdall/heimdall_worker'
|
27
|
+
require_relative 'heimdall/heimdall_proxy'
|
28
|
+
require_relative 'heimdall/heimdall_web'
|
metadata
ADDED
@@ -0,0 +1,54 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: heimdall-worker
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Dino Reic
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2022-01-31 00:00:00.000000000 Z
|
12
|
+
dependencies: []
|
13
|
+
description: Sidekicq alternative
|
14
|
+
email: reic.dino@gmail.com
|
15
|
+
executables: []
|
16
|
+
extensions: []
|
17
|
+
extra_rdoc_files: []
|
18
|
+
files:
|
19
|
+
- "./.version"
|
20
|
+
- "./lib/heimdall-worker.rb"
|
21
|
+
- "./lib/heimdall/heimdall_base.rb"
|
22
|
+
- "./lib/heimdall/heimdall_database.rb"
|
23
|
+
- "./lib/heimdall/heimdall_model.rb"
|
24
|
+
- "./lib/heimdall/heimdall_model_schedule.rb"
|
25
|
+
- "./lib/heimdall/heimdall_model_task.rb"
|
26
|
+
- "./lib/heimdall/heimdall_proxy.rb"
|
27
|
+
- "./lib/heimdall/heimdall_que.rb"
|
28
|
+
- "./lib/heimdall/heimdall_start.rb"
|
29
|
+
- "./lib/heimdall/heimdall_web.rb"
|
30
|
+
- "./lib/heimdall/heimdall_worker.rb"
|
31
|
+
homepage: https://github.com/dux/heimdall-worker
|
32
|
+
licenses:
|
33
|
+
- MIT
|
34
|
+
metadata: {}
|
35
|
+
post_install_message:
|
36
|
+
rdoc_options: []
|
37
|
+
require_paths:
|
38
|
+
- lib
|
39
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '0'
|
44
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - ">="
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '0'
|
49
|
+
requirements: []
|
50
|
+
rubygems_version: 3.2.22
|
51
|
+
signing_key:
|
52
|
+
specification_version: 4
|
53
|
+
summary: Simple, efficient background processing for Ruby.
|
54
|
+
test_files: []
|