heimdall-worker 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.version +1 -0
- data/lib/heimdall/heimdall_base.rb +152 -0
- data/lib/heimdall/heimdall_database.rb +95 -0
- data/lib/heimdall/heimdall_model.rb +33 -0
- data/lib/heimdall/heimdall_model_schedule.rb +81 -0
- data/lib/heimdall/heimdall_model_task.rb +131 -0
- data/lib/heimdall/heimdall_proxy.rb +43 -0
- data/lib/heimdall/heimdall_que.rb +100 -0
- data/lib/heimdall/heimdall_start.rb +91 -0
- data/lib/heimdall/heimdall_web.rb +126 -0
- data/lib/heimdall/heimdall_worker.rb +49 -0
- data/lib/heimdall-worker.rb +28 -0
- metadata +54 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 0b7a93b686ebbac0443797bd3202095c6f081a00020c04db22b665ce486f84b5
|
4
|
+
data.tar.gz: 61cd38475818a4ab93b9f3d85b83373c67671f08701f7e92148ed1ba15815016
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 8a8f4a06751167445de726e0f2b4c6f7843b4878e6131499965b1718a09a143433d9eb9ac46b006a58101a9f84d1fedc160796ecc4a49a0d37c7317488df470f
|
7
|
+
data.tar.gz: 521be25ae29004026ecdbdce3effe77530110f9d4fce058be43037c7455e4a0f9a9b857606ac433f2d43aaa293ce390b8d3fb3ec2d8b0387fe2a6ed33a101933
|
data/.version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
@@ -0,0 +1,152 @@
|
|
1
|
+
module Heimdall
|
2
|
+
EVENT_HOOKS ||= {}
|
3
|
+
|
4
|
+
extend self
|
5
|
+
|
6
|
+
def run
|
7
|
+
start
|
8
|
+
run_runner
|
9
|
+
end
|
10
|
+
|
11
|
+
def run_runner
|
12
|
+
return if CONFIG['running']
|
13
|
+
|
14
|
+
CONFIG.running = true
|
15
|
+
|
16
|
+
CONFIG.active_jobs = ObjectSpace
|
17
|
+
.each_object(Class)
|
18
|
+
.select { |klass| klass < ::Heimdall::Worker }
|
19
|
+
|
20
|
+
log 'HEIMDALL started with %s jobs (%s)' % [CONFIG.active_jobs.length, CONFIG.active_jobs.join(', ')], true
|
21
|
+
|
22
|
+
# will run forever in background and get new jobs every CONFIG.sleep
|
23
|
+
Thread.new do
|
24
|
+
while CONFIG.running
|
25
|
+
while que.has_free? && get_next
|
26
|
+
# add all new jobs to a que with minimal sleep
|
27
|
+
sleep 0.01
|
28
|
+
end
|
29
|
+
|
30
|
+
sleep CONFIG.sleep_lookup
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
Schedule.run
|
35
|
+
end
|
36
|
+
|
37
|
+
def call env
|
38
|
+
::Heimdall::Web.call env
|
39
|
+
end
|
40
|
+
|
41
|
+
def que
|
42
|
+
CONFIG.que
|
43
|
+
end
|
44
|
+
|
45
|
+
def config
|
46
|
+
CONFIG
|
47
|
+
end
|
48
|
+
|
49
|
+
def stop
|
50
|
+
log 'HEIMDALL is stopping'
|
51
|
+
|
52
|
+
CONFIG.running = false
|
53
|
+
|
54
|
+
while !poll.done?
|
55
|
+
sleep 0.3
|
56
|
+
end
|
57
|
+
|
58
|
+
log 'HEIMDALL STOPED'
|
59
|
+
end
|
60
|
+
|
61
|
+
def log text, screen = false
|
62
|
+
return if CONFIG.silent
|
63
|
+
puts text if screen
|
64
|
+
CONFIG.logger.info text
|
65
|
+
end
|
66
|
+
|
67
|
+
def logger
|
68
|
+
CONFIG.logger
|
69
|
+
end
|
70
|
+
|
71
|
+
def tasks filter = {}, limit = 100
|
72
|
+
Task
|
73
|
+
.order(Sequel.lit('id desc'))
|
74
|
+
.limit(limit)
|
75
|
+
.all
|
76
|
+
end
|
77
|
+
|
78
|
+
def server
|
79
|
+
CONFIG[:server] || raise('Heimdall not started')
|
80
|
+
end
|
81
|
+
|
82
|
+
def db
|
83
|
+
CONFIG[:db]
|
84
|
+
end
|
85
|
+
|
86
|
+
# add job to job server
|
87
|
+
def add job_klass = nil, opts = {}
|
88
|
+
if block_given?
|
89
|
+
Thread.current[:_heimdall_batch] = [Time.now.to_f, Digest::SHA1.hexdigest(rand.to_s)[0, 10]].join('').sub('.', '')
|
90
|
+
|
91
|
+
begin
|
92
|
+
yield self
|
93
|
+
ensure
|
94
|
+
Thread.current[:_heimdall_batch] = nil
|
95
|
+
end
|
96
|
+
else
|
97
|
+
if Thread.current[:_heimdall_batch]
|
98
|
+
opts = opts.merge _batch: Thread.current[:_heimdall_batch]
|
99
|
+
end
|
100
|
+
|
101
|
+
if job_klass.class == Symbol
|
102
|
+
job_klass = "#{job_klass}_job".classify.constantize
|
103
|
+
end
|
104
|
+
|
105
|
+
server.add(job_klass, opts)
|
106
|
+
|
107
|
+
true
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def restart job_id
|
112
|
+
Task[job_id.to_i].update status_sid: 'a', remaining_runs: 1, scheduled_at: Time.at(1)
|
113
|
+
end
|
114
|
+
|
115
|
+
# run next job from job server
|
116
|
+
def get_next
|
117
|
+
if (task = server.pop)
|
118
|
+
que.add do
|
119
|
+
begin
|
120
|
+
job_class = task.job_class.new
|
121
|
+
|
122
|
+
task.logger 'STARTED'
|
123
|
+
|
124
|
+
Timeout::timeout job_class.cattr.timeout.to_i do
|
125
|
+
job_class.call task.opts.to_hwia
|
126
|
+
task.set_done job_class.log
|
127
|
+
end
|
128
|
+
rescue => error
|
129
|
+
log_data = ["#{error.class}: #{error.message}"]
|
130
|
+
|
131
|
+
if job_class.cattr.backtrace
|
132
|
+
log_data.push error.backtrace.map{|el| " #{el}"}.join($/)
|
133
|
+
log_data.push ''
|
134
|
+
end
|
135
|
+
|
136
|
+
task.set_fail log_data.join("\n")
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
# Heimdall.job_on :fail do |task, status| ...
|
143
|
+
def job_on name, &block
|
144
|
+
allowed = [:all, :done, :start, :error, :fail]
|
145
|
+
|
146
|
+
unless allowed.include?(name)
|
147
|
+
raise ArgumentError, 'Event kind %s does not exist' % name
|
148
|
+
end
|
149
|
+
|
150
|
+
EVENT_HOOKS[name] = block
|
151
|
+
end
|
152
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
# Talks to the database
|
2
|
+
|
3
|
+
module Heimdall
|
4
|
+
class Database
|
5
|
+
STATUS ||= {
|
6
|
+
a: 'Added',
|
7
|
+
r: 'Running',
|
8
|
+
d: 'Done',
|
9
|
+
f: 'Failed',
|
10
|
+
}.to_hwia
|
11
|
+
|
12
|
+
def db
|
13
|
+
CONFIG[:db]
|
14
|
+
end
|
15
|
+
|
16
|
+
def add job_class, opts = {}
|
17
|
+
opts = opts.dup
|
18
|
+
uid = nil
|
19
|
+
|
20
|
+
job_cattr = job_class.cattr
|
21
|
+
|
22
|
+
# job unique ID, unique accross all jobs. Genareate complicated SHA1 on many keys?
|
23
|
+
# used to define UID on jobs that last long
|
24
|
+
# example: PDF generation that last a long time. user hits page refresh multiple times, only one job is added
|
25
|
+
# [@uique_id_string, @time_to_be_valid_in_seconds or 0 for inf]
|
26
|
+
if (ttl_seconds = (opts.delete(:_unique_for) || job_cattr.unique_for))
|
27
|
+
unless ttl_seconds.is_numeric?
|
28
|
+
raise ArgumentError, '_ttl argument is not numneric'
|
29
|
+
end
|
30
|
+
|
31
|
+
uid = Digest::SHA1.hexdigest((job_class.to_s + opts.to_s).chars.sort.join)
|
32
|
+
|
33
|
+
if db[:tasks].where(uid: uid).where(Sequel.lit('created_at > ?', Time.now - ttl_seconds.to_i.seconds)).first
|
34
|
+
Heimdall.logger.info '%s skipped add by TTL check' % job_class
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
batch = opts.delete :_batch
|
39
|
+
retries = job_class.cattr.retries.to_i
|
40
|
+
retries = 1 if retries < 1
|
41
|
+
|
42
|
+
scheduled_at = opts.delete(:_scheduled_at).to_i
|
43
|
+
|
44
|
+
task = Task.create({
|
45
|
+
created_at: Time.now,
|
46
|
+
scheduled_at: Time.at(scheduled_at),
|
47
|
+
job: job_class.to_s,
|
48
|
+
opts: opts.to_json,
|
49
|
+
uid: uid,
|
50
|
+
batch: batch,
|
51
|
+
remaining_runs: retries,
|
52
|
+
total_runs: 0,
|
53
|
+
status_sid: 'a'
|
54
|
+
})
|
55
|
+
|
56
|
+
task.logger 'ADDED - %s' % opts.to_json
|
57
|
+
|
58
|
+
task
|
59
|
+
end
|
60
|
+
|
61
|
+
def pop
|
62
|
+
if (task = available_base.first)
|
63
|
+
task.total_runs += 1
|
64
|
+
task.update({
|
65
|
+
status_sid: 'r',
|
66
|
+
started_at: Time.now,
|
67
|
+
restart_at: Time.now + task.job_class.cattr.timeout.to_i.seconds,
|
68
|
+
finished_at: nil,
|
69
|
+
total_runs: task.total_runs
|
70
|
+
})
|
71
|
+
|
72
|
+
task
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def available_ids
|
77
|
+
available_base.select(:id).to_a.map{|el| el[:id] }
|
78
|
+
end
|
79
|
+
|
80
|
+
private
|
81
|
+
|
82
|
+
def available_base
|
83
|
+
Task
|
84
|
+
.order(Sequel.lit('RANDOM()'))
|
85
|
+
.xwhere(%{
|
86
|
+
remaining_runs>0
|
87
|
+
and (
|
88
|
+
(scheduled_at<:time and status_sid in :list)
|
89
|
+
or
|
90
|
+
(status_sid = 'r' and restart_at<:time)
|
91
|
+
)
|
92
|
+
}, time: Time.now, list: %w[a e])
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module Heimdall
|
2
|
+
PER_PAGE = 50
|
3
|
+
|
4
|
+
Model = Class.new(Sequel::Model(Heimdall::CONFIG[:db]))
|
5
|
+
|
6
|
+
def Model.first_or_new filter
|
7
|
+
object = where(filter).first || new(filter)
|
8
|
+
yield object if block_given? && !object.id
|
9
|
+
object
|
10
|
+
end
|
11
|
+
|
12
|
+
Model.dataset_module do
|
13
|
+
def page num, size = nil
|
14
|
+
num = num.to_i
|
15
|
+
num = 1 if !num || num < 1
|
16
|
+
|
17
|
+
size ||= PER_PAGE
|
18
|
+
self.limit(size).offset((num - 1) * size).all
|
19
|
+
end
|
20
|
+
|
21
|
+
def xwhere *args
|
22
|
+
self.where Sequel.lit *args
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
Model.class_eval do
|
27
|
+
def update opts
|
28
|
+
# do not call callbacks
|
29
|
+
this.update opts
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
@@ -0,0 +1,81 @@
|
|
1
|
+
# Class to handle scheduled and repeted jobs
|
2
|
+
|
3
|
+
# https://crontab.guru/every-1-minute
|
4
|
+
|
5
|
+
module Heimdall
|
6
|
+
class Schedule < Model
|
7
|
+
class << self
|
8
|
+
# sets schedules and fills in memory objects
|
9
|
+
def set_schedules
|
10
|
+
@active_schedules = []
|
11
|
+
|
12
|
+
jobs = CONFIG.active_jobs.select{|el| el.cattr.every || el.cattr.cron }
|
13
|
+
|
14
|
+
for job in Schedule.all
|
15
|
+
job.delete unless jobs.map(&:to_s).include?(job)
|
16
|
+
end
|
17
|
+
|
18
|
+
for job_class in jobs
|
19
|
+
if job_class.cattr.cron
|
20
|
+
CronParser.new(job_class.cattr.cron)
|
21
|
+
end
|
22
|
+
|
23
|
+
row = self.first_or_new name: job_class.to_s
|
24
|
+
row.every = job_class.cattr.every
|
25
|
+
row.cron = job_class.cattr.cron
|
26
|
+
row.next_run_at ||= Time.now - 1.day
|
27
|
+
row.save
|
28
|
+
@active_schedules.push row
|
29
|
+
end
|
30
|
+
|
31
|
+
run
|
32
|
+
end
|
33
|
+
|
34
|
+
# run schedules in a loop
|
35
|
+
def run
|
36
|
+
return if @running
|
37
|
+
@running = true
|
38
|
+
|
39
|
+
require 'parse-cron'
|
40
|
+
|
41
|
+
set_schedules
|
42
|
+
|
43
|
+
if CONFIG[:disable_cron]
|
44
|
+
puts 'Heimdall - CRON disabled in config'
|
45
|
+
else
|
46
|
+
Thread.new do
|
47
|
+
loop do
|
48
|
+
for schedule in @active_schedules
|
49
|
+
if schedule.next_run_at < Time.now
|
50
|
+
schedule.set_next_run
|
51
|
+
schedule.job.call
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
sleep 1
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
###
|
63
|
+
|
64
|
+
def job
|
65
|
+
name.constantize
|
66
|
+
end
|
67
|
+
|
68
|
+
def set_next_run
|
69
|
+
if self[:cron].present?
|
70
|
+
self[:next_run_at] = CronParser.new(cron).next(Time.now)
|
71
|
+
elsif self[:every].present?
|
72
|
+
self[:next_run_at] = Time.now + every.to_i.seconds
|
73
|
+
end
|
74
|
+
|
75
|
+
self[:last_run_at] = Time.now
|
76
|
+
|
77
|
+
save
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
@@ -0,0 +1,131 @@
|
|
1
|
+
module Heimdall
|
2
|
+
class Task < Model
|
3
|
+
def name
|
4
|
+
if job == 'HeimdallProxyWorker'
|
5
|
+
begin
|
6
|
+
parts = Marshal.load Base64.decode64 opts['b64']
|
7
|
+
'%s#%s (ProxyWorker)' % parts
|
8
|
+
rescue
|
9
|
+
'HeimdallProxyWorker (?)'
|
10
|
+
end
|
11
|
+
else
|
12
|
+
job
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def status
|
17
|
+
Heimdall::Database::STATUS[self[:status_sid]] || 'Unknown'
|
18
|
+
end
|
19
|
+
|
20
|
+
def job_class
|
21
|
+
self[:job].constantize
|
22
|
+
end
|
23
|
+
|
24
|
+
def has_error?
|
25
|
+
self[:status_sid] == 'e'
|
26
|
+
end
|
27
|
+
|
28
|
+
def opts
|
29
|
+
if self[:opts].class == Hash
|
30
|
+
self[:opts]
|
31
|
+
else
|
32
|
+
@opts ||= JSON.load self[:opts]
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def full_status
|
37
|
+
@colorize = false
|
38
|
+
full_text_status
|
39
|
+
end
|
40
|
+
|
41
|
+
def full_color_status
|
42
|
+
@colorize = true
|
43
|
+
full_text_status
|
44
|
+
end
|
45
|
+
|
46
|
+
def full_text_status
|
47
|
+
if status_sid == 'd'
|
48
|
+
colorize 'Finished', :green
|
49
|
+
elsif status_sid == 'e'
|
50
|
+
colorize "Failed after #{self[:total_runs]} tries", :red
|
51
|
+
elsif status_sid == 'a'
|
52
|
+
colorize 'In que', :gray
|
53
|
+
elsif status_sid == 'r'
|
54
|
+
diff = (Time.now - self[:started_at]).to_i
|
55
|
+
'Running for %s sec' % diff
|
56
|
+
else
|
57
|
+
'Unknow: %s' % status_sid
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def run_time
|
62
|
+
if self[:started_at] && self[:finished_at]
|
63
|
+
'%s sec' % (self[:finished_at] - self[:started_at]).to_i
|
64
|
+
else
|
65
|
+
'-'
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def next_run
|
70
|
+
if ['d', 'r'].include?(status_sid) || (status_sid == 'a' && remaining_runs == 0)
|
71
|
+
'-'
|
72
|
+
else
|
73
|
+
if scheduled_at > Time.now
|
74
|
+
Time.ago scheduled_at
|
75
|
+
else
|
76
|
+
'now'
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def set_done log_data
|
82
|
+
self.update status_sid: 'd', log: log_data, remaining_runs: 0, finished_at: Time.now
|
83
|
+
logger 'DONE'
|
84
|
+
end
|
85
|
+
|
86
|
+
def set_fail log_data
|
87
|
+
self.update status_sid: 'e',
|
88
|
+
log: log_data,
|
89
|
+
finished_at: Time.now,
|
90
|
+
scheduled_at: Time.now + Heimdall::CONFIG.repeat_after,
|
91
|
+
remaining_runs: remaining_runs - 1
|
92
|
+
|
93
|
+
logger :error
|
94
|
+
end
|
95
|
+
|
96
|
+
def logger msg
|
97
|
+
return if Heimdall::CONFIG.silent
|
98
|
+
|
99
|
+
status = msg.to_s.downcase.include?('error') ? :error : :info
|
100
|
+
message = "#{self.class}[#{self.id}] (#{self.job}) - #{msg}"
|
101
|
+
Heimdall.logger.send status, message
|
102
|
+
end
|
103
|
+
|
104
|
+
def export
|
105
|
+
data = to_h
|
106
|
+
data['log'] = data['log'].to_s.split($/)
|
107
|
+
data['info'] = {
|
108
|
+
name: name,
|
109
|
+
status: full_text_status,
|
110
|
+
run_time: run_time,
|
111
|
+
has_error: has_error?
|
112
|
+
}
|
113
|
+
data
|
114
|
+
end
|
115
|
+
|
116
|
+
def to_json
|
117
|
+
JSON.pretty_generate export
|
118
|
+
end
|
119
|
+
|
120
|
+
private
|
121
|
+
|
122
|
+
def colorize text, color
|
123
|
+
if @colorize
|
124
|
+
%[<span style="color: #{color};">#{text}</span>]
|
125
|
+
else
|
126
|
+
text
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# Provides proxy access to heimdall, for easy job offloading
|
2
|
+
|
3
|
+
# TODO: at and in -> @object.heimdall(at: Time.now + 5.hours).do_something
|
4
|
+
|
5
|
+
# makes @foo.bar(:baz)
|
6
|
+
# work via @foo.heimdall.bar(:baz)
|
7
|
+
# delayed
|
8
|
+
|
9
|
+
class Object
|
10
|
+
def heimdall
|
11
|
+
Heimdall::ObjectProxy.new self
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
module Heimdall
|
16
|
+
class ObjectProxy
|
17
|
+
def initialize object
|
18
|
+
@object = object
|
19
|
+
end
|
20
|
+
|
21
|
+
def method_missing name, *args
|
22
|
+
dump = Marshal.dump [@object, name, args]
|
23
|
+
dump = Base64.encode64 dump
|
24
|
+
dump = dump.sub %r{=*\s*$}, ''
|
25
|
+
Heimdall.add HeimdallProxyWorker, b64: dump
|
26
|
+
true
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
class HeimdallProxyWorker < Heimdall::Worker
|
32
|
+
retries Heimdall::CONFIG[:retries]
|
33
|
+
timeout 3.minutes
|
34
|
+
|
35
|
+
def call opts
|
36
|
+
object, name, args = Marshal.load Base64.decode64 opts[:b64]
|
37
|
+
response = object.send name, *args
|
38
|
+
|
39
|
+
if [Integer, Symbol, String].include?(response.class)
|
40
|
+
log(response.to_s).to_s[0, 1_000]
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,100 @@
|
|
1
|
+
# runs x parralel ques
|
2
|
+
|
3
|
+
module Heimdall
|
4
|
+
class Que
|
5
|
+
attr_reader :active, :size
|
6
|
+
|
7
|
+
def initialize size = 5
|
8
|
+
@size = size
|
9
|
+
@active = []
|
10
|
+
end
|
11
|
+
|
12
|
+
# add to block, it is possible to add
|
13
|
+
def add &block
|
14
|
+
in_ques do |i|
|
15
|
+
if is_free?(i)
|
16
|
+
@active[i] = Thread.new(&block)
|
17
|
+
return true
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
false
|
22
|
+
end
|
23
|
+
|
24
|
+
# does it have a free slot?
|
25
|
+
def has_free?
|
26
|
+
in_ques do |i|
|
27
|
+
return true if is_free?(i)
|
28
|
+
end
|
29
|
+
|
30
|
+
false
|
31
|
+
end
|
32
|
+
|
33
|
+
# are the all jobs done?
|
34
|
+
def done?
|
35
|
+
in_ques do |i|
|
36
|
+
return false unless is_free?(i)
|
37
|
+
end
|
38
|
+
|
39
|
+
true
|
40
|
+
end
|
41
|
+
|
42
|
+
def status_text i
|
43
|
+
if @active[i]
|
44
|
+
status = @active[i].status
|
45
|
+
if status.class == String
|
46
|
+
'Active (%s)' % status
|
47
|
+
else
|
48
|
+
'Free'
|
49
|
+
end
|
50
|
+
else
|
51
|
+
'Free'
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
private
|
56
|
+
|
57
|
+
# loop trough active ques
|
58
|
+
def in_ques
|
59
|
+
0.upto @size - 1 do |i|
|
60
|
+
yield i
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
# Thread statuses - "sleep", "run", "aborting", false, nil
|
65
|
+
def is_free? num
|
66
|
+
current = @active[num]
|
67
|
+
!current || ["aborting", false, nil].include?(current.status)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
# func = proc do
|
73
|
+
# num = rand*10
|
74
|
+
# sleep num
|
75
|
+
# puts 'num %s' % num
|
76
|
+
# end
|
77
|
+
|
78
|
+
# jobs = []
|
79
|
+
# 1.upto(10) do
|
80
|
+
# jobs.push func
|
81
|
+
# end
|
82
|
+
|
83
|
+
# que = Heimdall::Que.new
|
84
|
+
|
85
|
+
# while jobs.first
|
86
|
+
# if que.has_free?
|
87
|
+
# que.add &jobs.shift
|
88
|
+
# puts 'added'
|
89
|
+
# else
|
90
|
+
# puts 'waiting'
|
91
|
+
# sleep 1
|
92
|
+
# end
|
93
|
+
# end
|
94
|
+
|
95
|
+
# while !que.done?
|
96
|
+
# puts 'no still done'
|
97
|
+
# sleep 1
|
98
|
+
# end
|
99
|
+
|
100
|
+
# puts 'done'
|
@@ -0,0 +1,91 @@
|
|
1
|
+
module Heimdall
|
2
|
+
CONFIG ||= {}.to_hwia
|
3
|
+
|
4
|
+
extend self
|
5
|
+
|
6
|
+
def start
|
7
|
+
return if CONFIG['db']
|
8
|
+
|
9
|
+
CONFIG.db = Sequel.connect ENV.fetch('HEIMDALL_DB')
|
10
|
+
|
11
|
+
# init tables unless exist
|
12
|
+
unless CONFIG.db.tables.include?(:tasks)
|
13
|
+
CONFIG.db.create_table :tasks do
|
14
|
+
primary_key :id
|
15
|
+
Time :created_at
|
16
|
+
Time :scheduled_at
|
17
|
+
Time :started_at
|
18
|
+
Time :finished_at
|
19
|
+
Time :restart_at
|
20
|
+
Integer :total_runs
|
21
|
+
Integer :remaining_runs
|
22
|
+
String :batch
|
23
|
+
String :uid
|
24
|
+
String :job
|
25
|
+
String :opts
|
26
|
+
String :log
|
27
|
+
String :status_sid, limit: 1
|
28
|
+
end
|
29
|
+
|
30
|
+
CONFIG.db.add_index :tasks, :scheduled_at
|
31
|
+
CONFIG.db.add_index :tasks, :restart_at
|
32
|
+
CONFIG.db.add_index :tasks, :remaining_runs
|
33
|
+
CONFIG.db.add_index :tasks, :uid
|
34
|
+
CONFIG.db.add_index :tasks, :batch
|
35
|
+
CONFIG.db.add_index :tasks, :status_sid
|
36
|
+
end
|
37
|
+
|
38
|
+
unless CONFIG.db.tables.include?(:schedules)
|
39
|
+
CONFIG.db.create_table :schedules do
|
40
|
+
primary_key :id
|
41
|
+
String :name
|
42
|
+
String :every
|
43
|
+
String :cron
|
44
|
+
Time :next_run_at
|
45
|
+
Time :last_run_at
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
# allow custom locations
|
50
|
+
log_location = ENV['HEIMDALL_LOG'] || './log/heimdall.log'
|
51
|
+
log_location = STDOUT if log_location.upcase == 'STDOUT'
|
52
|
+
|
53
|
+
# log errors to screen and file
|
54
|
+
error_log_locations = [STDOUT]
|
55
|
+
|
56
|
+
if log_location != STDOUT
|
57
|
+
error_log_locations.push log_location.sub('.log', '_sql_errors.log')
|
58
|
+
end
|
59
|
+
|
60
|
+
for el in error_log_locations
|
61
|
+
logger = Logger.new el, 1, 1_240_000
|
62
|
+
logger.level = :error
|
63
|
+
CONFIG.db.loggers << logger
|
64
|
+
end
|
65
|
+
|
66
|
+
# models can be loaded only when we have DB connection (Sequel gem req)
|
67
|
+
require_relative 'heimdall_model'
|
68
|
+
require_relative 'heimdall_model_schedule'
|
69
|
+
require_relative 'heimdall_model_task'
|
70
|
+
|
71
|
+
unless CONFIG[:server]
|
72
|
+
# how many seconds to wait before getting new jobs
|
73
|
+
CONFIG[:sleep_lookup] ||= 1
|
74
|
+
|
75
|
+
# default number of retries
|
76
|
+
CONFIG[:retries] ||= 2
|
77
|
+
|
78
|
+
# repeat failed job after
|
79
|
+
CONFIG[:repeat_after] ||= 1.minute
|
80
|
+
|
81
|
+
CONFIG[:logger_file] ||= ENV['HEIMDALL_LOG'] || './log/heimdall.log'
|
82
|
+
CONFIG[:logger_file] = STDOUT if CONFIG[:logger_file].upcase == 'STDOUT'
|
83
|
+
CONFIG[:logger] ||= Logger.new(CONFIG.logger_file, 3, 10_240_000)
|
84
|
+
CONFIG[:silent] ||= false
|
85
|
+
CONFIG[:que_size] ||= 5
|
86
|
+
|
87
|
+
CONFIG.que = Heimdall::Que.new CONFIG.que_size
|
88
|
+
CONFIG.server = Heimdall::Database.new
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
@@ -0,0 +1,126 @@
|
|
1
|
+
require "bundler/setup"
|
2
|
+
|
3
|
+
|
4
|
+
module Heimdall
|
5
|
+
class Web < Sinatra::Base
|
6
|
+
if ENV['RACK_ENV'] == 'development'
|
7
|
+
require "sinatra/reloader"
|
8
|
+
register Sinatra::Reloader
|
9
|
+
end
|
10
|
+
|
11
|
+
helpers do
|
12
|
+
def info_buton name, count, param_name = nil, param_value = nil
|
13
|
+
klass = count > 0 ? 'bg-primary' : 'bg-success'
|
14
|
+
if param_value
|
15
|
+
%[<a class="badge #{klass}" href="?#{param_name}=#{param_value}">#{name} ⋅ #{count}</a>]
|
16
|
+
else
|
17
|
+
%[<span class="badge #{klass}">#{name} ⋅ #{count}</span>]
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def action name
|
22
|
+
path = request.path.split '/'
|
23
|
+
path[2] = name
|
24
|
+
path.join('/')
|
25
|
+
end
|
26
|
+
|
27
|
+
def paginate
|
28
|
+
page = (params[:page] || 1).to_i
|
29
|
+
has_prev = page > 1
|
30
|
+
has_next = @tasks.length > 49
|
31
|
+
|
32
|
+
HtmlTag.div class: 'btn-group' do |n|
|
33
|
+
n.a has_prev ? '<' : '⋅', class: 'btn btn-secondary', href: has_prev ? "/heimdall?page=#{page - 1}" : nil
|
34
|
+
n.button [page, @total].join(' / '), class: 'btn btn-secondary', onclick: "if (page = prompt('Jump to page:', #{page})){ Page.load('/heimdall?page='+page) }"
|
35
|
+
n.a has_next ? '>' : '-', class: 'btn btn-secondary', href: has_next ? "/heimdall?page=#{page + 1}" : nil
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
###
|
41
|
+
|
42
|
+
get '/' do
|
43
|
+
'Heimdall root. You probably want to go to <a href="/heimdall">/heimdall</a>'
|
44
|
+
end
|
45
|
+
|
46
|
+
get '/heimdall' do
|
47
|
+
@tasks = Heimdall::Task.order(Sequel.lit('id desc'))
|
48
|
+
@tasks = @tasks.where(status_sid: params[:s]) if params[:s]
|
49
|
+
@tasks = @tasks.xwhere('lower(job) like ?', "%#{params[:q].downcase}%") if params[:q]
|
50
|
+
# rr @tasks.sql
|
51
|
+
@tasks = @tasks.page(params[:page])
|
52
|
+
@total = (Heimdall::Task.count / Heimdall::PER_PAGE) + 1
|
53
|
+
|
54
|
+
base = Heimdall::Task.where(Sequel.lit('created_at>?', Time.now - 1.day))
|
55
|
+
@count_in_que = base.xwhere('remaining_runs>0').count
|
56
|
+
@count_running = base.where(status_sid: 'r').count
|
57
|
+
@count_done = base.where(status_sid: 'd').count
|
58
|
+
@count_failed = base.where(status_sid: 'e').count
|
59
|
+
|
60
|
+
@title = 'Tasks'
|
61
|
+
|
62
|
+
haml :tasks, layout: true
|
63
|
+
end
|
64
|
+
|
65
|
+
get '/heimdall/public/*' do
|
66
|
+
file = Pathname.new File.dirname(__FILE__) + '/public/%s' % params[:splat].first
|
67
|
+
|
68
|
+
if file.exist?
|
69
|
+
case params[:splat].first.split('.').last.to_sym
|
70
|
+
when :js
|
71
|
+
content_type :js
|
72
|
+
when :png
|
73
|
+
content_type :png
|
74
|
+
end
|
75
|
+
|
76
|
+
file.read
|
77
|
+
else
|
78
|
+
status 404
|
79
|
+
'File not found'
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
get '/heimdall/info/:id' do
|
84
|
+
task = Heimdall::Task[params[:id].to_i]
|
85
|
+
|
86
|
+
if task
|
87
|
+
JSON.pretty_generate task.export
|
88
|
+
else
|
89
|
+
status 404
|
90
|
+
{ error: 'not found' }.to_json
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
get '/heimdall/task/:id' do
|
95
|
+
@task = Heimdall::Task[params[:id].to_i]
|
96
|
+
|
97
|
+
if @task
|
98
|
+
@title = 'Task %s' % @task.id
|
99
|
+
haml :task, layout: true
|
100
|
+
else
|
101
|
+
status 404
|
102
|
+
haml 'Task not found', layout: true
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
post '/heimdall/restart/:id' do
|
107
|
+
Heimdall.restart params[:id].to_i
|
108
|
+
|
109
|
+
'restarted'
|
110
|
+
end
|
111
|
+
|
112
|
+
get '/heimdall/schedules' do
|
113
|
+
@schedules = Heimdall::Schedule.order(Sequel.lit('id desc'))
|
114
|
+
@title = 'Schedules'
|
115
|
+
|
116
|
+
haml :schedules, layout: true
|
117
|
+
end
|
118
|
+
|
119
|
+
get '/heimdall/ques' do
|
120
|
+
@que = Heimdall.que
|
121
|
+
@title = 'Ques'
|
122
|
+
|
123
|
+
haml :ques, layout: true
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
# class TestWorker < Heimdall::Worker
|
2
|
+
# timeout 10
|
3
|
+
#
|
4
|
+
# def call opts
|
5
|
+
# ...
|
6
|
+
# end
|
7
|
+
# end
|
8
|
+
#
|
9
|
+
# to add to que
|
10
|
+
#
|
11
|
+
# Heimdall.add :test, num: 1233
|
12
|
+
# or
|
13
|
+
# TestWorker.call num: 1233
|
14
|
+
|
15
|
+
module Heimdall
|
16
|
+
class Worker
|
17
|
+
cattr :retries, 3
|
18
|
+
cattr :retry_after, 60
|
19
|
+
cattr :timeout, nil
|
20
|
+
cattr :every, nil
|
21
|
+
cattr :cron, nil
|
22
|
+
cattr :backtrace, true
|
23
|
+
cattr :unique_for, nil
|
24
|
+
|
25
|
+
###
|
26
|
+
|
27
|
+
class << self
|
28
|
+
def call opts = {}
|
29
|
+
Heimdall.add self, opts.to_hwia
|
30
|
+
end
|
31
|
+
alias :perform :call
|
32
|
+
end
|
33
|
+
|
34
|
+
###
|
35
|
+
|
36
|
+
def initialize
|
37
|
+
@_log = []
|
38
|
+
end
|
39
|
+
|
40
|
+
def log text = nil
|
41
|
+
if text
|
42
|
+
@_log.push text
|
43
|
+
text
|
44
|
+
else
|
45
|
+
@_log.join $/
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# With start Heimdall you can add jobs
|
2
|
+
# Heimdall.start
|
3
|
+
|
4
|
+
# With run you can add and have running server that will process jobs
|
5
|
+
# Heimdall.run
|
6
|
+
|
7
|
+
# to stop, you should not need that
|
8
|
+
# Heimdall.stop
|
9
|
+
|
10
|
+
# Heimdall.add :test, num: 123
|
11
|
+
|
12
|
+
###
|
13
|
+
|
14
|
+
unless ENV['HEIMDALL_DB']
|
15
|
+
puts 'You are using Heimdall (https://....) but you have not defined HEIMDALL_DB'
|
16
|
+
puts 'Examples:'
|
17
|
+
puts 'HEIMDALL_DB=sqlite://./db/heimdall.sqlite'
|
18
|
+
puts 'HEIMDALL_DB=postgres://localhost:5432/heimdall'
|
19
|
+
exit
|
20
|
+
end
|
21
|
+
|
22
|
+
require_relative 'heimdall/heimdall_start'
|
23
|
+
require_relative 'heimdall/heimdall_base'
|
24
|
+
require_relative 'heimdall/heimdall_que'
|
25
|
+
require_relative 'heimdall/heimdall_database'
|
26
|
+
require_relative 'heimdall/heimdall_worker'
|
27
|
+
require_relative 'heimdall/heimdall_proxy'
|
28
|
+
require_relative 'heimdall/heimdall_web'
|
metadata
ADDED
@@ -0,0 +1,54 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: heimdall-worker
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Dino Reic
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2022-01-31 00:00:00.000000000 Z
|
12
|
+
dependencies: []
|
13
|
+
description: Sidekicq alternative
|
14
|
+
email: reic.dino@gmail.com
|
15
|
+
executables: []
|
16
|
+
extensions: []
|
17
|
+
extra_rdoc_files: []
|
18
|
+
files:
|
19
|
+
- "./.version"
|
20
|
+
- "./lib/heimdall-worker.rb"
|
21
|
+
- "./lib/heimdall/heimdall_base.rb"
|
22
|
+
- "./lib/heimdall/heimdall_database.rb"
|
23
|
+
- "./lib/heimdall/heimdall_model.rb"
|
24
|
+
- "./lib/heimdall/heimdall_model_schedule.rb"
|
25
|
+
- "./lib/heimdall/heimdall_model_task.rb"
|
26
|
+
- "./lib/heimdall/heimdall_proxy.rb"
|
27
|
+
- "./lib/heimdall/heimdall_que.rb"
|
28
|
+
- "./lib/heimdall/heimdall_start.rb"
|
29
|
+
- "./lib/heimdall/heimdall_web.rb"
|
30
|
+
- "./lib/heimdall/heimdall_worker.rb"
|
31
|
+
homepage: https://github.com/dux/heimdall-worker
|
32
|
+
licenses:
|
33
|
+
- MIT
|
34
|
+
metadata: {}
|
35
|
+
post_install_message:
|
36
|
+
rdoc_options: []
|
37
|
+
require_paths:
|
38
|
+
- lib
|
39
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '0'
|
44
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - ">="
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '0'
|
49
|
+
requirements: []
|
50
|
+
rubygems_version: 3.2.22
|
51
|
+
signing_key:
|
52
|
+
specification_version: 4
|
53
|
+
summary: Simple, efficient background processing for Ruby.
|
54
|
+
test_files: []
|