fasten 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.rubocop.yml +4 -0
- data/Gemfile.lock +4 -2
- data/README.md +13 -10
- data/bin/console +8 -0
- data/fasten.gemspec +1 -0
- data/lib/fasten/dag.rb +71 -39
- data/lib/fasten/executor.rb +90 -52
- data/lib/fasten/log_support.rb +31 -2
- data/lib/fasten/task.rb +0 -8
- data/lib/fasten/ui.rb +165 -0
- data/lib/fasten/version.rb +1 -1
- data/lib/fasten/worker.rb +113 -0
- data/lib/fasten.rb +16 -6
- metadata +18 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1ca1b131f4ee26ffb7f5dcaa5408e314988c3568d155b9cb804daba84c588f90
|
4
|
+
data.tar.gz: 6bd08febfd39b5e0ab11c95e6e687acdb6778c67f493469033e49364f47b6012
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5fa5a15d9c430cbbe53fee25917c9dcaba4dfb40e70b6b5323d1a46036d0e2a53fdc1d4960226bddf14a76946499fcc8e777b9431596c7c92bfd261093476ed3
|
7
|
+
data.tar.gz: 4d0596481e975f9c4406f1c46ce707cde8f1f9d11b4089e94a825fe5281b3139e6a9fdf82ac3e905b606ef16a6157019aa04ee3bdd7a3687ebe5310d2303ede3
|
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,8 +1,9 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fasten (0.
|
4
|
+
fasten (0.2.0)
|
5
5
|
binding_of_caller
|
6
|
+
curses
|
6
7
|
|
7
8
|
GEM
|
8
9
|
remote: https://rubygems.org/
|
@@ -11,6 +12,7 @@ GEM
|
|
11
12
|
binding_of_caller (0.8.0)
|
12
13
|
debug_inspector (>= 0.0.1)
|
13
14
|
coderay (1.1.2)
|
15
|
+
curses (1.2.4)
|
14
16
|
debug_inspector (0.0.3)
|
15
17
|
diff-lcs (1.3)
|
16
18
|
jaro_winkler (1.5.1)
|
@@ -30,7 +32,7 @@ GEM
|
|
30
32
|
rspec-mocks (~> 3.8.0)
|
31
33
|
rspec-core (3.8.0)
|
32
34
|
rspec-support (~> 3.8.0)
|
33
|
-
rspec-expectations (3.8.
|
35
|
+
rspec-expectations (3.8.2)
|
34
36
|
diff-lcs (>= 1.2.0, < 2.0)
|
35
37
|
rspec-support (~> 3.8.0)
|
36
38
|
rspec-mocks (3.8.0)
|
data/README.md
CHANGED
@@ -4,18 +4,21 @@ FIXME: Add intro here
|
|
4
4
|
|
5
5
|
## Roadmap
|
6
6
|
|
7
|
-
- [
|
8
|
-
- [
|
9
|
-
- [
|
10
|
-
- [
|
7
|
+
- [x] define task to be run to reach a goal, with dependencies (similar to rake, make, and others)
|
8
|
+
- [x] run task in parallel by default
|
9
|
+
- [x] dynamicly change the number of worker processes
|
10
|
+
- [x] allow to programatically define new tasks
|
11
11
|
- [ ] allow to programatically redefine existing tasks
|
12
|
-
- [
|
13
|
-
- [
|
14
|
-
- [
|
12
|
+
- [x] keep each task log in a separate file, to easily debug errors
|
13
|
+
- [x] early stop in case of failure
|
14
|
+
- [x] for non-tty execution, report with a simple progress log in STDOUT
|
15
15
|
- [ ] calculate ETA of script, based on previos executions of the same script
|
16
|
+
- [ ] calculate ETA without previous information
|
16
17
|
- [ ] for tty execution: start in background and provide a curses interface. When quit, the background process will keep running
|
17
18
|
- [ ] provide a cli for running simple tasks
|
18
|
-
- [
|
19
|
+
- [x] provide a curses mode that displays number of workers, running tasks, remaining tasks, etc. Curses mode is the default.
|
20
|
+
- [ ] display ETA in curses mode
|
21
|
+
- [ ] curses mode can control (pause/stop/resume) other running executions
|
19
22
|
|
20
23
|
|
21
24
|
## Installation
|
@@ -46,7 +49,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
|
|
46
49
|
|
47
50
|
## Contributing
|
48
51
|
|
49
|
-
Bug reports and pull requests are welcome on GitHub at https://github.com/
|
52
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/a0/fasten. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
|
50
53
|
|
51
54
|
## License
|
52
55
|
|
@@ -54,4 +57,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
|
|
54
57
|
|
55
58
|
## Code of Conduct
|
56
59
|
|
57
|
-
Everyone interacting in the Fasten project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/
|
60
|
+
Everyone interacting in the Fasten project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/a0/fasten/blob/master/CODE_OF_CONDUCT.md).
|
data/bin/console
CHANGED
@@ -6,6 +6,14 @@ require 'fasten'
|
|
6
6
|
# You can add fixtures and/or initialization code here to make experimenting
|
7
7
|
# with your gem easier. You can also use a different console, if you like.
|
8
8
|
|
9
|
+
# call reload! for reloading clases in development
|
10
|
+
def reload!(base = 'fasten')
|
11
|
+
$LOADED_FEATURES.select { |feature| feature =~ %r{/#{base}/lib/} }.each do |feature|
|
12
|
+
puts "Updating #{feature.gsub(/.*#{base}/, base)}: #{load feature}"
|
13
|
+
end
|
14
|
+
nil
|
15
|
+
end
|
16
|
+
|
9
17
|
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
18
|
require 'pry'
|
11
19
|
Pry.start
|
data/fasten.gemspec
CHANGED
@@ -29,6 +29,7 @@ Gem::Specification.new do |spec|
|
|
29
29
|
spec.add_development_dependency 'rubocop'
|
30
30
|
|
31
31
|
spec.add_runtime_dependency 'binding_of_caller'
|
32
|
+
spec.add_runtime_dependency 'curses'
|
32
33
|
|
33
34
|
raise 'RubyGems 2.0 or newer is required to protect against public gem pushes.' unless spec.respond_to?(:metadata)
|
34
35
|
|
data/lib/fasten/dag.rb
CHANGED
@@ -1,83 +1,115 @@
|
|
1
1
|
module Fasten
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
@
|
8
|
-
@
|
2
|
+
module DAG
|
3
|
+
attr_reader :task_list, :task_done_list, :task_error_list, :task_pending_list, :task_running_list
|
4
|
+
|
5
|
+
def initialize_dag
|
6
|
+
@task_map = {}
|
7
|
+
@task_list = []
|
8
|
+
@task_done_list = []
|
9
|
+
@task_error_list = []
|
10
|
+
@task_pending_list = []
|
11
|
+
@task_running_list = []
|
9
12
|
end
|
10
13
|
|
11
14
|
def add(task)
|
12
|
-
raise "Task '#{task.name}' already defined" if
|
15
|
+
raise "Task '#{task.name}' already defined" if @task_map[task.name]
|
13
16
|
|
14
|
-
@
|
15
|
-
|
17
|
+
@task_map[task.name] = task
|
18
|
+
@task_list << task
|
19
|
+
@task_waiting_list = nil
|
16
20
|
end
|
17
21
|
|
18
|
-
def update_task(task
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
+
def update_task(task)
|
23
|
+
if task.error
|
24
|
+
update_error_task task
|
25
|
+
else
|
26
|
+
update_done_task task
|
27
|
+
end
|
28
|
+
end
|
22
29
|
|
23
|
-
|
24
|
-
@
|
30
|
+
def update_done_task(task)
|
31
|
+
@task_done_list << task
|
32
|
+
@task_pending_list.delete task
|
25
33
|
task.dependants.each do |dependant_task|
|
26
|
-
dependant_task.depends
|
34
|
+
dependant_task.depends.delete task
|
27
35
|
end
|
28
36
|
|
29
|
-
|
37
|
+
move_pending_to_waiting
|
38
|
+
end
|
39
|
+
|
40
|
+
def update_error_task(task)
|
41
|
+
@task_error_list << task
|
42
|
+
@task_pending_list.delete task
|
30
43
|
end
|
31
44
|
|
32
45
|
def next_task
|
33
|
-
|
46
|
+
task_waiting_list.pop
|
34
47
|
end
|
35
48
|
|
36
|
-
def
|
37
|
-
return @
|
49
|
+
def task_waiting_list
|
50
|
+
return @task_waiting_list if @task_waiting_list
|
38
51
|
|
39
52
|
reset_tasks
|
40
53
|
setup_tasks_dependencies
|
41
|
-
|
42
|
-
update_waiting
|
43
|
-
|
44
|
-
@waiting
|
54
|
+
move_pending_to_waiting
|
45
55
|
end
|
46
56
|
|
47
57
|
protected
|
48
58
|
|
49
|
-
def
|
50
|
-
move_list =
|
51
|
-
task.depends.zero?
|
59
|
+
def move_pending_to_waiting
|
60
|
+
move_list = task_pending_list.select do |task|
|
61
|
+
task.depends.count.zero?
|
52
62
|
end
|
53
63
|
|
54
|
-
@
|
55
|
-
@
|
64
|
+
@task_waiting_list ||= []
|
65
|
+
@task_pending_list -= move_list
|
66
|
+
@task_waiting_list += move_list
|
56
67
|
end
|
57
68
|
|
58
69
|
def reset_tasks
|
59
|
-
@
|
60
|
-
@
|
61
|
-
|
70
|
+
@task_pending_list.clear
|
71
|
+
@task_done_list.clear
|
72
|
+
@task_list.each do |task|
|
62
73
|
task.dependants = []
|
63
|
-
task.depends =
|
64
|
-
|
65
|
-
|
74
|
+
task.depends = []
|
75
|
+
task.level = 0
|
76
|
+
task.done ? @task_done_list << task : @task_pending_list << task
|
66
77
|
end
|
67
78
|
end
|
68
79
|
|
69
80
|
def setup_tasks_dependencies
|
70
|
-
@
|
81
|
+
@task_pending_list.each do |task|
|
71
82
|
next unless task.after
|
72
83
|
|
73
84
|
[task.after].flatten.each do |after|
|
74
|
-
after_task = after.is_a?(Task) ? after :
|
85
|
+
after_task = after.is_a?(Task) ? after : @task_map[after]
|
75
86
|
raise "Dependency task '#{after}' not found on task '#{task.name}'." unless after_task
|
76
87
|
|
77
|
-
task.depends
|
88
|
+
task.depends << after_task
|
89
|
+
task.level += 1
|
78
90
|
after_task.dependants << task
|
79
91
|
end
|
80
92
|
end
|
81
93
|
end
|
94
|
+
|
95
|
+
def no_waiting_tasks?
|
96
|
+
task_waiting_list.empty?
|
97
|
+
end
|
98
|
+
|
99
|
+
def no_running_tasks?
|
100
|
+
task_running_list.empty?
|
101
|
+
end
|
102
|
+
|
103
|
+
def tasks_waiting?
|
104
|
+
!task_waiting_list.empty?
|
105
|
+
end
|
106
|
+
|
107
|
+
def tasks_running?
|
108
|
+
!task_running_list.empty?
|
109
|
+
end
|
110
|
+
|
111
|
+
def tasks_failed?
|
112
|
+
!task_error_list.empty?
|
113
|
+
end
|
82
114
|
end
|
83
115
|
end
|
data/lib/fasten/executor.rb
CHANGED
@@ -1,87 +1,125 @@
|
|
1
1
|
module Fasten
|
2
2
|
class Executor < Task
|
3
3
|
include Fasten::LogSupport
|
4
|
+
include Fasten::DAG
|
5
|
+
include Fasten::UI
|
6
|
+
|
7
|
+
def initialize(name: nil, workers: 8, worker_class: Fasten::Worker, fasten_dir: '.fasten')
|
8
|
+
super name: name || "#{self.class} #{$PID}", workers: workers, pid: $PID, state: :IDLE, worker_class: worker_class, fasten_dir: fasten_dir
|
9
|
+
initialize_dag
|
10
|
+
|
11
|
+
self.worker_list = []
|
12
|
+
log_path = "#{fasten_dir}/log/executor/#{self.name}.log"
|
13
|
+
FileUtils.mkdir_p File.dirname(log_path)
|
14
|
+
self.log_file = File.new(log_path, 'a')
|
15
|
+
Fasten.logger.reopen log_file
|
16
|
+
end
|
17
|
+
|
18
|
+
def perform
|
19
|
+
log_ini self, running_stats
|
20
|
+
self.state = :RUNNING
|
4
21
|
|
5
|
-
|
6
|
-
|
7
|
-
|
22
|
+
run_ui do
|
23
|
+
perform_loop
|
24
|
+
end
|
8
25
|
|
9
|
-
self.
|
10
|
-
self
|
11
|
-
self.dag = Fasten::DAG.new
|
12
|
-
self.running = false
|
13
|
-
self.child_jobs = {}
|
14
|
-
self.running_tasks = []
|
26
|
+
self.state = :IDLE
|
27
|
+
log_fin self, running_stats
|
15
28
|
end
|
16
29
|
|
17
|
-
def
|
18
|
-
|
30
|
+
def done_stats
|
31
|
+
"#{task_done_list.count}/#{task_list.count}"
|
19
32
|
end
|
20
33
|
|
21
|
-
def
|
22
|
-
|
23
|
-
|
24
|
-
self.running = true
|
34
|
+
def running_stats
|
35
|
+
"#{task_done_list.count + task_running_list.count}/#{task_list.count}"
|
36
|
+
end
|
25
37
|
|
26
|
-
|
38
|
+
def perform_loop
|
39
|
+
loop do
|
40
|
+
wait_for_running_tasks
|
41
|
+
raise_error_in_failure
|
42
|
+
remove_workers_as_needed
|
43
|
+
dispatch_pending_tasks
|
44
|
+
|
45
|
+
break if no_running_tasks? && no_waiting_tasks?
|
46
|
+
end
|
27
47
|
|
28
|
-
|
29
|
-
log_fin self
|
48
|
+
remove_all_workers
|
30
49
|
end
|
31
50
|
|
32
|
-
|
51
|
+
def wait_for_running_tasks
|
52
|
+
while (no_waiting_tasks? && tasks_running?) || task_running_list.count >= workers || (tasks_running? && tasks_failed?)
|
53
|
+
ui_update
|
54
|
+
reads = worker_list.map(&:parent_read)
|
55
|
+
reads, _writes, _errors = IO.select(reads, [], [], 0.2)
|
33
56
|
|
34
|
-
|
35
|
-
|
57
|
+
receive_workers_tasks(reads)
|
58
|
+
end
|
59
|
+
ui_update
|
36
60
|
end
|
37
61
|
|
38
|
-
def
|
39
|
-
|
40
|
-
|
62
|
+
def receive_workers_tasks(reads)
|
63
|
+
reads&.each do |read|
|
64
|
+
worker = worker_list.find { |item| item.parent_read == read }
|
65
|
+
task = worker.receive
|
41
66
|
|
42
|
-
|
43
|
-
while running
|
44
|
-
next_task = dag.next_task
|
67
|
+
task_running_list.delete task
|
45
68
|
|
46
|
-
|
47
|
-
run_next_task next_task
|
69
|
+
update_task task
|
48
70
|
|
49
|
-
|
71
|
+
log_fin task, done_stats
|
50
72
|
end
|
51
|
-
|
52
|
-
wait_remaining
|
53
73
|
end
|
54
74
|
|
55
|
-
def
|
56
|
-
return unless
|
75
|
+
def raise_error_in_failure
|
76
|
+
return unless tasks_failed?
|
77
|
+
|
78
|
+
remove_all_workers
|
57
79
|
|
58
|
-
|
59
|
-
done_task = child_jobs.delete pid
|
60
|
-
return unless done_task
|
80
|
+
count = task_error_list.count
|
61
81
|
|
62
|
-
|
63
|
-
|
82
|
+
raise "Stopping because the following #{count} #{count == 1 ? 'task' : 'tasks'} failed: #{task_error_list.map(&:to_s).join(', ')}"
|
83
|
+
end
|
84
|
+
|
85
|
+
def remove_workers_as_needed
|
86
|
+
while worker_list.count > workers
|
87
|
+
return unless (worker = worker_list.find { |item| item.running_task.nil? })
|
64
88
|
|
65
|
-
|
89
|
+
worker.kill
|
90
|
+
worker_list.delete worker
|
91
|
+
end
|
66
92
|
end
|
67
93
|
|
68
|
-
def
|
69
|
-
|
94
|
+
def find_or_create_worker
|
95
|
+
worker = worker_list.find { |item| item.running_task.nil? }
|
96
|
+
|
97
|
+
unless worker
|
98
|
+
@worker_id = (@worker_id || 0) + 1
|
99
|
+
worker = worker_class.new executor: self, name: "#{worker_class} #{format '%02X', @worker_id}"
|
100
|
+
worker.fork
|
101
|
+
worker_list << worker
|
102
|
+
|
103
|
+
log_info "Worker created: #{worker}"
|
104
|
+
end
|
105
|
+
|
106
|
+
worker
|
107
|
+
end
|
70
108
|
|
71
|
-
|
72
|
-
|
109
|
+
def dispatch_pending_tasks
|
110
|
+
while tasks_waiting? && task_running_list.count < workers
|
111
|
+
worker = find_or_create_worker
|
73
112
|
|
74
|
-
|
75
|
-
|
76
|
-
|
113
|
+
task = next_task
|
114
|
+
log_ini task, "on worker #{worker}"
|
115
|
+
worker.dispatch(task)
|
116
|
+
task_running_list << task
|
77
117
|
end
|
78
|
-
child_jobs[pid] = next_task
|
79
118
|
end
|
80
119
|
|
81
|
-
def
|
82
|
-
|
83
|
-
|
84
|
-
dag.update_task child_task, done: true
|
120
|
+
def remove_all_workers
|
121
|
+
while (worker = worker_list.pop)
|
122
|
+
worker.kill
|
85
123
|
end
|
86
124
|
end
|
87
125
|
end
|
data/lib/fasten/log_support.rb
CHANGED
@@ -8,10 +8,39 @@ module Fasten
|
|
8
8
|
define_method "log_#{method}" do |msg|
|
9
9
|
return unless Fasten.logger.respond_to?(method)
|
10
10
|
|
11
|
-
|
12
|
-
|
11
|
+
caller_name = name if respond_to? :name
|
12
|
+
caller_name ||= Kernel.binding.of_caller(1).eval('self').class
|
13
|
+
Fasten.logger.send(method, caller_name) { msg }
|
13
14
|
end
|
14
15
|
end
|
16
|
+
|
17
|
+
def log_ini(object, message = nil)
|
18
|
+
object.ini ||= Time.new
|
19
|
+
log_info "Init #{object.class} #{object} #{message}"
|
20
|
+
end
|
21
|
+
|
22
|
+
def log_fin(object, message = nil)
|
23
|
+
object.fin ||= Time.new
|
24
|
+
diff = object.fin - object.ini
|
25
|
+
|
26
|
+
log_info "Done #{object.class} #{object} #{message} in #{diff}"
|
27
|
+
end
|
28
|
+
|
29
|
+
def redirect_std(path)
|
30
|
+
@saved_stdout = $stdout.clone
|
31
|
+
@saved_stderr = $stderr.clone
|
32
|
+
|
33
|
+
FileUtils.mkdir_p File.dirname(path)
|
34
|
+
log = File.new path, 'a'
|
35
|
+
|
36
|
+
$stdout.reopen log
|
37
|
+
$stderr.reopen log
|
38
|
+
end
|
39
|
+
|
40
|
+
def restore_std
|
41
|
+
$stdout.reopen(@saved_stdout)
|
42
|
+
$stderr.reopen(@saved_stderr)
|
43
|
+
end
|
15
44
|
end
|
16
45
|
end
|
17
46
|
|
data/lib/fasten/task.rb
CHANGED
data/lib/fasten/ui.rb
ADDED
@@ -0,0 +1,165 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Fasten
|
4
|
+
module UI
|
5
|
+
include Curses
|
6
|
+
|
7
|
+
SPINNER_STR = '⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏'
|
8
|
+
SPINNER_LEN = SPINNER_STR.length
|
9
|
+
PROGRESSBAR_STR = ' ▏▎▍▌▋▊▉'
|
10
|
+
PROGRESSBAR_LEN = PROGRESSBAR_STR.length
|
11
|
+
|
12
|
+
def run_ui
|
13
|
+
ui_setup
|
14
|
+
ui_title
|
15
|
+
ui_workers
|
16
|
+
|
17
|
+
yield
|
18
|
+
ensure
|
19
|
+
close_screen
|
20
|
+
end
|
21
|
+
|
22
|
+
def ui_setup
|
23
|
+
init_screen
|
24
|
+
self.ui_rows = lines
|
25
|
+
self.ui_cols = cols
|
26
|
+
ui_setup_color
|
27
|
+
noecho
|
28
|
+
cbreak
|
29
|
+
nonl
|
30
|
+
curs_set 0
|
31
|
+
end
|
32
|
+
|
33
|
+
def ui_setup_color
|
34
|
+
start_color
|
35
|
+
use_default_colors
|
36
|
+
init_pair 1, Curses::COLOR_YELLOW, -1
|
37
|
+
init_pair 2, Curses::COLOR_GREEN, -1
|
38
|
+
init_pair 3, Curses::COLOR_RED, -1
|
39
|
+
init_pair 4, Curses::COLOR_WHITE, -1
|
40
|
+
end
|
41
|
+
|
42
|
+
def ui_text_aligned(row, align, str, attrs = nil)
|
43
|
+
if align == :center
|
44
|
+
setpos row, (ui_cols - str.length) / 2
|
45
|
+
elsif align == :right
|
46
|
+
setpos row, ui_cols - str.length
|
47
|
+
else
|
48
|
+
setpos row, 0
|
49
|
+
end
|
50
|
+
|
51
|
+
attrset attrs if attrs
|
52
|
+
addstr str
|
53
|
+
attroff attrs if attrs
|
54
|
+
|
55
|
+
str.length
|
56
|
+
end
|
57
|
+
|
58
|
+
def ui_title
|
59
|
+
ui_text_aligned(0, :left, 'Fasten your seatbelts!')
|
60
|
+
ui_text_aligned(0, :center, name.to_s)
|
61
|
+
ui_text_aligned(0, :right, Time.new.to_s)
|
62
|
+
end
|
63
|
+
|
64
|
+
def ui_update
|
65
|
+
clear
|
66
|
+
ui_title
|
67
|
+
ui_workers
|
68
|
+
ui_tasks
|
69
|
+
|
70
|
+
refresh
|
71
|
+
end
|
72
|
+
|
73
|
+
def ui_workers_summary
|
74
|
+
"Procs: #{task_running_list.count} run #{worker_list.count - task_running_list.count} idle #{workers} max"
|
75
|
+
end
|
76
|
+
|
77
|
+
def ui_workers
|
78
|
+
l = ui_text_aligned(1, :left, ui_workers_summary) + 1
|
79
|
+
|
80
|
+
worker_list.each_with_index do |worker, index|
|
81
|
+
setpos 1, l + index
|
82
|
+
attrs = worker.running? ? A_STANDOUT : color_pair(4) | A_DIM
|
83
|
+
attrset attrs
|
84
|
+
addstr worker.running? ? 'R' : '_'
|
85
|
+
attroff attrs
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def ui_progressbar(row, col_ini, col_fin, count, total)
|
90
|
+
slice = total.to_f / (col_fin - col_ini + 1)
|
91
|
+
col_ini.upto col_fin do |col|
|
92
|
+
setpos row, col
|
93
|
+
count -= slice
|
94
|
+
if count.positive?
|
95
|
+
addstr PROGRESSBAR_STR[-1]
|
96
|
+
elsif count > -slice
|
97
|
+
addstr PROGRESSBAR_STR[(count * PROGRESSBAR_LEN / slice) % PROGRESSBAR_LEN]
|
98
|
+
else
|
99
|
+
addstr '.'
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
def ui_task_string(task, y, x, icon: nil)
|
105
|
+
setpos y, x
|
106
|
+
|
107
|
+
case task.state
|
108
|
+
when :RUNNING
|
109
|
+
attrs = color_pair(1) | A_TOP
|
110
|
+
icon = SPINNER_STR[task.worker&.spinner] if icon
|
111
|
+
when :ERROR
|
112
|
+
attrs = color_pair(3)
|
113
|
+
icon = '✘︎' if icon
|
114
|
+
when :DONE
|
115
|
+
attrs = color_pair(2)
|
116
|
+
icon = '✔︎' if icon
|
117
|
+
else
|
118
|
+
attrs = color_pair(4) | A_DIM
|
119
|
+
icon = '…' if icon
|
120
|
+
end
|
121
|
+
|
122
|
+
str = icon ? "#{icon} #{task}" : task.to_s
|
123
|
+
|
124
|
+
attrset attrs if attrs
|
125
|
+
addstr str
|
126
|
+
attroff attrs if attrs
|
127
|
+
|
128
|
+
x + str.length
|
129
|
+
end
|
130
|
+
|
131
|
+
def ui_tasks
|
132
|
+
worker_list.each do |worker|
|
133
|
+
worker.spinner = (worker.spinner + 1) % SPINNER_LEN if worker.running?
|
134
|
+
end
|
135
|
+
|
136
|
+
count_done = task_done_list.count
|
137
|
+
count_total = task_list.count
|
138
|
+
tl = count_total.to_s.length
|
139
|
+
col_ini = ui_text_aligned(2, :left, format("Tasks: %#{tl}d/%s", count_done, count_total)) + 1
|
140
|
+
col_fin = ui_cols - 5
|
141
|
+
ui_text_aligned(2, :right, "#{(count_done * 100/count_total).to_i}%") if count_total.positive?
|
142
|
+
|
143
|
+
ui_progressbar(2, col_ini, col_fin, count_done, count_total)
|
144
|
+
|
145
|
+
max = 2
|
146
|
+
task_list.each_with_index do |task, index|
|
147
|
+
next if 3 + index >= ui_rows
|
148
|
+
|
149
|
+
x = ui_task_string(task, 3 + index, 2, icon: true)
|
150
|
+
max = x if x > max
|
151
|
+
end
|
152
|
+
|
153
|
+
task_list.each_with_index do |task, index|
|
154
|
+
next if 3 + index >= ui_rows || task.depends.nil? || task.depends.empty?
|
155
|
+
|
156
|
+
setpos 3 + index, max
|
157
|
+
x = max + 2
|
158
|
+
addstr ':'
|
159
|
+
task.depends.each do |dependant_task|
|
160
|
+
x = ui_task_string(dependant_task, 3 + index, x) + 1
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
165
|
+
end
|
data/lib/fasten/version.rb
CHANGED
@@ -0,0 +1,113 @@
|
|
1
|
+
module Fasten
|
2
|
+
class Worker < Task
|
3
|
+
include Fasten::LogSupport
|
4
|
+
|
5
|
+
def initialize(executor:, name: nil)
|
6
|
+
super executor: executor, name: name, spinner: 0
|
7
|
+
end
|
8
|
+
|
9
|
+
def perform(task)
|
10
|
+
system task.shell if task.shell
|
11
|
+
end
|
12
|
+
|
13
|
+
def fork
|
14
|
+
create_pipes
|
15
|
+
|
16
|
+
self.pid = Process.fork do
|
17
|
+
close_parent_pipes
|
18
|
+
|
19
|
+
process_incoming_tasks
|
20
|
+
end
|
21
|
+
|
22
|
+
close_child_pipes
|
23
|
+
end
|
24
|
+
|
25
|
+
def dispatch(task)
|
26
|
+
Marshal.dump(Task.new(task.to_h.merge(depends: nil, dependants: nil)), parent_write)
|
27
|
+
self.running_task = task
|
28
|
+
task.worker = self
|
29
|
+
task.state = :RUNNING
|
30
|
+
end
|
31
|
+
|
32
|
+
def receive
|
33
|
+
updated_task = Marshal.load(parent_read) # rubocop:disable Security/MarshalLoad
|
34
|
+
|
35
|
+
%i[ini fin response error].each { |key| running_task[key] = updated_task[key] }
|
36
|
+
|
37
|
+
task = running_task
|
38
|
+
self.running_task = nil
|
39
|
+
task.state = task.error ? :ERROR : :DONE
|
40
|
+
|
41
|
+
task
|
42
|
+
end
|
43
|
+
|
44
|
+
def kill
|
45
|
+
log_info 'Removing worker'
|
46
|
+
Process.kill(:KILL, pid)
|
47
|
+
close_parent_pipes
|
48
|
+
rescue StandardError => error
|
49
|
+
log_warn "Ignoring error killing worker #{self}, error: #{error}"
|
50
|
+
end
|
51
|
+
|
52
|
+
def idle?
|
53
|
+
running_task.nil?
|
54
|
+
end
|
55
|
+
|
56
|
+
def running?
|
57
|
+
!idle?
|
58
|
+
end
|
59
|
+
|
60
|
+
protected
|
61
|
+
|
62
|
+
def create_pipes
|
63
|
+
self.child_read, self.parent_write = IO.pipe
|
64
|
+
self.parent_read, self.child_write = IO.pipe
|
65
|
+
end
|
66
|
+
|
67
|
+
def close_parent_pipes
|
68
|
+
parent_read.close unless parent_read.closed?
|
69
|
+
parent_write.close unless parent_write.closed?
|
70
|
+
end
|
71
|
+
|
72
|
+
def close_child_pipes
|
73
|
+
child_read.close unless child_read.closed?
|
74
|
+
child_write.close unless child_write.closed?
|
75
|
+
end
|
76
|
+
|
77
|
+
def process_incoming_tasks
|
78
|
+
log_ini self, 'process_incoming_tasks'
|
79
|
+
|
80
|
+
while (object = Marshal.load(child_read)) # rubocop:disable Security/MarshalLoad
|
81
|
+
run_task(object) if object.is_a? Fasten::Task
|
82
|
+
end
|
83
|
+
|
84
|
+
log_fin self, 'process_incoming_tasks'
|
85
|
+
rescue EOFError
|
86
|
+
log_info 'Terminating on EOF'
|
87
|
+
end
|
88
|
+
|
89
|
+
def run_task(task)
|
90
|
+
log_ini task, 'perform'
|
91
|
+
Fasten.logger.reopen(STDOUT)
|
92
|
+
redirect_std "#{executor.fasten_dir}/log/task/#{task.name}.log"
|
93
|
+
|
94
|
+
perform_task task
|
95
|
+
|
96
|
+
restore_std
|
97
|
+
Fasten.logger.reopen(executor.log_file)
|
98
|
+
log_fin task, 'perform'
|
99
|
+
end
|
100
|
+
|
101
|
+
def perform_task(task)
|
102
|
+
log_ini task, 'perform'
|
103
|
+
|
104
|
+
perform(task)
|
105
|
+
|
106
|
+
log_fin task, 'perform'
|
107
|
+
Marshal.dump(task, child_write)
|
108
|
+
rescue StandardError => error
|
109
|
+
task.error = error
|
110
|
+
Marshal.dump(task, child_write)
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
data/lib/fasten.rb
CHANGED
@@ -5,31 +5,41 @@ require 'yaml'
|
|
5
5
|
require 'binding_of_caller'
|
6
6
|
require 'logger'
|
7
7
|
require 'ostruct'
|
8
|
+
require 'curses'
|
8
9
|
|
9
10
|
require 'fasten/log_support'
|
10
11
|
require 'fasten/task'
|
12
|
+
require 'fasten/ui'
|
11
13
|
require 'fasten/dag'
|
12
14
|
require 'fasten/executor'
|
15
|
+
require 'fasten/worker'
|
13
16
|
require 'fasten/version'
|
14
17
|
|
15
18
|
module Fasten
|
16
19
|
class << self
|
17
20
|
include Fasten::LogSupport
|
18
21
|
|
19
|
-
def load(path)
|
20
|
-
executor = Fasten::Executor.new
|
22
|
+
def load(path, **options)
|
23
|
+
executor = Fasten::Executor.new(**options)
|
24
|
+
executor.load(path)
|
25
|
+
|
26
|
+
executor
|
27
|
+
end
|
28
|
+
end
|
21
29
|
|
22
|
-
|
30
|
+
class Executor
|
31
|
+
def load(path)
|
32
|
+
items = YAML.safe_load(File.read(path)).each do |name, params|
|
23
33
|
params.each do |key, val|
|
24
34
|
next unless val.is_a?(String) && (match = %r{^/(.+)/$}.match(val))
|
25
35
|
|
26
36
|
params[key] = Regexp.new(match[1])
|
27
37
|
end
|
28
|
-
|
38
|
+
|
39
|
+
add Fasten::Task.new({ name: name }.merge(params))
|
29
40
|
end
|
30
41
|
|
31
|
-
log_info "Loaded #{
|
32
|
-
executor
|
42
|
+
log_info "Loaded #{items.count} tasks from #{path}"
|
33
43
|
end
|
34
44
|
end
|
35
45
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fasten
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Aldrin Martoq
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-10-
|
11
|
+
date: 2018-10-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -94,6 +94,20 @@ dependencies:
|
|
94
94
|
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: curses
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
97
111
|
description: Fasten your seatbelts! Run jobs in parallel, intelligently.
|
98
112
|
email:
|
99
113
|
- contacto@a0.cl
|
@@ -121,7 +135,9 @@ files:
|
|
121
135
|
- lib/fasten/executor.rb
|
122
136
|
- lib/fasten/log_support.rb
|
123
137
|
- lib/fasten/task.rb
|
138
|
+
- lib/fasten/ui.rb
|
124
139
|
- lib/fasten/version.rb
|
140
|
+
- lib/fasten/worker.rb
|
125
141
|
homepage: https://github.com/a0/fasten/
|
126
142
|
licenses:
|
127
143
|
- MIT
|