fasten 0.7.2 → 0.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/bin/console +1 -1
- data/lib/fasten/defaults.rb +47 -0
- data/lib/fasten/runner.rb +38 -42
- data/lib/fasten/support/stats.rb +5 -5
- data/lib/fasten/support/yaml.rb +2 -2
- data/lib/fasten/task.rb +1 -1
- data/lib/fasten/task_manager.rb +130 -0
- data/lib/fasten/ui/console.rb +6 -7
- data/lib/fasten/ui/curses.rb +16 -13
- data/lib/fasten/version.rb +1 -1
- data/lib/fasten.rb +9 -29
- metadata +4 -3
- data/lib/fasten/support/dag.rb +0 -130
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c4790a7db3f8bdbbaee848efc77a62dbc944338363a678c57b233e865df47b34
|
4
|
+
data.tar.gz: 87b97a2473f12e69ab2bd619264b25b7a01664997ee2ff140000d2ba7c68c5e7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4ac38b2b2b77e25c8b1911f41c94b091ccf48827b034c78ff9a939442a77004ca9821cd124aae32cc6ee099f344c605c70631ce6aaa5f2bd299150e9ae5a2149
|
7
|
+
data.tar.gz: 88294128e454055b9bd322ed3dfa9a7e86223ba191110041ab4ba55ab688ece163f549b470ae4fd58db4d6a16eb75aaf2b30ffb90f2e51f7288d410cb0e8a17a
|
data/Gemfile.lock
CHANGED
data/bin/console
CHANGED
@@ -7,7 +7,7 @@ require 'fasten'
|
|
7
7
|
# with your gem easier. You can also use a different console, if you like.
|
8
8
|
|
9
9
|
# call reload! for reloading clases in development
|
10
|
-
def reload!(base = 'fasten')
|
10
|
+
def reload!(base = 'a0-fasten-ruby')
|
11
11
|
$LOADED_FEATURES.select { |feature| feature =~ %r{/#{base}/lib/} }.each do |feature|
|
12
12
|
puts "Updating #{feature.gsub(/.*#{base}/, base)}: #{load feature}"
|
13
13
|
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Fasten
|
4
|
+
class << self
|
5
|
+
def default_name
|
6
|
+
File.basename(Dir.getwd)
|
7
|
+
end
|
8
|
+
|
9
|
+
def default_stats
|
10
|
+
true
|
11
|
+
end
|
12
|
+
|
13
|
+
def default_summary
|
14
|
+
false
|
15
|
+
end
|
16
|
+
|
17
|
+
def default_workers
|
18
|
+
Parallel.physical_processor_count
|
19
|
+
end
|
20
|
+
|
21
|
+
def default_worker_class
|
22
|
+
Worker
|
23
|
+
end
|
24
|
+
|
25
|
+
def default_fasten_dir
|
26
|
+
'fasten'
|
27
|
+
end
|
28
|
+
|
29
|
+
def default_use_threads
|
30
|
+
!OS.posix?
|
31
|
+
end
|
32
|
+
|
33
|
+
def default_ui_mode
|
34
|
+
return @default_ui_mode if defined? @default_ui_mode
|
35
|
+
|
36
|
+
require 'fasten/ui/curses'
|
37
|
+
|
38
|
+
@default_ui_mode = :curses
|
39
|
+
rescue StandardError, LoadError
|
40
|
+
@default_ui_mode = :console
|
41
|
+
end
|
42
|
+
|
43
|
+
def default_developer
|
44
|
+
STDIN.tty? && STDOUT.tty?
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
data/lib/fasten/runner.rb
CHANGED
@@ -3,7 +3,6 @@ require 'parallel'
|
|
3
3
|
require 'pry'
|
4
4
|
require 'os'
|
5
5
|
|
6
|
-
require 'fasten/support/dag'
|
7
6
|
require 'fasten/support/logger'
|
8
7
|
require 'fasten/support/state'
|
9
8
|
require 'fasten/support/stats'
|
@@ -13,34 +12,29 @@ require 'fasten/timeout_queue'
|
|
13
12
|
|
14
13
|
module Fasten
|
15
14
|
class Runner
|
16
|
-
include Fasten::Support::DAG
|
17
15
|
include Fasten::Support::Logger
|
18
16
|
include Fasten::Support::State
|
19
17
|
include Fasten::Support::Stats
|
20
18
|
include Fasten::Support::UI
|
21
19
|
include Fasten::Support::Yaml
|
22
20
|
|
23
|
-
attr_accessor :name, :
|
21
|
+
attr_accessor :name, :stats, :summary, :workers, :worker_class, :fasten_dir, :use_threads, :ui_mode, :developer, :worker_list, :queue, :tasks
|
24
22
|
|
25
|
-
def initialize(
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
23
|
+
def initialize(**options)
|
24
|
+
%i[name stats summary workers worker_class fasten_dir use_threads ui_mode developer].each do |key|
|
25
|
+
options[key] = Fasten.send "default_#{key}" unless options.key? key
|
26
|
+
end
|
27
|
+
|
28
|
+
@tasks = TaskManager.new
|
29
|
+
|
30
|
+
reconfigure(options)
|
30
31
|
end
|
31
32
|
|
32
33
|
def reconfigure(**options)
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
self.fasten_dir = options[:fasten_dir] if options.key?(:fasten_dir)
|
38
|
-
self.developer = options[:developer] if options.key?(:developer)
|
39
|
-
self.use_threads = options[:use_threads] if options.key?(:use_threads)
|
40
|
-
self.summary = options[:summary] if options.key?(:summary)
|
41
|
-
self.ui_mode = options[:ui_mode] if options.key?(:ui_mode)
|
42
|
-
|
43
|
-
initialize_dag
|
34
|
+
%i[name stats summary workers worker_class fasten_dir use_threads ui_mode developer].each do |key|
|
35
|
+
send "#{key}=", options[key] if options.key? key
|
36
|
+
end
|
37
|
+
|
44
38
|
initialize_stats
|
45
39
|
initialize_logger
|
46
40
|
|
@@ -48,7 +42,9 @@ module Fasten
|
|
48
42
|
end
|
49
43
|
|
50
44
|
def task(name, **opts, &block)
|
51
|
-
|
45
|
+
tasks << task = Task.new(name: name, **opts, block: block)
|
46
|
+
|
47
|
+
task
|
52
48
|
end
|
53
49
|
|
54
50
|
def register(&block)
|
@@ -64,7 +60,7 @@ module Fasten
|
|
64
60
|
perform_loop
|
65
61
|
end
|
66
62
|
|
67
|
-
self.state =
|
63
|
+
self.state = tasks.map(&:state).all?(:DONE) ? :DONE : :FAIL
|
68
64
|
log_fin self, running_counters
|
69
65
|
|
70
66
|
stats_add_entry(state, self)
|
@@ -76,20 +72,20 @@ module Fasten
|
|
76
72
|
|
77
73
|
def map(list, &block)
|
78
74
|
list.each do |item|
|
79
|
-
|
75
|
+
task item.to_s, request: item, &block
|
80
76
|
end
|
81
77
|
|
82
78
|
perform
|
83
79
|
|
84
|
-
|
80
|
+
tasks.map(&:response)
|
85
81
|
end
|
86
82
|
|
87
83
|
def done_counters
|
88
|
-
"#{
|
84
|
+
"#{tasks.done.count}/#{tasks.count}"
|
89
85
|
end
|
90
86
|
|
91
87
|
def running_counters
|
92
|
-
"#{
|
88
|
+
"#{tasks.done.count + tasks.running.count}/#{tasks.count}"
|
93
89
|
end
|
94
90
|
|
95
91
|
def perform_loop
|
@@ -103,25 +99,25 @@ module Fasten
|
|
103
99
|
dispatch_pending_tasks
|
104
100
|
end
|
105
101
|
|
106
|
-
break if
|
102
|
+
break if tasks.no_running? && tasks.no_waiting? || state == :QUIT
|
107
103
|
end
|
108
104
|
|
109
105
|
remove_all_workers
|
110
106
|
end
|
111
107
|
|
112
108
|
def check_state
|
113
|
-
if state == :PAUSING &&
|
109
|
+
if state == :PAUSING && tasks.no_running?
|
114
110
|
self.state = :PAUSED
|
115
111
|
ui.message = nil
|
116
112
|
ui.force_clear
|
117
|
-
elsif state == :QUITTING &&
|
113
|
+
elsif state == :QUITTING && tasks.no_running?
|
118
114
|
self.state = :QUIT
|
119
115
|
ui.force_clear
|
120
116
|
end
|
121
117
|
end
|
122
118
|
|
123
119
|
def should_wait_for_running_tasks?
|
124
|
-
|
120
|
+
tasks.running? && (tasks.no_waiting? || tasks.failed? || %i[PAUSING QUITTING].include?(state)) || tasks.running.count >= workers
|
125
121
|
end
|
126
122
|
|
127
123
|
def wait_for_running_tasks
|
@@ -134,21 +130,20 @@ module Fasten
|
|
134
130
|
while should_wait_for_running_tasks?
|
135
131
|
ui.update
|
136
132
|
|
137
|
-
|
138
|
-
|
139
|
-
receive_workers_tasks_thread(tasks)
|
133
|
+
receive_workers_tasks_thread queue.receive_with_timeout(0.5)
|
140
134
|
end
|
141
135
|
|
142
136
|
ui.update
|
143
137
|
end
|
144
138
|
|
145
|
-
def receive_workers_tasks_thread(
|
146
|
-
|
147
|
-
|
139
|
+
def receive_workers_tasks_thread(items)
|
140
|
+
items&.each do |task|
|
141
|
+
tasks.running.delete task
|
148
142
|
|
149
143
|
task.worker.running_task = task.worker.state = nil
|
150
144
|
|
151
|
-
|
145
|
+
tasks.update task
|
146
|
+
stats_add_entry(task.state, task)
|
152
147
|
|
153
148
|
log_fin task, done_counters
|
154
149
|
ui.force_clear
|
@@ -173,9 +168,10 @@ module Fasten
|
|
173
168
|
|
174
169
|
task = worker.receive_response_from_child
|
175
170
|
|
176
|
-
|
171
|
+
tasks.running.delete task
|
177
172
|
|
178
|
-
|
173
|
+
tasks.update task
|
174
|
+
stats_add_entry(task.state, task)
|
179
175
|
|
180
176
|
log_fin task, done_counters
|
181
177
|
ui.force_clear
|
@@ -189,7 +185,7 @@ module Fasten
|
|
189
185
|
end
|
190
186
|
|
191
187
|
def raise_error_in_failure
|
192
|
-
return unless
|
188
|
+
return unless tasks.failed?
|
193
189
|
|
194
190
|
show_error_tasks
|
195
191
|
|
@@ -238,13 +234,13 @@ module Fasten
|
|
238
234
|
end
|
239
235
|
|
240
236
|
def dispatch_pending_tasks
|
241
|
-
while
|
237
|
+
while tasks.waiting? && tasks.running.count < workers
|
242
238
|
worker = find_or_create_worker
|
243
239
|
|
244
|
-
task =
|
240
|
+
task = tasks.next
|
245
241
|
log_ini task, "on worker #{worker}"
|
246
242
|
worker.send_request_to_child(task)
|
247
|
-
|
243
|
+
tasks.running << task
|
248
244
|
|
249
245
|
ui.force_clear
|
250
246
|
end
|
data/lib/fasten/support/stats.rb
CHANGED
@@ -25,7 +25,7 @@ module Fasten
|
|
25
25
|
stats_data << row.to_h
|
26
26
|
end
|
27
27
|
|
28
|
-
@
|
28
|
+
@tasks.waiting = nil
|
29
29
|
rescue StandardError
|
30
30
|
nil
|
31
31
|
end
|
@@ -127,10 +127,10 @@ module Fasten
|
|
127
127
|
end
|
128
128
|
|
129
129
|
def update_stats(history, entry)
|
130
|
-
entry['cnt'] =
|
131
|
-
entry['avg'] = avg = history.sum.to_f /
|
132
|
-
entry['std'] = std = Math.sqrt(history.
|
133
|
-
entry['err'] = std / Math.sqrt(
|
130
|
+
entry['cnt'] = cnt = history.size
|
131
|
+
entry['avg'] = avg = history.sum.to_f / cnt
|
132
|
+
entry['std'] = std = Math.sqrt(history.map { |x| (x - avg)**2 }.sum / cnt)
|
133
|
+
entry['err'] = std / Math.sqrt(cnt) if cnt.positive?
|
134
134
|
end
|
135
135
|
end
|
136
136
|
end
|
data/lib/fasten/support/yaml.rb
CHANGED
@@ -13,7 +13,7 @@ module Fasten
|
|
13
13
|
params = {}
|
14
14
|
end
|
15
15
|
|
16
|
-
|
16
|
+
task name, params
|
17
17
|
end
|
18
18
|
|
19
19
|
log_info "Loaded #{items.count} tasks from #{path}"
|
@@ -22,7 +22,7 @@ module Fasten
|
|
22
22
|
def save_yaml(path)
|
23
23
|
keys = %i[after shell]
|
24
24
|
|
25
|
-
items =
|
25
|
+
items = tasks.map do |task|
|
26
26
|
data = task.to_h.select do |key, _val|
|
27
27
|
keys.include? key
|
28
28
|
end
|
data/lib/fasten/task.rb
CHANGED
@@ -7,7 +7,7 @@ module Fasten
|
|
7
7
|
attr_accessor :name, :after, :shell, :ruby
|
8
8
|
attr_accessor :dependants, :depends, :request, :response, :worker, :run_score, :block
|
9
9
|
|
10
|
-
def initialize(name
|
10
|
+
def initialize(name:, shell: nil, ruby: nil, block: nil, request: nil, after: nil)
|
11
11
|
self.name = name
|
12
12
|
self.shell = shell
|
13
13
|
self.ruby = ruby
|
@@ -0,0 +1,130 @@
|
|
1
|
+
module Fasten
|
2
|
+
class TaskManager < Array
|
3
|
+
attr_reader :done, :failed, :pending, :running, :targets
|
4
|
+
|
5
|
+
def initialize(targets: nil)
|
6
|
+
super()
|
7
|
+
|
8
|
+
@map = {}
|
9
|
+
@done = []
|
10
|
+
@failed = []
|
11
|
+
@pending = []
|
12
|
+
@running = []
|
13
|
+
@targets = targets
|
14
|
+
@waiting = nil
|
15
|
+
end
|
16
|
+
|
17
|
+
def push(*items)
|
18
|
+
items.each do |item|
|
19
|
+
self << item
|
20
|
+
end
|
21
|
+
|
22
|
+
self
|
23
|
+
end
|
24
|
+
|
25
|
+
def <<(task)
|
26
|
+
raise "Object class #{task.class} not allowed" unless task.is_a? Task
|
27
|
+
raise "Task '#{task.name}' already defined" if @map[task.name]
|
28
|
+
|
29
|
+
@map[task.name] = task
|
30
|
+
@waiting = nil
|
31
|
+
super
|
32
|
+
end
|
33
|
+
|
34
|
+
def next
|
35
|
+
waiting.shift
|
36
|
+
end
|
37
|
+
|
38
|
+
def update(task)
|
39
|
+
pending.delete task
|
40
|
+
|
41
|
+
if task.state == :DONE
|
42
|
+
done << task
|
43
|
+
task.dependants.each { |dependant_task| dependant_task.depends.delete task }
|
44
|
+
|
45
|
+
move_pending_to_waiting
|
46
|
+
else
|
47
|
+
error << task
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def waiting
|
52
|
+
return @waiting if @waiting
|
53
|
+
|
54
|
+
reset
|
55
|
+
setup_dependencies
|
56
|
+
setup_scores
|
57
|
+
move_pending_to_waiting
|
58
|
+
end
|
59
|
+
|
60
|
+
def no_waiting?
|
61
|
+
waiting.empty?
|
62
|
+
end
|
63
|
+
|
64
|
+
def no_running?
|
65
|
+
running.empty?
|
66
|
+
end
|
67
|
+
|
68
|
+
def waiting?
|
69
|
+
!waiting.empty?
|
70
|
+
end
|
71
|
+
|
72
|
+
def running?
|
73
|
+
!running.empty?
|
74
|
+
end
|
75
|
+
|
76
|
+
def failed?
|
77
|
+
!failed.empty?
|
78
|
+
end
|
79
|
+
|
80
|
+
protected
|
81
|
+
|
82
|
+
def reset
|
83
|
+
by_state = group_by(&:state)
|
84
|
+
|
85
|
+
@done = by_state.delete(:DONE) || []
|
86
|
+
@failed = by_state.delete(:FAIL) || []
|
87
|
+
@running = by_state.delete(:RUNNING) || []
|
88
|
+
@waiting = []
|
89
|
+
|
90
|
+
return unless @targets.nil? || @targets.empty?
|
91
|
+
|
92
|
+
@pending = by_state.values.flatten.each do |task|
|
93
|
+
task.depends = []
|
94
|
+
task.dependants = []
|
95
|
+
task.state = nil
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
def setup_dependencies
|
100
|
+
@pending.each do |task|
|
101
|
+
next unless task.after
|
102
|
+
|
103
|
+
[task.after].flatten.each do |after|
|
104
|
+
after_task = after.is_a?(Task) ? after : @map[after]
|
105
|
+
raise "Dependency task '#{after}' not found on task '#{task.name}'." unless after_task
|
106
|
+
|
107
|
+
task.depends << after_task
|
108
|
+
after_task.dependants << task
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def setup_scores
|
114
|
+
@pending.each do |task|
|
115
|
+
task.run_score = task.dependants.count
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
def move_pending_to_waiting
|
120
|
+
to_move = pending.select { |task| task.depends.count.zero? }
|
121
|
+
|
122
|
+
@pending -= to_move
|
123
|
+
@waiting += to_move
|
124
|
+
@waiting.sort_by!.with_index do |x, index|
|
125
|
+
x.state = :WAIT
|
126
|
+
[-x.run_score, index]
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
data/lib/fasten/ui/console.rb
CHANGED
@@ -5,17 +5,15 @@ module Fasten
|
|
5
5
|
class Console
|
6
6
|
extend Forwardable
|
7
7
|
|
8
|
-
def_delegators :runner, :worker_list, :
|
8
|
+
def_delegators :runner, :worker_list, :tasks, :worker_list
|
9
9
|
def_delegators :runner, :name, :workers, :workers=, :state, :state=, :hformat
|
10
10
|
|
11
11
|
attr_accessor :runner
|
12
12
|
|
13
13
|
def initialize(runner:)
|
14
14
|
@runner = runner
|
15
|
-
@
|
16
|
-
|
17
|
-
task_error_list: []
|
18
|
-
}
|
15
|
+
@old_done = []
|
16
|
+
@old_failed = []
|
19
17
|
end
|
20
18
|
|
21
19
|
def setup
|
@@ -33,8 +31,9 @@ module Fasten
|
|
33
31
|
|
34
32
|
def update
|
35
33
|
setup unless @setup_done
|
36
|
-
|
37
|
-
display_task_message(
|
34
|
+
|
35
|
+
display_task_message(tasks.done, @old_done, 'Done in')
|
36
|
+
display_task_message(tasks.failed, @old_failed, 'Fail in')
|
38
37
|
end
|
39
38
|
|
40
39
|
def cleanup
|
data/lib/fasten/ui/curses.rb
CHANGED
@@ -9,7 +9,7 @@ module Fasten
|
|
9
9
|
include ::Curses
|
10
10
|
extend Forwardable
|
11
11
|
|
12
|
-
def_delegators :runner, :worker_list, :
|
12
|
+
def_delegators :runner, :worker_list, :tasks, :worker_list
|
13
13
|
def_delegators :runner, :name, :workers, :workers=, :state, :state=
|
14
14
|
|
15
15
|
attr_accessor :n_rows, :n_cols, :selected, :sel_index, :clear_needed, :message, :runner
|
@@ -106,11 +106,11 @@ module Fasten
|
|
106
106
|
self.workers += 1
|
107
107
|
self.message = "Increasing workers to #{workers}"
|
108
108
|
elsif key == Curses::Key::DOWN
|
109
|
-
self.sel_index = sel_index ? [sel_index + 1,
|
110
|
-
self.selected =
|
109
|
+
self.sel_index = sel_index ? [sel_index + 1, tasks.count - 1].min : 0
|
110
|
+
self.selected = tasks[sel_index]
|
111
111
|
elsif key == Curses::Key::UP
|
112
|
-
self.sel_index = sel_index ? [sel_index - 1, 0].max :
|
113
|
-
self.selected =
|
112
|
+
self.sel_index = sel_index ? [sel_index - 1, 0].max : tasks.count - 1
|
113
|
+
self.selected = tasks[sel_index]
|
114
114
|
elsif key == 'q'
|
115
115
|
self.message = 'Will quit when running tasks end'
|
116
116
|
self.state = :QUITTING
|
@@ -125,8 +125,8 @@ module Fasten
|
|
125
125
|
end
|
126
126
|
|
127
127
|
def ui_workers_summary
|
128
|
-
running_count =
|
129
|
-
waiting_count =
|
128
|
+
running_count = tasks.running.count
|
129
|
+
waiting_count = tasks.waiting.count
|
130
130
|
workers_count = worker_list.count
|
131
131
|
|
132
132
|
"Procs running: #{running_count} idle: #{workers_count - running_count} waiting: #{waiting_count} #{runner.use_threads ? 'threads' : 'processes'}: #{workers}"
|
@@ -196,15 +196,18 @@ module Fasten
|
|
196
196
|
|
197
197
|
def ui_task_color(task)
|
198
198
|
rev = task == selected ? A_REVERSE : 0
|
199
|
+
|
199
200
|
case task.state
|
200
201
|
when :RUNNING
|
201
202
|
color_pair(1) | A_TOP | rev
|
202
|
-
when :FAIL
|
203
|
-
color_pair(3) | A_TOP | rev
|
204
203
|
when :DONE
|
205
204
|
color_pair(2) | A_TOP | rev
|
205
|
+
when :FAIL
|
206
|
+
color_pair(3) | A_TOP | rev
|
207
|
+
when :WAIT
|
208
|
+
A_TOP | rev
|
206
209
|
else
|
207
|
-
|
210
|
+
color_pair(4) | A_DIM | rev
|
208
211
|
end
|
209
212
|
end
|
210
213
|
|
@@ -228,8 +231,8 @@ module Fasten
|
|
228
231
|
worker.spinner = (worker.spinner + 1) % SPINNER_LEN if worker.running?
|
229
232
|
end
|
230
233
|
|
231
|
-
count_done =
|
232
|
-
count_total =
|
234
|
+
count_done = tasks.done.count
|
235
|
+
count_total = tasks.count
|
233
236
|
tl = count_total.to_s.length
|
234
237
|
col_ini = ui_text_aligned(2, :left, format("Tasks %#{tl}d/%d", count_done, count_total)) + 1
|
235
238
|
col_fin = n_cols - 5
|
@@ -238,7 +241,7 @@ module Fasten
|
|
238
241
|
ui_progressbar(2, col_ini, col_fin, count_done, count_total)
|
239
242
|
|
240
243
|
max = 2
|
241
|
-
list =
|
244
|
+
list = tasks.sort_by.with_index { |x, index| [x.run_score, index] }
|
242
245
|
list.each_with_index do |task, index|
|
243
246
|
next if 3 + index >= n_rows
|
244
247
|
|
data/lib/fasten/version.rb
CHANGED
data/lib/fasten.rb
CHANGED
@@ -2,10 +2,13 @@
|
|
2
2
|
|
3
3
|
require 'optparse'
|
4
4
|
require 'fasten/task'
|
5
|
+
require 'fasten/task_manager'
|
5
6
|
require 'fasten/runner'
|
6
7
|
require 'fasten/worker'
|
7
8
|
require 'fasten/version'
|
8
9
|
|
10
|
+
require 'fasten/defaults'
|
11
|
+
|
9
12
|
module Fasten
|
10
13
|
class << self
|
11
14
|
def runner_from_yaml(path, **options)
|
@@ -35,28 +38,6 @@ module Fasten
|
|
35
38
|
runner(**options).register(&block)
|
36
39
|
end
|
37
40
|
|
38
|
-
def default_name
|
39
|
-
File.basename(Dir.getwd)
|
40
|
-
end
|
41
|
-
|
42
|
-
def default_workers
|
43
|
-
Parallel.physical_processor_count
|
44
|
-
end
|
45
|
-
|
46
|
-
def default_developer
|
47
|
-
STDIN.tty? && STDOUT.tty?
|
48
|
-
end
|
49
|
-
|
50
|
-
def default_ui_mode
|
51
|
-
return @default_ui_mode if defined? @default_ui_mode
|
52
|
-
|
53
|
-
require 'fasten/ui/curses'
|
54
|
-
|
55
|
-
@default_ui_mode = :curses
|
56
|
-
rescue StandardError, LoadError
|
57
|
-
@default_ui_mode = :console
|
58
|
-
end
|
59
|
-
|
60
41
|
def load_fasten(args)
|
61
42
|
args.each do |path|
|
62
43
|
if File.directory? path
|
@@ -107,10 +88,10 @@ module Fasten
|
|
107
88
|
opts.on '--ui=UI', String, "Type of UI: curses, console. (default: #{default_ui_mode} on this machine)" do |ui_mode|
|
108
89
|
@options[:ui_mode] = ui_mode
|
109
90
|
end
|
110
|
-
opts.on '-t', '--threads',
|
91
|
+
opts.on '-t', '--threads', "Use threads workers for parallel execution#{default_use_threads && ' (default on this machine)' || nil}" do
|
111
92
|
@options[:use_threads] = true
|
112
93
|
end
|
113
|
-
opts.on '-p', '--processes',
|
94
|
+
opts.on '-p', '--processes', "Use process workers for parallel execution#{!default_use_threads && ' (default on this machine)' || nil}" do
|
114
95
|
@options[:use_threads] = false
|
115
96
|
end
|
116
97
|
opts.on '-v', '--version', 'Display version info' do
|
@@ -118,15 +99,14 @@ module Fasten
|
|
118
99
|
exit 0
|
119
100
|
end
|
120
101
|
opts.on_tail '-h', '--help', 'Shows this help' do
|
121
|
-
|
122
|
-
exit 0
|
102
|
+
show_help
|
123
103
|
end
|
124
104
|
end
|
125
105
|
end
|
126
106
|
|
127
|
-
def show_help
|
107
|
+
def show_help(exit_code = 0)
|
128
108
|
puts opt_parser
|
129
|
-
exit
|
109
|
+
exit exit_code
|
130
110
|
end
|
131
111
|
|
132
112
|
def invoke
|
@@ -135,7 +115,7 @@ module Fasten
|
|
135
115
|
runner @options
|
136
116
|
load_fasten @load_path
|
137
117
|
|
138
|
-
show_help if runner.
|
118
|
+
show_help 1 if runner.tasks.empty?
|
139
119
|
|
140
120
|
runner.perform
|
141
121
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fasten
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Aldrin Martoq
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-11-
|
11
|
+
date: 2018-11-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -175,9 +175,9 @@ files:
|
|
175
175
|
- exe/fasten
|
176
176
|
- fasten.gemspec
|
177
177
|
- lib/fasten.rb
|
178
|
+
- lib/fasten/defaults.rb
|
178
179
|
- lib/fasten/runner.rb
|
179
180
|
- lib/fasten/std_thread_proxy.rb
|
180
|
-
- lib/fasten/support/dag.rb
|
181
181
|
- lib/fasten/support/fork_worker.rb
|
182
182
|
- lib/fasten/support/logger.rb
|
183
183
|
- lib/fasten/support/state.rb
|
@@ -186,6 +186,7 @@ files:
|
|
186
186
|
- lib/fasten/support/ui.rb
|
187
187
|
- lib/fasten/support/yaml.rb
|
188
188
|
- lib/fasten/task.rb
|
189
|
+
- lib/fasten/task_manager.rb
|
189
190
|
- lib/fasten/timeout_queue.rb
|
190
191
|
- lib/fasten/ui/console.rb
|
191
192
|
- lib/fasten/ui/curses.rb
|
data/lib/fasten/support/dag.rb
DELETED
@@ -1,130 +0,0 @@
|
|
1
|
-
module Fasten
|
2
|
-
module Support
|
3
|
-
module DAG
|
4
|
-
attr_reader :task_map, :task_list, :task_done_list, :task_error_list, :task_pending_list, :task_running_list
|
5
|
-
|
6
|
-
def initialize_dag
|
7
|
-
@task_map ||= {}
|
8
|
-
@task_list ||= []
|
9
|
-
@task_done_list ||= []
|
10
|
-
@task_error_list ||= []
|
11
|
-
@task_pending_list ||= []
|
12
|
-
@task_running_list ||= []
|
13
|
-
|
14
|
-
nil
|
15
|
-
end
|
16
|
-
|
17
|
-
def add(task)
|
18
|
-
raise "Task '#{task.name}' already defined" if @task_map[task.name]
|
19
|
-
|
20
|
-
@task_map[task.name] = task
|
21
|
-
@task_list << task
|
22
|
-
@task_waiting_list = nil
|
23
|
-
end
|
24
|
-
|
25
|
-
def update_task(task)
|
26
|
-
task.state == :DONE ? update_done_task(task) : update_error_task(task)
|
27
|
-
|
28
|
-
stats_add_entry(task.state, task)
|
29
|
-
end
|
30
|
-
|
31
|
-
def update_done_task(task)
|
32
|
-
@task_done_list << task
|
33
|
-
@task_pending_list.delete task
|
34
|
-
task.dependants.each { |dependant_task| dependant_task.depends.delete task }
|
35
|
-
|
36
|
-
move_pending_to_waiting
|
37
|
-
end
|
38
|
-
|
39
|
-
def update_error_task(task)
|
40
|
-
@task_error_list << task
|
41
|
-
@task_pending_list.delete task
|
42
|
-
end
|
43
|
-
|
44
|
-
def next_task
|
45
|
-
task_waiting_list.shift
|
46
|
-
end
|
47
|
-
|
48
|
-
def task_waiting_list
|
49
|
-
return @task_waiting_list if @task_waiting_list
|
50
|
-
|
51
|
-
reset_tasks
|
52
|
-
setup_tasks_dependencies
|
53
|
-
setup_tasks_scores
|
54
|
-
move_pending_to_waiting
|
55
|
-
end
|
56
|
-
|
57
|
-
protected
|
58
|
-
|
59
|
-
def move_pending_to_waiting
|
60
|
-
move_list = task_pending_list.select { |task| task.depends.count.zero? }
|
61
|
-
|
62
|
-
@task_waiting_list ||= []
|
63
|
-
@task_pending_list -= move_list
|
64
|
-
@task_waiting_list += move_list
|
65
|
-
@task_waiting_list.sort_by!.with_index do |x, index|
|
66
|
-
x.state = :WAIT
|
67
|
-
[-x.run_score, index]
|
68
|
-
end
|
69
|
-
end
|
70
|
-
|
71
|
-
def reset_tasks
|
72
|
-
@task_pending_list.clear
|
73
|
-
@task_done_list.clear
|
74
|
-
@task_error_list.clear
|
75
|
-
|
76
|
-
@task_list.each do |task|
|
77
|
-
task.dependants = []
|
78
|
-
task.depends = []
|
79
|
-
|
80
|
-
if task.state == :DONE
|
81
|
-
@task_done_list << task
|
82
|
-
elsif task.state == :FAIL
|
83
|
-
@task_error_list << task
|
84
|
-
else
|
85
|
-
task.state = :IDLE
|
86
|
-
@task_pending_list << task
|
87
|
-
end
|
88
|
-
end
|
89
|
-
end
|
90
|
-
|
91
|
-
def setup_tasks_dependencies
|
92
|
-
@task_pending_list.each do |task|
|
93
|
-
next unless task.after
|
94
|
-
|
95
|
-
[task.after].flatten.each do |after|
|
96
|
-
after_task = after.is_a?(Task) ? after : @task_map[after]
|
97
|
-
raise "Dependency task '#{after}' not found on task '#{task.name}'." unless after_task
|
98
|
-
|
99
|
-
task.depends << after_task
|
100
|
-
after_task.dependants << task
|
101
|
-
end
|
102
|
-
end
|
103
|
-
end
|
104
|
-
|
105
|
-
def setup_tasks_scores
|
106
|
-
@task_pending_list.each { |task| task.run_score = task.dependants.count }
|
107
|
-
end
|
108
|
-
|
109
|
-
def no_waiting_tasks?
|
110
|
-
task_waiting_list.empty?
|
111
|
-
end
|
112
|
-
|
113
|
-
def no_running_tasks?
|
114
|
-
task_running_list.empty?
|
115
|
-
end
|
116
|
-
|
117
|
-
def tasks_waiting?
|
118
|
-
!task_waiting_list.empty?
|
119
|
-
end
|
120
|
-
|
121
|
-
def tasks_running?
|
122
|
-
!task_running_list.empty?
|
123
|
-
end
|
124
|
-
|
125
|
-
def tasks_failed?
|
126
|
-
!task_error_list.empty?
|
127
|
-
end
|
128
|
-
end
|
129
|
-
end
|
130
|
-
end
|