fasten 0.7.2 → 0.7.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/bin/console +1 -1
- data/lib/fasten/defaults.rb +47 -0
- data/lib/fasten/runner.rb +38 -42
- data/lib/fasten/support/stats.rb +5 -5
- data/lib/fasten/support/yaml.rb +2 -2
- data/lib/fasten/task.rb +1 -1
- data/lib/fasten/task_manager.rb +130 -0
- data/lib/fasten/ui/console.rb +6 -7
- data/lib/fasten/ui/curses.rb +16 -13
- data/lib/fasten/version.rb +1 -1
- data/lib/fasten.rb +9 -29
- metadata +4 -3
- data/lib/fasten/support/dag.rb +0 -130
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c4790a7db3f8bdbbaee848efc77a62dbc944338363a678c57b233e865df47b34
|
4
|
+
data.tar.gz: 87b97a2473f12e69ab2bd619264b25b7a01664997ee2ff140000d2ba7c68c5e7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4ac38b2b2b77e25c8b1911f41c94b091ccf48827b034c78ff9a939442a77004ca9821cd124aae32cc6ee099f344c605c70631ce6aaa5f2bd299150e9ae5a2149
|
7
|
+
data.tar.gz: 88294128e454055b9bd322ed3dfa9a7e86223ba191110041ab4ba55ab688ece163f549b470ae4fd58db4d6a16eb75aaf2b30ffb90f2e51f7288d410cb0e8a17a
|
data/Gemfile.lock
CHANGED
data/bin/console
CHANGED
@@ -7,7 +7,7 @@ require 'fasten'
|
|
7
7
|
# with your gem easier. You can also use a different console, if you like.
|
8
8
|
|
9
9
|
# call reload! for reloading clases in development
|
10
|
-
def reload!(base = 'fasten')
|
10
|
+
def reload!(base = 'a0-fasten-ruby')
|
11
11
|
$LOADED_FEATURES.select { |feature| feature =~ %r{/#{base}/lib/} }.each do |feature|
|
12
12
|
puts "Updating #{feature.gsub(/.*#{base}/, base)}: #{load feature}"
|
13
13
|
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Fasten
|
4
|
+
class << self
|
5
|
+
def default_name
|
6
|
+
File.basename(Dir.getwd)
|
7
|
+
end
|
8
|
+
|
9
|
+
def default_stats
|
10
|
+
true
|
11
|
+
end
|
12
|
+
|
13
|
+
def default_summary
|
14
|
+
false
|
15
|
+
end
|
16
|
+
|
17
|
+
def default_workers
|
18
|
+
Parallel.physical_processor_count
|
19
|
+
end
|
20
|
+
|
21
|
+
def default_worker_class
|
22
|
+
Worker
|
23
|
+
end
|
24
|
+
|
25
|
+
def default_fasten_dir
|
26
|
+
'fasten'
|
27
|
+
end
|
28
|
+
|
29
|
+
def default_use_threads
|
30
|
+
!OS.posix?
|
31
|
+
end
|
32
|
+
|
33
|
+
def default_ui_mode
|
34
|
+
return @default_ui_mode if defined? @default_ui_mode
|
35
|
+
|
36
|
+
require 'fasten/ui/curses'
|
37
|
+
|
38
|
+
@default_ui_mode = :curses
|
39
|
+
rescue StandardError, LoadError
|
40
|
+
@default_ui_mode = :console
|
41
|
+
end
|
42
|
+
|
43
|
+
def default_developer
|
44
|
+
STDIN.tty? && STDOUT.tty?
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
data/lib/fasten/runner.rb
CHANGED
@@ -3,7 +3,6 @@ require 'parallel'
|
|
3
3
|
require 'pry'
|
4
4
|
require 'os'
|
5
5
|
|
6
|
-
require 'fasten/support/dag'
|
7
6
|
require 'fasten/support/logger'
|
8
7
|
require 'fasten/support/state'
|
9
8
|
require 'fasten/support/stats'
|
@@ -13,34 +12,29 @@ require 'fasten/timeout_queue'
|
|
13
12
|
|
14
13
|
module Fasten
|
15
14
|
class Runner
|
16
|
-
include Fasten::Support::DAG
|
17
15
|
include Fasten::Support::Logger
|
18
16
|
include Fasten::Support::State
|
19
17
|
include Fasten::Support::Stats
|
20
18
|
include Fasten::Support::UI
|
21
19
|
include Fasten::Support::Yaml
|
22
20
|
|
23
|
-
attr_accessor :name, :
|
21
|
+
attr_accessor :name, :stats, :summary, :workers, :worker_class, :fasten_dir, :use_threads, :ui_mode, :developer, :worker_list, :queue, :tasks
|
24
22
|
|
25
|
-
def initialize(
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
23
|
+
def initialize(**options)
|
24
|
+
%i[name stats summary workers worker_class fasten_dir use_threads ui_mode developer].each do |key|
|
25
|
+
options[key] = Fasten.send "default_#{key}" unless options.key? key
|
26
|
+
end
|
27
|
+
|
28
|
+
@tasks = TaskManager.new
|
29
|
+
|
30
|
+
reconfigure(options)
|
30
31
|
end
|
31
32
|
|
32
33
|
def reconfigure(**options)
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
self.fasten_dir = options[:fasten_dir] if options.key?(:fasten_dir)
|
38
|
-
self.developer = options[:developer] if options.key?(:developer)
|
39
|
-
self.use_threads = options[:use_threads] if options.key?(:use_threads)
|
40
|
-
self.summary = options[:summary] if options.key?(:summary)
|
41
|
-
self.ui_mode = options[:ui_mode] if options.key?(:ui_mode)
|
42
|
-
|
43
|
-
initialize_dag
|
34
|
+
%i[name stats summary workers worker_class fasten_dir use_threads ui_mode developer].each do |key|
|
35
|
+
send "#{key}=", options[key] if options.key? key
|
36
|
+
end
|
37
|
+
|
44
38
|
initialize_stats
|
45
39
|
initialize_logger
|
46
40
|
|
@@ -48,7 +42,9 @@ module Fasten
|
|
48
42
|
end
|
49
43
|
|
50
44
|
def task(name, **opts, &block)
|
51
|
-
|
45
|
+
tasks << task = Task.new(name: name, **opts, block: block)
|
46
|
+
|
47
|
+
task
|
52
48
|
end
|
53
49
|
|
54
50
|
def register(&block)
|
@@ -64,7 +60,7 @@ module Fasten
|
|
64
60
|
perform_loop
|
65
61
|
end
|
66
62
|
|
67
|
-
self.state =
|
63
|
+
self.state = tasks.map(&:state).all?(:DONE) ? :DONE : :FAIL
|
68
64
|
log_fin self, running_counters
|
69
65
|
|
70
66
|
stats_add_entry(state, self)
|
@@ -76,20 +72,20 @@ module Fasten
|
|
76
72
|
|
77
73
|
def map(list, &block)
|
78
74
|
list.each do |item|
|
79
|
-
|
75
|
+
task item.to_s, request: item, &block
|
80
76
|
end
|
81
77
|
|
82
78
|
perform
|
83
79
|
|
84
|
-
|
80
|
+
tasks.map(&:response)
|
85
81
|
end
|
86
82
|
|
87
83
|
def done_counters
|
88
|
-
"#{
|
84
|
+
"#{tasks.done.count}/#{tasks.count}"
|
89
85
|
end
|
90
86
|
|
91
87
|
def running_counters
|
92
|
-
"#{
|
88
|
+
"#{tasks.done.count + tasks.running.count}/#{tasks.count}"
|
93
89
|
end
|
94
90
|
|
95
91
|
def perform_loop
|
@@ -103,25 +99,25 @@ module Fasten
|
|
103
99
|
dispatch_pending_tasks
|
104
100
|
end
|
105
101
|
|
106
|
-
break if
|
102
|
+
break if tasks.no_running? && tasks.no_waiting? || state == :QUIT
|
107
103
|
end
|
108
104
|
|
109
105
|
remove_all_workers
|
110
106
|
end
|
111
107
|
|
112
108
|
def check_state
|
113
|
-
if state == :PAUSING &&
|
109
|
+
if state == :PAUSING && tasks.no_running?
|
114
110
|
self.state = :PAUSED
|
115
111
|
ui.message = nil
|
116
112
|
ui.force_clear
|
117
|
-
elsif state == :QUITTING &&
|
113
|
+
elsif state == :QUITTING && tasks.no_running?
|
118
114
|
self.state = :QUIT
|
119
115
|
ui.force_clear
|
120
116
|
end
|
121
117
|
end
|
122
118
|
|
123
119
|
def should_wait_for_running_tasks?
|
124
|
-
|
120
|
+
tasks.running? && (tasks.no_waiting? || tasks.failed? || %i[PAUSING QUITTING].include?(state)) || tasks.running.count >= workers
|
125
121
|
end
|
126
122
|
|
127
123
|
def wait_for_running_tasks
|
@@ -134,21 +130,20 @@ module Fasten
|
|
134
130
|
while should_wait_for_running_tasks?
|
135
131
|
ui.update
|
136
132
|
|
137
|
-
|
138
|
-
|
139
|
-
receive_workers_tasks_thread(tasks)
|
133
|
+
receive_workers_tasks_thread queue.receive_with_timeout(0.5)
|
140
134
|
end
|
141
135
|
|
142
136
|
ui.update
|
143
137
|
end
|
144
138
|
|
145
|
-
def receive_workers_tasks_thread(
|
146
|
-
|
147
|
-
|
139
|
+
def receive_workers_tasks_thread(items)
|
140
|
+
items&.each do |task|
|
141
|
+
tasks.running.delete task
|
148
142
|
|
149
143
|
task.worker.running_task = task.worker.state = nil
|
150
144
|
|
151
|
-
|
145
|
+
tasks.update task
|
146
|
+
stats_add_entry(task.state, task)
|
152
147
|
|
153
148
|
log_fin task, done_counters
|
154
149
|
ui.force_clear
|
@@ -173,9 +168,10 @@ module Fasten
|
|
173
168
|
|
174
169
|
task = worker.receive_response_from_child
|
175
170
|
|
176
|
-
|
171
|
+
tasks.running.delete task
|
177
172
|
|
178
|
-
|
173
|
+
tasks.update task
|
174
|
+
stats_add_entry(task.state, task)
|
179
175
|
|
180
176
|
log_fin task, done_counters
|
181
177
|
ui.force_clear
|
@@ -189,7 +185,7 @@ module Fasten
|
|
189
185
|
end
|
190
186
|
|
191
187
|
def raise_error_in_failure
|
192
|
-
return unless
|
188
|
+
return unless tasks.failed?
|
193
189
|
|
194
190
|
show_error_tasks
|
195
191
|
|
@@ -238,13 +234,13 @@ module Fasten
|
|
238
234
|
end
|
239
235
|
|
240
236
|
def dispatch_pending_tasks
|
241
|
-
while
|
237
|
+
while tasks.waiting? && tasks.running.count < workers
|
242
238
|
worker = find_or_create_worker
|
243
239
|
|
244
|
-
task =
|
240
|
+
task = tasks.next
|
245
241
|
log_ini task, "on worker #{worker}"
|
246
242
|
worker.send_request_to_child(task)
|
247
|
-
|
243
|
+
tasks.running << task
|
248
244
|
|
249
245
|
ui.force_clear
|
250
246
|
end
|
data/lib/fasten/support/stats.rb
CHANGED
@@ -25,7 +25,7 @@ module Fasten
|
|
25
25
|
stats_data << row.to_h
|
26
26
|
end
|
27
27
|
|
28
|
-
@
|
28
|
+
@tasks.waiting = nil
|
29
29
|
rescue StandardError
|
30
30
|
nil
|
31
31
|
end
|
@@ -127,10 +127,10 @@ module Fasten
|
|
127
127
|
end
|
128
128
|
|
129
129
|
def update_stats(history, entry)
|
130
|
-
entry['cnt'] =
|
131
|
-
entry['avg'] = avg = history.sum.to_f /
|
132
|
-
entry['std'] = std = Math.sqrt(history.
|
133
|
-
entry['err'] = std / Math.sqrt(
|
130
|
+
entry['cnt'] = cnt = history.size
|
131
|
+
entry['avg'] = avg = history.sum.to_f / cnt
|
132
|
+
entry['std'] = std = Math.sqrt(history.map { |x| (x - avg)**2 }.sum / cnt)
|
133
|
+
entry['err'] = std / Math.sqrt(cnt) if cnt.positive?
|
134
134
|
end
|
135
135
|
end
|
136
136
|
end
|
data/lib/fasten/support/yaml.rb
CHANGED
@@ -13,7 +13,7 @@ module Fasten
|
|
13
13
|
params = {}
|
14
14
|
end
|
15
15
|
|
16
|
-
|
16
|
+
task name, params
|
17
17
|
end
|
18
18
|
|
19
19
|
log_info "Loaded #{items.count} tasks from #{path}"
|
@@ -22,7 +22,7 @@ module Fasten
|
|
22
22
|
def save_yaml(path)
|
23
23
|
keys = %i[after shell]
|
24
24
|
|
25
|
-
items =
|
25
|
+
items = tasks.map do |task|
|
26
26
|
data = task.to_h.select do |key, _val|
|
27
27
|
keys.include? key
|
28
28
|
end
|
data/lib/fasten/task.rb
CHANGED
@@ -7,7 +7,7 @@ module Fasten
|
|
7
7
|
attr_accessor :name, :after, :shell, :ruby
|
8
8
|
attr_accessor :dependants, :depends, :request, :response, :worker, :run_score, :block
|
9
9
|
|
10
|
-
def initialize(name
|
10
|
+
def initialize(name:, shell: nil, ruby: nil, block: nil, request: nil, after: nil)
|
11
11
|
self.name = name
|
12
12
|
self.shell = shell
|
13
13
|
self.ruby = ruby
|
@@ -0,0 +1,130 @@
|
|
1
|
+
module Fasten
|
2
|
+
class TaskManager < Array
|
3
|
+
attr_reader :done, :failed, :pending, :running, :targets
|
4
|
+
|
5
|
+
def initialize(targets: nil)
|
6
|
+
super()
|
7
|
+
|
8
|
+
@map = {}
|
9
|
+
@done = []
|
10
|
+
@failed = []
|
11
|
+
@pending = []
|
12
|
+
@running = []
|
13
|
+
@targets = targets
|
14
|
+
@waiting = nil
|
15
|
+
end
|
16
|
+
|
17
|
+
def push(*items)
|
18
|
+
items.each do |item|
|
19
|
+
self << item
|
20
|
+
end
|
21
|
+
|
22
|
+
self
|
23
|
+
end
|
24
|
+
|
25
|
+
def <<(task)
|
26
|
+
raise "Object class #{task.class} not allowed" unless task.is_a? Task
|
27
|
+
raise "Task '#{task.name}' already defined" if @map[task.name]
|
28
|
+
|
29
|
+
@map[task.name] = task
|
30
|
+
@waiting = nil
|
31
|
+
super
|
32
|
+
end
|
33
|
+
|
34
|
+
def next
|
35
|
+
waiting.shift
|
36
|
+
end
|
37
|
+
|
38
|
+
def update(task)
|
39
|
+
pending.delete task
|
40
|
+
|
41
|
+
if task.state == :DONE
|
42
|
+
done << task
|
43
|
+
task.dependants.each { |dependant_task| dependant_task.depends.delete task }
|
44
|
+
|
45
|
+
move_pending_to_waiting
|
46
|
+
else
|
47
|
+
error << task
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def waiting
|
52
|
+
return @waiting if @waiting
|
53
|
+
|
54
|
+
reset
|
55
|
+
setup_dependencies
|
56
|
+
setup_scores
|
57
|
+
move_pending_to_waiting
|
58
|
+
end
|
59
|
+
|
60
|
+
def no_waiting?
|
61
|
+
waiting.empty?
|
62
|
+
end
|
63
|
+
|
64
|
+
def no_running?
|
65
|
+
running.empty?
|
66
|
+
end
|
67
|
+
|
68
|
+
def waiting?
|
69
|
+
!waiting.empty?
|
70
|
+
end
|
71
|
+
|
72
|
+
def running?
|
73
|
+
!running.empty?
|
74
|
+
end
|
75
|
+
|
76
|
+
def failed?
|
77
|
+
!failed.empty?
|
78
|
+
end
|
79
|
+
|
80
|
+
protected
|
81
|
+
|
82
|
+
def reset
|
83
|
+
by_state = group_by(&:state)
|
84
|
+
|
85
|
+
@done = by_state.delete(:DONE) || []
|
86
|
+
@failed = by_state.delete(:FAIL) || []
|
87
|
+
@running = by_state.delete(:RUNNING) || []
|
88
|
+
@waiting = []
|
89
|
+
|
90
|
+
return unless @targets.nil? || @targets.empty?
|
91
|
+
|
92
|
+
@pending = by_state.values.flatten.each do |task|
|
93
|
+
task.depends = []
|
94
|
+
task.dependants = []
|
95
|
+
task.state = nil
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
def setup_dependencies
|
100
|
+
@pending.each do |task|
|
101
|
+
next unless task.after
|
102
|
+
|
103
|
+
[task.after].flatten.each do |after|
|
104
|
+
after_task = after.is_a?(Task) ? after : @map[after]
|
105
|
+
raise "Dependency task '#{after}' not found on task '#{task.name}'." unless after_task
|
106
|
+
|
107
|
+
task.depends << after_task
|
108
|
+
after_task.dependants << task
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def setup_scores
|
114
|
+
@pending.each do |task|
|
115
|
+
task.run_score = task.dependants.count
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
def move_pending_to_waiting
|
120
|
+
to_move = pending.select { |task| task.depends.count.zero? }
|
121
|
+
|
122
|
+
@pending -= to_move
|
123
|
+
@waiting += to_move
|
124
|
+
@waiting.sort_by!.with_index do |x, index|
|
125
|
+
x.state = :WAIT
|
126
|
+
[-x.run_score, index]
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
data/lib/fasten/ui/console.rb
CHANGED
@@ -5,17 +5,15 @@ module Fasten
|
|
5
5
|
class Console
|
6
6
|
extend Forwardable
|
7
7
|
|
8
|
-
def_delegators :runner, :worker_list, :
|
8
|
+
def_delegators :runner, :worker_list, :tasks, :worker_list
|
9
9
|
def_delegators :runner, :name, :workers, :workers=, :state, :state=, :hformat
|
10
10
|
|
11
11
|
attr_accessor :runner
|
12
12
|
|
13
13
|
def initialize(runner:)
|
14
14
|
@runner = runner
|
15
|
-
@
|
16
|
-
|
17
|
-
task_error_list: []
|
18
|
-
}
|
15
|
+
@old_done = []
|
16
|
+
@old_failed = []
|
19
17
|
end
|
20
18
|
|
21
19
|
def setup
|
@@ -33,8 +31,9 @@ module Fasten
|
|
33
31
|
|
34
32
|
def update
|
35
33
|
setup unless @setup_done
|
36
|
-
|
37
|
-
display_task_message(
|
34
|
+
|
35
|
+
display_task_message(tasks.done, @old_done, 'Done in')
|
36
|
+
display_task_message(tasks.failed, @old_failed, 'Fail in')
|
38
37
|
end
|
39
38
|
|
40
39
|
def cleanup
|
data/lib/fasten/ui/curses.rb
CHANGED
@@ -9,7 +9,7 @@ module Fasten
|
|
9
9
|
include ::Curses
|
10
10
|
extend Forwardable
|
11
11
|
|
12
|
-
def_delegators :runner, :worker_list, :
|
12
|
+
def_delegators :runner, :worker_list, :tasks, :worker_list
|
13
13
|
def_delegators :runner, :name, :workers, :workers=, :state, :state=
|
14
14
|
|
15
15
|
attr_accessor :n_rows, :n_cols, :selected, :sel_index, :clear_needed, :message, :runner
|
@@ -106,11 +106,11 @@ module Fasten
|
|
106
106
|
self.workers += 1
|
107
107
|
self.message = "Increasing workers to #{workers}"
|
108
108
|
elsif key == Curses::Key::DOWN
|
109
|
-
self.sel_index = sel_index ? [sel_index + 1,
|
110
|
-
self.selected =
|
109
|
+
self.sel_index = sel_index ? [sel_index + 1, tasks.count - 1].min : 0
|
110
|
+
self.selected = tasks[sel_index]
|
111
111
|
elsif key == Curses::Key::UP
|
112
|
-
self.sel_index = sel_index ? [sel_index - 1, 0].max :
|
113
|
-
self.selected =
|
112
|
+
self.sel_index = sel_index ? [sel_index - 1, 0].max : tasks.count - 1
|
113
|
+
self.selected = tasks[sel_index]
|
114
114
|
elsif key == 'q'
|
115
115
|
self.message = 'Will quit when running tasks end'
|
116
116
|
self.state = :QUITTING
|
@@ -125,8 +125,8 @@ module Fasten
|
|
125
125
|
end
|
126
126
|
|
127
127
|
def ui_workers_summary
|
128
|
-
running_count =
|
129
|
-
waiting_count =
|
128
|
+
running_count = tasks.running.count
|
129
|
+
waiting_count = tasks.waiting.count
|
130
130
|
workers_count = worker_list.count
|
131
131
|
|
132
132
|
"Procs running: #{running_count} idle: #{workers_count - running_count} waiting: #{waiting_count} #{runner.use_threads ? 'threads' : 'processes'}: #{workers}"
|
@@ -196,15 +196,18 @@ module Fasten
|
|
196
196
|
|
197
197
|
def ui_task_color(task)
|
198
198
|
rev = task == selected ? A_REVERSE : 0
|
199
|
+
|
199
200
|
case task.state
|
200
201
|
when :RUNNING
|
201
202
|
color_pair(1) | A_TOP | rev
|
202
|
-
when :FAIL
|
203
|
-
color_pair(3) | A_TOP | rev
|
204
203
|
when :DONE
|
205
204
|
color_pair(2) | A_TOP | rev
|
205
|
+
when :FAIL
|
206
|
+
color_pair(3) | A_TOP | rev
|
207
|
+
when :WAIT
|
208
|
+
A_TOP | rev
|
206
209
|
else
|
207
|
-
|
210
|
+
color_pair(4) | A_DIM | rev
|
208
211
|
end
|
209
212
|
end
|
210
213
|
|
@@ -228,8 +231,8 @@ module Fasten
|
|
228
231
|
worker.spinner = (worker.spinner + 1) % SPINNER_LEN if worker.running?
|
229
232
|
end
|
230
233
|
|
231
|
-
count_done =
|
232
|
-
count_total =
|
234
|
+
count_done = tasks.done.count
|
235
|
+
count_total = tasks.count
|
233
236
|
tl = count_total.to_s.length
|
234
237
|
col_ini = ui_text_aligned(2, :left, format("Tasks %#{tl}d/%d", count_done, count_total)) + 1
|
235
238
|
col_fin = n_cols - 5
|
@@ -238,7 +241,7 @@ module Fasten
|
|
238
241
|
ui_progressbar(2, col_ini, col_fin, count_done, count_total)
|
239
242
|
|
240
243
|
max = 2
|
241
|
-
list =
|
244
|
+
list = tasks.sort_by.with_index { |x, index| [x.run_score, index] }
|
242
245
|
list.each_with_index do |task, index|
|
243
246
|
next if 3 + index >= n_rows
|
244
247
|
|
data/lib/fasten/version.rb
CHANGED
data/lib/fasten.rb
CHANGED
@@ -2,10 +2,13 @@
|
|
2
2
|
|
3
3
|
require 'optparse'
|
4
4
|
require 'fasten/task'
|
5
|
+
require 'fasten/task_manager'
|
5
6
|
require 'fasten/runner'
|
6
7
|
require 'fasten/worker'
|
7
8
|
require 'fasten/version'
|
8
9
|
|
10
|
+
require 'fasten/defaults'
|
11
|
+
|
9
12
|
module Fasten
|
10
13
|
class << self
|
11
14
|
def runner_from_yaml(path, **options)
|
@@ -35,28 +38,6 @@ module Fasten
|
|
35
38
|
runner(**options).register(&block)
|
36
39
|
end
|
37
40
|
|
38
|
-
def default_name
|
39
|
-
File.basename(Dir.getwd)
|
40
|
-
end
|
41
|
-
|
42
|
-
def default_workers
|
43
|
-
Parallel.physical_processor_count
|
44
|
-
end
|
45
|
-
|
46
|
-
def default_developer
|
47
|
-
STDIN.tty? && STDOUT.tty?
|
48
|
-
end
|
49
|
-
|
50
|
-
def default_ui_mode
|
51
|
-
return @default_ui_mode if defined? @default_ui_mode
|
52
|
-
|
53
|
-
require 'fasten/ui/curses'
|
54
|
-
|
55
|
-
@default_ui_mode = :curses
|
56
|
-
rescue StandardError, LoadError
|
57
|
-
@default_ui_mode = :console
|
58
|
-
end
|
59
|
-
|
60
41
|
def load_fasten(args)
|
61
42
|
args.each do |path|
|
62
43
|
if File.directory? path
|
@@ -107,10 +88,10 @@ module Fasten
|
|
107
88
|
opts.on '--ui=UI', String, "Type of UI: curses, console. (default: #{default_ui_mode} on this machine)" do |ui_mode|
|
108
89
|
@options[:ui_mode] = ui_mode
|
109
90
|
end
|
110
|
-
opts.on '-t', '--threads',
|
91
|
+
opts.on '-t', '--threads', "Use threads workers for parallel execution#{default_use_threads && ' (default on this machine)' || nil}" do
|
111
92
|
@options[:use_threads] = true
|
112
93
|
end
|
113
|
-
opts.on '-p', '--processes',
|
94
|
+
opts.on '-p', '--processes', "Use process workers for parallel execution#{!default_use_threads && ' (default on this machine)' || nil}" do
|
114
95
|
@options[:use_threads] = false
|
115
96
|
end
|
116
97
|
opts.on '-v', '--version', 'Display version info' do
|
@@ -118,15 +99,14 @@ module Fasten
|
|
118
99
|
exit 0
|
119
100
|
end
|
120
101
|
opts.on_tail '-h', '--help', 'Shows this help' do
|
121
|
-
|
122
|
-
exit 0
|
102
|
+
show_help
|
123
103
|
end
|
124
104
|
end
|
125
105
|
end
|
126
106
|
|
127
|
-
def show_help
|
107
|
+
def show_help(exit_code = 0)
|
128
108
|
puts opt_parser
|
129
|
-
exit
|
109
|
+
exit exit_code
|
130
110
|
end
|
131
111
|
|
132
112
|
def invoke
|
@@ -135,7 +115,7 @@ module Fasten
|
|
135
115
|
runner @options
|
136
116
|
load_fasten @load_path
|
137
117
|
|
138
|
-
show_help if runner.
|
118
|
+
show_help 1 if runner.tasks.empty?
|
139
119
|
|
140
120
|
runner.perform
|
141
121
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fasten
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Aldrin Martoq
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-11-
|
11
|
+
date: 2018-11-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -175,9 +175,9 @@ files:
|
|
175
175
|
- exe/fasten
|
176
176
|
- fasten.gemspec
|
177
177
|
- lib/fasten.rb
|
178
|
+
- lib/fasten/defaults.rb
|
178
179
|
- lib/fasten/runner.rb
|
179
180
|
- lib/fasten/std_thread_proxy.rb
|
180
|
-
- lib/fasten/support/dag.rb
|
181
181
|
- lib/fasten/support/fork_worker.rb
|
182
182
|
- lib/fasten/support/logger.rb
|
183
183
|
- lib/fasten/support/state.rb
|
@@ -186,6 +186,7 @@ files:
|
|
186
186
|
- lib/fasten/support/ui.rb
|
187
187
|
- lib/fasten/support/yaml.rb
|
188
188
|
- lib/fasten/task.rb
|
189
|
+
- lib/fasten/task_manager.rb
|
189
190
|
- lib/fasten/timeout_queue.rb
|
190
191
|
- lib/fasten/ui/console.rb
|
191
192
|
- lib/fasten/ui/curses.rb
|
data/lib/fasten/support/dag.rb
DELETED
@@ -1,130 +0,0 @@
|
|
1
|
-
module Fasten
|
2
|
-
module Support
|
3
|
-
module DAG
|
4
|
-
attr_reader :task_map, :task_list, :task_done_list, :task_error_list, :task_pending_list, :task_running_list
|
5
|
-
|
6
|
-
def initialize_dag
|
7
|
-
@task_map ||= {}
|
8
|
-
@task_list ||= []
|
9
|
-
@task_done_list ||= []
|
10
|
-
@task_error_list ||= []
|
11
|
-
@task_pending_list ||= []
|
12
|
-
@task_running_list ||= []
|
13
|
-
|
14
|
-
nil
|
15
|
-
end
|
16
|
-
|
17
|
-
def add(task)
|
18
|
-
raise "Task '#{task.name}' already defined" if @task_map[task.name]
|
19
|
-
|
20
|
-
@task_map[task.name] = task
|
21
|
-
@task_list << task
|
22
|
-
@task_waiting_list = nil
|
23
|
-
end
|
24
|
-
|
25
|
-
def update_task(task)
|
26
|
-
task.state == :DONE ? update_done_task(task) : update_error_task(task)
|
27
|
-
|
28
|
-
stats_add_entry(task.state, task)
|
29
|
-
end
|
30
|
-
|
31
|
-
def update_done_task(task)
|
32
|
-
@task_done_list << task
|
33
|
-
@task_pending_list.delete task
|
34
|
-
task.dependants.each { |dependant_task| dependant_task.depends.delete task }
|
35
|
-
|
36
|
-
move_pending_to_waiting
|
37
|
-
end
|
38
|
-
|
39
|
-
def update_error_task(task)
|
40
|
-
@task_error_list << task
|
41
|
-
@task_pending_list.delete task
|
42
|
-
end
|
43
|
-
|
44
|
-
def next_task
|
45
|
-
task_waiting_list.shift
|
46
|
-
end
|
47
|
-
|
48
|
-
def task_waiting_list
|
49
|
-
return @task_waiting_list if @task_waiting_list
|
50
|
-
|
51
|
-
reset_tasks
|
52
|
-
setup_tasks_dependencies
|
53
|
-
setup_tasks_scores
|
54
|
-
move_pending_to_waiting
|
55
|
-
end
|
56
|
-
|
57
|
-
protected
|
58
|
-
|
59
|
-
def move_pending_to_waiting
|
60
|
-
move_list = task_pending_list.select { |task| task.depends.count.zero? }
|
61
|
-
|
62
|
-
@task_waiting_list ||= []
|
63
|
-
@task_pending_list -= move_list
|
64
|
-
@task_waiting_list += move_list
|
65
|
-
@task_waiting_list.sort_by!.with_index do |x, index|
|
66
|
-
x.state = :WAIT
|
67
|
-
[-x.run_score, index]
|
68
|
-
end
|
69
|
-
end
|
70
|
-
|
71
|
-
def reset_tasks
|
72
|
-
@task_pending_list.clear
|
73
|
-
@task_done_list.clear
|
74
|
-
@task_error_list.clear
|
75
|
-
|
76
|
-
@task_list.each do |task|
|
77
|
-
task.dependants = []
|
78
|
-
task.depends = []
|
79
|
-
|
80
|
-
if task.state == :DONE
|
81
|
-
@task_done_list << task
|
82
|
-
elsif task.state == :FAIL
|
83
|
-
@task_error_list << task
|
84
|
-
else
|
85
|
-
task.state = :IDLE
|
86
|
-
@task_pending_list << task
|
87
|
-
end
|
88
|
-
end
|
89
|
-
end
|
90
|
-
|
91
|
-
def setup_tasks_dependencies
|
92
|
-
@task_pending_list.each do |task|
|
93
|
-
next unless task.after
|
94
|
-
|
95
|
-
[task.after].flatten.each do |after|
|
96
|
-
after_task = after.is_a?(Task) ? after : @task_map[after]
|
97
|
-
raise "Dependency task '#{after}' not found on task '#{task.name}'." unless after_task
|
98
|
-
|
99
|
-
task.depends << after_task
|
100
|
-
after_task.dependants << task
|
101
|
-
end
|
102
|
-
end
|
103
|
-
end
|
104
|
-
|
105
|
-
def setup_tasks_scores
|
106
|
-
@task_pending_list.each { |task| task.run_score = task.dependants.count }
|
107
|
-
end
|
108
|
-
|
109
|
-
def no_waiting_tasks?
|
110
|
-
task_waiting_list.empty?
|
111
|
-
end
|
112
|
-
|
113
|
-
def no_running_tasks?
|
114
|
-
task_running_list.empty?
|
115
|
-
end
|
116
|
-
|
117
|
-
def tasks_waiting?
|
118
|
-
!task_waiting_list.empty?
|
119
|
-
end
|
120
|
-
|
121
|
-
def tasks_running?
|
122
|
-
!task_running_list.empty?
|
123
|
-
end
|
124
|
-
|
125
|
-
def tasks_failed?
|
126
|
-
!task_error_list.empty?
|
127
|
-
end
|
128
|
-
end
|
129
|
-
end
|
130
|
-
end
|