fasten 0.5.2 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/lib/fasten/dag.rb +16 -2
- data/lib/fasten/executor.rb +28 -16
- data/lib/fasten/logger.rb +11 -2
- data/lib/fasten/state.rb +30 -0
- data/lib/fasten/stats.rb +71 -22
- data/lib/fasten/task.rb +21 -1
- data/lib/fasten/ui/console.rb +1 -0
- data/lib/fasten/ui/curses.rb +29 -15
- data/lib/fasten/version.rb +1 -1
- data/lib/fasten/worker.rb +16 -17
- data/lib/fasten/yaml.rb +48 -0
- data/lib/fasten.rb +4 -3
- metadata +4 -3
- data/lib/fasten/load_save.rb +0 -73
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e475b26b22ab143005ca207f29ee7036571b71934a7fa2bba0ea2be74ba1aaab
|
4
|
+
data.tar.gz: 39429fea342cf0c43e1b84cef14e897a1d12f56a5f11e4c555c28176bfa1cdda
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7bd2923a0ae1d4cf5f31b36a3999be1bfec058a99acdd381712e91454e2ed9baf7193bb87de835359b300a9317cdd47f2c92e850c54a36e7dcf26dd88776d720
|
7
|
+
data.tar.gz: b4307bc7ee694ad3740b0c0b361d4c5a4888f5e405d32b6aa91ceee8c0023ee1a6421771a6915a2f3916d0610cdf5136f329f47076e33bb066e7e3eb4fb572e3
|
data/Gemfile.lock
CHANGED
data/lib/fasten/dag.rb
CHANGED
@@ -45,7 +45,7 @@ module Fasten
|
|
45
45
|
end
|
46
46
|
|
47
47
|
def next_task
|
48
|
-
task_waiting_list.
|
48
|
+
task_waiting_list.shift
|
49
49
|
end
|
50
50
|
|
51
51
|
def task_waiting_list
|
@@ -67,16 +67,30 @@ module Fasten
|
|
67
67
|
@task_waiting_list ||= []
|
68
68
|
@task_pending_list -= move_list
|
69
69
|
@task_waiting_list += move_list
|
70
|
+
@task_waiting_list.sort_by!.with_index do |x, index|
|
71
|
+
x.state = :WAIT
|
72
|
+
[-x.run_score, index]
|
73
|
+
end
|
70
74
|
end
|
71
75
|
|
72
76
|
def reset_tasks
|
73
77
|
@task_pending_list.clear
|
74
78
|
@task_done_list.clear
|
79
|
+
@task_error_list.clear
|
80
|
+
|
75
81
|
@task_list.each do |task|
|
76
82
|
task.dependants = []
|
77
83
|
task.depends = []
|
78
84
|
task.run_score = 0
|
79
|
-
|
85
|
+
|
86
|
+
if task.state == :DONE
|
87
|
+
@task_done_list << task
|
88
|
+
elsif task.state == :FAIL
|
89
|
+
@task_error_list << task
|
90
|
+
else
|
91
|
+
task.state = :IDLE
|
92
|
+
@task_pending_list << task
|
93
|
+
end
|
80
94
|
end
|
81
95
|
end
|
82
96
|
|
data/lib/fasten/executor.rb
CHANGED
@@ -1,21 +1,28 @@
|
|
1
1
|
module Fasten
|
2
|
-
class Executor
|
2
|
+
class Executor
|
3
3
|
include Fasten::Logger
|
4
|
+
include Fasten::State
|
4
5
|
include Fasten::DAG
|
5
6
|
include Fasten::UI
|
6
|
-
include Fasten::
|
7
|
+
include Fasten::Yaml
|
7
8
|
include Fasten::Stats
|
8
9
|
|
9
|
-
|
10
|
-
|
11
|
-
|
10
|
+
attr_accessor :name, :workers, :worker_class, :pid, :fasten_dir, :developer, :stats, :worker_list, :block
|
11
|
+
|
12
|
+
def initialize(name: nil, developer: STDIN.tty? && STDOUT.tty?, workers: Parallel.physical_processor_count, worker_class: Worker, fasten_dir: '.fasten')
|
13
|
+
self.stats = name && true
|
14
|
+
self.name = name || "#{self.class} #{$PID}"
|
15
|
+
self.state = :IDLE
|
16
|
+
self.workers = workers
|
17
|
+
self.worker_class = worker_class
|
18
|
+
self.fasten_dir = fasten_dir
|
19
|
+
self.developer = developer
|
20
|
+
|
12
21
|
initialize_dag
|
22
|
+
initialize_stats
|
23
|
+
initialize_logger
|
13
24
|
|
14
25
|
self.worker_list = []
|
15
|
-
log_path = "#{fasten_dir}/log/executor/#{self.name}.log"
|
16
|
-
FileUtils.mkdir_p File.dirname(log_path)
|
17
|
-
self.log_file = File.new(log_path, 'a')
|
18
|
-
Fasten.logger.reopen log_file
|
19
26
|
end
|
20
27
|
|
21
28
|
def perform
|
@@ -62,7 +69,7 @@ module Fasten
|
|
62
69
|
def check_state
|
63
70
|
if state == :PAUSING && no_running_tasks?
|
64
71
|
self.state = :PAUSED
|
65
|
-
|
72
|
+
ui.message = nil
|
66
73
|
ui.force_clear
|
67
74
|
elsif state == :QUITTING && no_running_tasks?
|
68
75
|
self.state = :QUIT
|
@@ -101,17 +108,22 @@ module Fasten
|
|
101
108
|
end
|
102
109
|
end
|
103
110
|
|
104
|
-
def
|
105
|
-
return unless tasks_failed?
|
106
|
-
|
111
|
+
def show_error_tasks
|
107
112
|
task_error_list.each do |task|
|
108
113
|
log_info "task: #{task} error:#{task.error}\n#{task.error&.backtrace&.join("\n")}"
|
109
114
|
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def raise_error_in_failure
|
118
|
+
return unless tasks_failed?
|
119
|
+
|
120
|
+
show_error_tasks
|
121
|
+
|
122
|
+
message = "Stopping because the following tasks failed: #{task_error_list.map(&:to_s).join(', ')}"
|
110
123
|
|
111
124
|
if developer
|
112
125
|
ui.cleanup
|
113
|
-
puts
|
114
|
-
task_error_list.map(&:to_s).each { |x| puts " #{x}" }
|
126
|
+
puts message
|
115
127
|
|
116
128
|
puts 'Entering development console'
|
117
129
|
|
@@ -119,7 +131,7 @@ module Fasten
|
|
119
131
|
else
|
120
132
|
remove_all_workers
|
121
133
|
|
122
|
-
raise
|
134
|
+
raise message
|
123
135
|
end
|
124
136
|
end
|
125
137
|
|
data/lib/fasten/logger.rb
CHANGED
@@ -4,6 +4,8 @@ module Fasten
|
|
4
4
|
end
|
5
5
|
|
6
6
|
module Logger
|
7
|
+
attr_accessor :log_file
|
8
|
+
|
7
9
|
%w[debug info error].each do |method|
|
8
10
|
define_method "log_#{method}" do |msg|
|
9
11
|
return unless Fasten.logger.respond_to?(method)
|
@@ -14,16 +16,23 @@ module Fasten
|
|
14
16
|
end
|
15
17
|
end
|
16
18
|
|
19
|
+
def initialize_logger
|
20
|
+
log_path = "#{fasten_dir}/log/executor/#{name}.log"
|
21
|
+
FileUtils.mkdir_p File.dirname(log_path)
|
22
|
+
self.log_file = File.new(log_path, 'a')
|
23
|
+
Fasten.logger.reopen log_file
|
24
|
+
end
|
25
|
+
|
17
26
|
def log_ini(object, message = nil)
|
18
27
|
object.ini ||= Time.new
|
19
|
-
log_info "
|
28
|
+
log_info "Ini #{object.state} #{object.class} #{object} #{message}"
|
20
29
|
end
|
21
30
|
|
22
31
|
def log_fin(object, message = nil)
|
23
32
|
object.fin ||= Time.new
|
24
33
|
object.dif = object.fin - object.ini
|
25
34
|
|
26
|
-
log_info "
|
35
|
+
log_info "Fin #{object.state} #{object.class} #{object} #{message} in #{object.dif}"
|
27
36
|
end
|
28
37
|
|
29
38
|
def redirect_std(path)
|
data/lib/fasten/state.rb
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
module Fasten
|
2
|
+
module State
|
3
|
+
attr_accessor :error, :ini, :fin, :dif, :last
|
4
|
+
attr_writer :state
|
5
|
+
|
6
|
+
def state
|
7
|
+
@state || :IDLE
|
8
|
+
end
|
9
|
+
|
10
|
+
def running?
|
11
|
+
state == :RUNNING
|
12
|
+
end
|
13
|
+
|
14
|
+
def idle?
|
15
|
+
state == :IDLE
|
16
|
+
end
|
17
|
+
|
18
|
+
def pausing?
|
19
|
+
state == :PAUSING
|
20
|
+
end
|
21
|
+
|
22
|
+
def paused?
|
23
|
+
state == :PAUSED
|
24
|
+
end
|
25
|
+
|
26
|
+
def quitting?
|
27
|
+
state == :QUITTING
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
data/lib/fasten/stats.rb
CHANGED
@@ -1,30 +1,78 @@
|
|
1
1
|
module Fasten
|
2
2
|
module Stats
|
3
|
+
attr_writer :stats_data, :stats_entries
|
4
|
+
attr_reader :stats_path
|
5
|
+
|
6
|
+
def initialize_stats
|
7
|
+
return unless stats
|
8
|
+
|
9
|
+
@stats_path = "#{ENV['HOME']}/.fasten/stats/#{name}.csv" if ENV['HOME']
|
10
|
+
FileUtils.mkdir_p File.dirname(@stats_path)
|
11
|
+
rescue StandardError
|
12
|
+
@stats_path = nil
|
13
|
+
end
|
14
|
+
|
15
|
+
def load_stats
|
16
|
+
return unless @stats_path && File.exist?(@stats_path)
|
17
|
+
|
18
|
+
self.stats_data = []
|
19
|
+
CSV.foreach(@stats_path, headers: true) do |row|
|
20
|
+
stats_data << row.to_h
|
21
|
+
end
|
22
|
+
|
23
|
+
@task_waiting_list = nil
|
24
|
+
rescue StandardError
|
25
|
+
nil
|
26
|
+
ensure
|
27
|
+
self.stats ||= {}
|
28
|
+
end
|
29
|
+
|
30
|
+
def save_stats
|
31
|
+
return unless @stats_path && stats_data
|
32
|
+
|
33
|
+
keys = %w[state kind name run cnt avg std err]
|
34
|
+
|
35
|
+
CSV.open(@stats_path, 'wb') do |csv|
|
36
|
+
csv << keys
|
37
|
+
|
38
|
+
stats_data.each do |data|
|
39
|
+
csv << keys.map { |i| data[i] }
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
3
44
|
def stats_create_entry(state, target)
|
4
|
-
{
|
5
|
-
'
|
6
|
-
'
|
7
|
-
'
|
8
|
-
'
|
9
|
-
'
|
10
|
-
'
|
11
|
-
|
45
|
+
{ 'state' => state.to_s,
|
46
|
+
'kind' => stats_kind_for(target),
|
47
|
+
'name' => target.name,
|
48
|
+
'ini' => target.ini.to_f,
|
49
|
+
'fin' => target.fin.to_f,
|
50
|
+
'run' => target.fin - target.ini,
|
51
|
+
'worker' => target.respond_to?(:worker) ? target.worker.name : nil }
|
52
|
+
end
|
53
|
+
|
54
|
+
def stats_data
|
55
|
+
@stats_data ||= []
|
56
|
+
end
|
57
|
+
|
58
|
+
def stats_entries
|
59
|
+
@stats_entries ||= []
|
60
|
+
end
|
61
|
+
|
62
|
+
def stats_kind_for(object)
|
63
|
+
object.is_a?(Fasten::Executor) ? 'executor' : 'task'
|
12
64
|
end
|
13
65
|
|
14
66
|
def stats_add_entry(state, target)
|
15
67
|
return unless target.ini && target.fin
|
16
68
|
|
17
69
|
entry = stats_create_entry(state, target)
|
18
|
-
self.stats_data ||= []
|
19
|
-
self.stats_entries ||= []
|
20
70
|
stats_data << entry
|
21
71
|
stats_entries << entry
|
22
72
|
|
23
73
|
history = stats_history(entry)
|
24
74
|
|
25
|
-
|
26
|
-
update_avg(history, entry)
|
27
|
-
update_std(history, entry)
|
75
|
+
update_stats(history, entry)
|
28
76
|
end
|
29
77
|
|
30
78
|
FLOAT_FORMATTER = ->(f) { format('%7.3f', f) }
|
@@ -61,8 +109,8 @@ module Fasten
|
|
61
109
|
sub, tot = stats_table_run
|
62
110
|
|
63
111
|
Hirb::Console.render_output(stats_entries,
|
64
|
-
fields: %w[state kind name run cnt avg std], unicode: true, class: 'Hirb::Helpers::AutoTable',
|
65
|
-
filters: { 'run' => FLOAT_FORMATTER, 'avg' => FLOAT_FORMATTER, 'std' => FLOAT_FORMATTER },
|
112
|
+
fields: %w[state kind name run cnt avg std err worker], unicode: true, class: 'Hirb::Helpers::AutoTable',
|
113
|
+
filters: { 'run' => FLOAT_FORMATTER, 'avg' => FLOAT_FORMATTER, 'std' => FLOAT_FORMATTER, 'err' => FLOAT_FORMATTER },
|
66
114
|
description: false)
|
67
115
|
|
68
116
|
puts format('∑tasks: %<task>s ∑executed: %<executed>s saved: %<saved>s workers: %<workers>s',
|
@@ -73,16 +121,17 @@ module Fasten
|
|
73
121
|
stats_data.select { |e| e['state'] == entry['state'] && e['kind'] == entry['kind'] && e['name'] == entry['name'] }
|
74
122
|
end
|
75
123
|
|
76
|
-
def
|
77
|
-
|
78
|
-
end
|
124
|
+
def stats_last(item)
|
125
|
+
return item.last if item.last
|
79
126
|
|
80
|
-
|
81
|
-
entry['avg'] = history.inject(0.0) { |s, x| s + x['run'].to_f } / history.size
|
127
|
+
item.last = stats_data.select { |e| e['kind'] == stats_kind_for(item) && e['name'] == item.name }.last || {}
|
82
128
|
end
|
83
129
|
|
84
|
-
def
|
85
|
-
entry['
|
130
|
+
def update_stats(history, entry)
|
131
|
+
entry['cnt'] = count = history.size
|
132
|
+
entry['avg'] = avg = history.inject(0.0) { |s, x| s + x['run'].to_f } / count
|
133
|
+
entry['std'] = std = Math.sqrt(history.inject(0.0) { |v, x| v + (x['run'].to_f - avg)**2 })
|
134
|
+
entry['err'] = std / Math.sqrt(count) if count.positive?
|
86
135
|
end
|
87
136
|
end
|
88
137
|
end
|
data/lib/fasten/task.rb
CHANGED
@@ -1,6 +1,26 @@
|
|
1
1
|
module Fasten
|
2
|
-
class Task
|
2
|
+
class Task
|
3
3
|
include Fasten::Logger
|
4
|
+
include Fasten::State
|
5
|
+
|
6
|
+
attr_accessor :name, :after, :shell, :ruby
|
7
|
+
attr_accessor :dependants, :depends, :request, :response, :worker, :run_score
|
8
|
+
|
9
|
+
def initialize(name: nil, shell: nil, ruby: nil, request: nil, after: nil)
|
10
|
+
self.name = name
|
11
|
+
self.after = after
|
12
|
+
self.shell = shell
|
13
|
+
self.ruby = ruby
|
14
|
+
self.request = request
|
15
|
+
end
|
16
|
+
|
17
|
+
def marshal_dump
|
18
|
+
[@name, @state, @ini, @fin, @dif, @request, @response, @shell, @ruby, @error]
|
19
|
+
end
|
20
|
+
|
21
|
+
def marshal_load(data)
|
22
|
+
@name, @state, @ini, @fin, @dif, @request, @response, @shell, @ruby, @error = data
|
23
|
+
end
|
4
24
|
|
5
25
|
def to_s
|
6
26
|
name
|
data/lib/fasten/ui/console.rb
CHANGED
@@ -4,6 +4,7 @@ module Fasten
|
|
4
4
|
module UI
|
5
5
|
class Console
|
6
6
|
extend Forwardable
|
7
|
+
|
7
8
|
def_delegators :executor, :worker_list, :task_list, :task_done_list, :task_error_list, :task_running_list, :task_waiting_list, :worker_list
|
8
9
|
def_delegators :executor, :name, :workers, :workers=, :state, :state=, :hformat
|
9
10
|
|
data/lib/fasten/ui/curses.rb
CHANGED
@@ -7,6 +7,7 @@ module Fasten
|
|
7
7
|
class Curses
|
8
8
|
include ::Curses
|
9
9
|
extend Forwardable
|
10
|
+
|
10
11
|
def_delegators :executor, :worker_list, :task_list, :task_done_list, :task_error_list, :task_running_list, :task_waiting_list, :worker_list
|
11
12
|
def_delegators :executor, :name, :workers, :workers=, :state, :state=
|
12
13
|
|
@@ -139,13 +140,13 @@ module Fasten
|
|
139
140
|
end
|
140
141
|
|
141
142
|
def ui_state
|
142
|
-
if
|
143
|
+
if executor.running?
|
143
144
|
attrs = color_pair(2)
|
144
|
-
elsif
|
145
|
+
elsif executor.pausing?
|
145
146
|
attrs = color_pair(1) | A_BLINK | A_STANDOUT
|
146
|
-
elsif
|
147
|
+
elsif executor.paused?
|
147
148
|
attrs = color_pair(1) | A_STANDOUT
|
148
|
-
elsif
|
149
|
+
elsif executor.quitting?
|
149
150
|
attrs = color_pair(3) | A_BLINK | A_STANDOUT
|
150
151
|
end
|
151
152
|
|
@@ -161,7 +162,7 @@ module Fasten
|
|
161
162
|
col_ini.upto col_fin do |col|
|
162
163
|
setpos row, col
|
163
164
|
count -= slice
|
164
|
-
if count
|
165
|
+
if count >= 0
|
165
166
|
addstr PROGRESSBAR_STR[-1]
|
166
167
|
elsif count > -slice
|
167
168
|
addstr PROGRESSBAR_STR[(count * PROGRESSBAR_LEN / slice) % PROGRESSBAR_LEN]
|
@@ -179,8 +180,10 @@ module Fasten
|
|
179
180
|
'✘'
|
180
181
|
when :DONE
|
181
182
|
'✔'
|
182
|
-
|
183
|
+
when :WAIT
|
183
184
|
'…'
|
185
|
+
else
|
186
|
+
' '
|
184
187
|
end
|
185
188
|
end
|
186
189
|
|
@@ -193,7 +196,11 @@ module Fasten
|
|
193
196
|
when :DONE
|
194
197
|
color_pair(2) | A_TOP
|
195
198
|
else
|
196
|
-
|
199
|
+
if task_waiting_list.include? task
|
200
|
+
A_TOP
|
201
|
+
else
|
202
|
+
color_pair(4) | A_DIM
|
203
|
+
end
|
197
204
|
end
|
198
205
|
end
|
199
206
|
|
@@ -227,7 +234,7 @@ module Fasten
|
|
227
234
|
ui_progressbar(2, col_ini, col_fin, count_done, count_total)
|
228
235
|
|
229
236
|
max = 2
|
230
|
-
list = task_list.sort_by
|
237
|
+
list = task_list.sort_by.with_index { |x, index| [x.run_score, index] }
|
231
238
|
list.each_with_index do |task, index|
|
232
239
|
next if 3 + index >= n_rows
|
233
240
|
|
@@ -238,16 +245,23 @@ module Fasten
|
|
238
245
|
list.each_with_index do |task, index|
|
239
246
|
next if 3 + index >= n_rows
|
240
247
|
|
241
|
-
if task.
|
242
|
-
|
243
|
-
ui_task_string(task, 3 + index,
|
244
|
-
elsif task.depends && !task.depends.empty?
|
245
|
-
setpos 3 + index, max
|
246
|
-
x = max + 2
|
247
|
-
addstr ':'
|
248
|
+
if task.depends && !task.depends.empty?
|
249
|
+
x = max
|
250
|
+
x = ui_task_string(task, 3 + index, x, str: ':') + 1
|
248
251
|
task.depends.each do |dependant_task|
|
249
252
|
x = ui_task_string(dependant_task, 3 + index, x) + 1
|
250
253
|
end
|
254
|
+
else
|
255
|
+
x = max + 1
|
256
|
+
last = executor.stats_last(task)
|
257
|
+
if task.dif
|
258
|
+
str = format ' %.2f s', task.dif
|
259
|
+
elsif last['avg'] && last['err']
|
260
|
+
str = format '≈ %.2f s ± %.2f %s', last['avg'], last['err'], task.worker&.name
|
261
|
+
elsif last['avg']
|
262
|
+
str = format '≈ %.2f s %s', last['avg'], task.worker&.name
|
263
|
+
end
|
264
|
+
ui_task_string(task, 3 + index, x, str: str) if str
|
251
265
|
end
|
252
266
|
end
|
253
267
|
end
|
data/lib/fasten/version.rb
CHANGED
data/lib/fasten/worker.rb
CHANGED
@@ -8,11 +8,16 @@ module Fasten
|
|
8
8
|
end
|
9
9
|
end
|
10
10
|
|
11
|
-
class Worker
|
11
|
+
class Worker
|
12
12
|
include Fasten::Logger
|
13
|
+
include Fasten::State
|
14
|
+
|
15
|
+
attr_accessor :executor, :name, :spinner, :child_read, :child_write, :parent_read, :parent_write, :pid, :block, :running_task
|
13
16
|
|
14
17
|
def initialize(executor:, name: nil)
|
15
|
-
|
18
|
+
self.executor = executor
|
19
|
+
self.name = name
|
20
|
+
self.spinner = 0
|
16
21
|
end
|
17
22
|
|
18
23
|
def perform(task)
|
@@ -48,40 +53,32 @@ module Fasten
|
|
48
53
|
end
|
49
54
|
|
50
55
|
def send_request(task)
|
51
|
-
Marshal.dump(Task.new(task.to_h.merge(depends: nil, dependants: nil)), parent_write)
|
52
|
-
self.running_task = task
|
53
|
-
task.worker = self
|
54
56
|
task.state = :RUNNING
|
57
|
+
task.worker = self
|
58
|
+
self.running_task = task
|
59
|
+
self.state = :RUNNING
|
60
|
+
Marshal.dump(task, parent_write)
|
55
61
|
end
|
56
62
|
|
57
63
|
def receive_response
|
58
64
|
updated_task = Marshal.load(parent_read) # rubocop:disable Security/MarshalLoad because pipe is a secure channel
|
59
65
|
|
60
|
-
%i[ini fin response error].each { |key| running_task
|
66
|
+
%i[state ini fin dif response error].each { |key| running_task.send "#{key}=", updated_task.send(key) }
|
61
67
|
|
62
68
|
task = running_task
|
63
|
-
self.running_task = nil
|
64
|
-
task.state = task.error ? :FAIL : :DONE
|
69
|
+
self.running_task = self.state = nil
|
65
70
|
|
66
71
|
task
|
67
72
|
end
|
68
73
|
|
69
74
|
def kill
|
70
75
|
log_info 'Removing worker'
|
71
|
-
Process.kill
|
76
|
+
Process.kill :KILL, pid
|
72
77
|
close_parent_pipes
|
73
78
|
rescue StandardError => error
|
74
79
|
log_warn "Ignoring error killing worker #{self}, error: #{error}"
|
75
80
|
end
|
76
81
|
|
77
|
-
def idle?
|
78
|
-
running_task.nil?
|
79
|
-
end
|
80
|
-
|
81
|
-
def running?
|
82
|
-
!idle?
|
83
|
-
end
|
84
|
-
|
85
82
|
protected
|
86
83
|
|
87
84
|
def create_pipes
|
@@ -127,7 +124,9 @@ module Fasten
|
|
127
124
|
log_ini task, 'perform'
|
128
125
|
|
129
126
|
perform(task)
|
127
|
+
task.state = :DONE
|
130
128
|
rescue StandardError => error
|
129
|
+
task.state = :FAIL
|
131
130
|
task.error = WorkerError.new(error)
|
132
131
|
ensure
|
133
132
|
log_fin task, 'perform'
|
data/lib/fasten/yaml.rb
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
module Fasten
|
2
|
+
module Yaml
|
3
|
+
def transform_params(params)
|
4
|
+
params.keys.each do |k|
|
5
|
+
val = params[k]
|
6
|
+
|
7
|
+
if val.is_a?(String) && (match = %r{^/(.+)/$}.match(val))
|
8
|
+
val = Regexp.new(match[1])
|
9
|
+
end
|
10
|
+
|
11
|
+
params[k.to_sym] = val
|
12
|
+
params.delete(k)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def load_yaml(path)
|
17
|
+
items = YAML.safe_load(File.read(path)).each do |name, params|
|
18
|
+
if params.is_a? String
|
19
|
+
params = { after: params }
|
20
|
+
elsif params.is_a? Hash
|
21
|
+
transform_params(params)
|
22
|
+
else
|
23
|
+
params = {}
|
24
|
+
end
|
25
|
+
|
26
|
+
add Fasten::Task.new({ name: name }.merge(params))
|
27
|
+
end
|
28
|
+
|
29
|
+
log_info "Loaded #{items.count} tasks from #{path}"
|
30
|
+
end
|
31
|
+
|
32
|
+
def save_yaml(path)
|
33
|
+
keys = %i[after shell]
|
34
|
+
|
35
|
+
items = task_list.map do |task|
|
36
|
+
data = task.to_h.select do |key, _val|
|
37
|
+
keys.include? key
|
38
|
+
end
|
39
|
+
|
40
|
+
[task.name, data]
|
41
|
+
end.to_h
|
42
|
+
|
43
|
+
File.write path, items.to_yaml
|
44
|
+
|
45
|
+
log_info "Loaded #{items.count} tasks into #{path}"
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
data/lib/fasten.rb
CHANGED
@@ -11,12 +11,13 @@ require 'csv'
|
|
11
11
|
require 'hirb'
|
12
12
|
require 'parallel'
|
13
13
|
|
14
|
+
require 'fasten/state'
|
14
15
|
require 'fasten/logger'
|
15
16
|
require 'fasten/stats'
|
16
17
|
require 'fasten/task'
|
17
18
|
require 'fasten/ui'
|
18
19
|
require 'fasten/dag'
|
19
|
-
require 'fasten/
|
20
|
+
require 'fasten/yaml'
|
20
21
|
require 'fasten/executor'
|
21
22
|
require 'fasten/worker'
|
22
23
|
require 'fasten/version'
|
@@ -25,9 +26,9 @@ module Fasten
|
|
25
26
|
class << self
|
26
27
|
include Fasten::Logger
|
27
28
|
|
28
|
-
def
|
29
|
+
def from_yaml(path, **options)
|
29
30
|
executor = Fasten::Executor.new(**options)
|
30
|
-
executor.
|
31
|
+
executor.load_yaml(path)
|
31
32
|
|
32
33
|
executor
|
33
34
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fasten
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Aldrin Martoq
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-10-
|
11
|
+
date: 2018-10-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -161,8 +161,8 @@ files:
|
|
161
161
|
- lib/fasten.rb
|
162
162
|
- lib/fasten/dag.rb
|
163
163
|
- lib/fasten/executor.rb
|
164
|
-
- lib/fasten/load_save.rb
|
165
164
|
- lib/fasten/logger.rb
|
165
|
+
- lib/fasten/state.rb
|
166
166
|
- lib/fasten/stats.rb
|
167
167
|
- lib/fasten/task.rb
|
168
168
|
- lib/fasten/ui.rb
|
@@ -170,6 +170,7 @@ files:
|
|
170
170
|
- lib/fasten/ui/curses.rb
|
171
171
|
- lib/fasten/version.rb
|
172
172
|
- lib/fasten/worker.rb
|
173
|
+
- lib/fasten/yaml.rb
|
173
174
|
homepage: https://github.com/a0/fasten/
|
174
175
|
licenses:
|
175
176
|
- MIT
|
data/lib/fasten/load_save.rb
DELETED
@@ -1,73 +0,0 @@
|
|
1
|
-
module Fasten
|
2
|
-
module LoadSave
|
3
|
-
attr_reader :stats_path
|
4
|
-
|
5
|
-
def load(path)
|
6
|
-
items = YAML.safe_load(File.read(path)).each do |name, params|
|
7
|
-
if params.is_a? String
|
8
|
-
params = { after: params }
|
9
|
-
else
|
10
|
-
params&.each do |key, val|
|
11
|
-
next unless val.is_a?(String) && (match = %r{^/(.+)/$}.match(val))
|
12
|
-
|
13
|
-
params[key] = Regexp.new(match[1])
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
|
-
add Fasten::Task.new({ name: name }.merge(params || {}))
|
18
|
-
end
|
19
|
-
|
20
|
-
log_info "Loaded #{items.count} tasks from #{path}"
|
21
|
-
end
|
22
|
-
|
23
|
-
def save(path)
|
24
|
-
keys = %i[after shell]
|
25
|
-
|
26
|
-
items = task_list.map do |task|
|
27
|
-
data = task.to_h.select do |key, _val|
|
28
|
-
keys.include? key
|
29
|
-
end
|
30
|
-
|
31
|
-
[task.name, data]
|
32
|
-
end.to_h
|
33
|
-
|
34
|
-
File.write path, items.to_yaml
|
35
|
-
|
36
|
-
log_info "Loaded #{items.count} tasks into #{path}"
|
37
|
-
end
|
38
|
-
|
39
|
-
def load_stats
|
40
|
-
return unless @stats_path && File.exist?(@stats_path)
|
41
|
-
|
42
|
-
self.stats_data = []
|
43
|
-
CSV.foreach(@stats_path, headers: true) do |row|
|
44
|
-
stats_data << row.to_h
|
45
|
-
end
|
46
|
-
|
47
|
-
@task_waiting_list = nil
|
48
|
-
rescue StandardError
|
49
|
-
nil
|
50
|
-
ensure
|
51
|
-
self.stats ||= {}
|
52
|
-
end
|
53
|
-
|
54
|
-
def save_stats
|
55
|
-
return unless @stats_path && stats_data
|
56
|
-
|
57
|
-
CSV.open(@stats_path, 'wb') do |csv|
|
58
|
-
csv << stats_data.first.keys
|
59
|
-
|
60
|
-
stats_data.each do |data|
|
61
|
-
csv << data.values
|
62
|
-
end
|
63
|
-
end
|
64
|
-
end
|
65
|
-
|
66
|
-
def setup_stats(name)
|
67
|
-
@stats_path = "#{ENV['HOME']}/.fasten/stats/#{name}.csv" if ENV['HOME'] && name
|
68
|
-
FileUtils.mkdir_p File.dirname(@stats_path)
|
69
|
-
rescue StandardError
|
70
|
-
@stats_path = nil
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|