fasten 0.5.4 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,130 @@
1
+ module Fasten
2
+ module Support
3
+ module DAG
4
+ attr_reader :task_map, :task_list, :task_done_list, :task_error_list, :task_pending_list, :task_running_list
5
+
6
+ def initialize_dag
7
+ @task_map ||= {}
8
+ @task_list ||= []
9
+ @task_done_list ||= []
10
+ @task_error_list ||= []
11
+ @task_pending_list ||= []
12
+ @task_running_list ||= []
13
+
14
+ nil
15
+ end
16
+
17
+ def add(task)
18
+ raise "Task '#{task.name}' already defined" if @task_map[task.name]
19
+
20
+ @task_map[task.name] = task
21
+ @task_list << task
22
+ @task_waiting_list = nil
23
+ end
24
+
25
+ def update_task(task)
26
+ task.state == :DONE ? update_done_task(task) : update_error_task(task)
27
+
28
+ stats_add_entry(task.state, task)
29
+ end
30
+
31
+ def update_done_task(task)
32
+ @task_done_list << task
33
+ @task_pending_list.delete task
34
+ task.dependants.each { |dependant_task| dependant_task.depends.delete task }
35
+
36
+ move_pending_to_waiting
37
+ end
38
+
39
+ def update_error_task(task)
40
+ @task_error_list << task
41
+ @task_pending_list.delete task
42
+ end
43
+
44
+ def next_task
45
+ task_waiting_list.shift
46
+ end
47
+
48
+ def task_waiting_list
49
+ return @task_waiting_list if @task_waiting_list
50
+
51
+ reset_tasks
52
+ setup_tasks_dependencies
53
+ setup_tasks_scores
54
+ move_pending_to_waiting
55
+ end
56
+
57
+ protected
58
+
59
+ def move_pending_to_waiting
60
+ move_list = task_pending_list.select { |task| task.depends.count.zero? }
61
+
62
+ @task_waiting_list ||= []
63
+ @task_pending_list -= move_list
64
+ @task_waiting_list += move_list
65
+ @task_waiting_list.sort_by!.with_index do |x, index|
66
+ x.state = :WAIT
67
+ [-x.run_score, index]
68
+ end
69
+ end
70
+
71
+ def reset_tasks
72
+ @task_pending_list.clear
73
+ @task_done_list.clear
74
+ @task_error_list.clear
75
+
76
+ @task_list.each do |task|
77
+ task.dependants = []
78
+ task.depends = []
79
+
80
+ if task.state == :DONE
81
+ @task_done_list << task
82
+ elsif task.state == :FAIL
83
+ @task_error_list << task
84
+ else
85
+ task.state = :IDLE
86
+ @task_pending_list << task
87
+ end
88
+ end
89
+ end
90
+
91
+ def setup_tasks_dependencies
92
+ @task_pending_list.each do |task|
93
+ next unless task.after
94
+
95
+ [task.after].flatten.each do |after|
96
+ after_task = after.is_a?(Task) ? after : @task_map[after]
97
+ raise "Dependency task '#{after}' not found on task '#{task.name}'." unless after_task
98
+
99
+ task.depends << after_task
100
+ after_task.dependants << task
101
+ end
102
+ end
103
+ end
104
+
105
+ def setup_tasks_scores
106
+ @task_pending_list.each { |task| task.run_score = task.dependants.count }
107
+ end
108
+
109
+ def no_waiting_tasks?
110
+ task_waiting_list.empty?
111
+ end
112
+
113
+ def no_running_tasks?
114
+ task_running_list.empty?
115
+ end
116
+
117
+ def tasks_waiting?
118
+ !task_waiting_list.empty?
119
+ end
120
+
121
+ def tasks_running?
122
+ !task_running_list.empty?
123
+ end
124
+
125
+ def tasks_failed?
126
+ !task_error_list.empty?
127
+ end
128
+ end
129
+ end
130
+ end
@@ -0,0 +1,107 @@
1
+ module Fasten
2
+ module Support
3
+ module ForkWorker
4
+ attr_accessor :pid
5
+
6
+ def start
7
+ create_pipes
8
+
9
+ self.pid = Process.fork do
10
+ close_parent_pipes
11
+
12
+ process_incoming_requests
13
+ end
14
+
15
+ close_child_pipes
16
+ end
17
+
18
+ def kill
19
+ log_info 'Removing worker'
20
+ Process.kill :KILL, pid
21
+ rescue StandardError => error
22
+ log_warn "Ignoring error killing worker #{self}, error: #{error}"
23
+ ensure
24
+ close_parent_pipes
25
+ close_child_pipes
26
+ end
27
+
28
+ def send_request_to_child(task)
29
+ task.state = :RUNNING
30
+ task.worker = self
31
+ self.running_task = task
32
+ self.state = :RUNNING
33
+
34
+ Marshal.dump(task, parent_write)
35
+ end
36
+
37
+ def receive_response_from_child
38
+ updated_task = Marshal.load(parent_read) # rubocop:disable Security/MarshalLoad because pipe is a secure channel
39
+
40
+ %i[state ini fin dif response error].each { |key| running_task.send "#{key}=", updated_task.send(key) }
41
+
42
+ task = running_task
43
+ self.running_task = self.state = nil
44
+
45
+ task
46
+ end
47
+
48
+ def receive_request_from_parent
49
+ Marshal.load(child_read) # rubocop:disable Security/MarshalLoad because pipe is a secure channel
50
+ end
51
+
52
+ def send_response_to_parent(task)
53
+ log_info "Sending task response back to runner #{task}"
54
+
55
+ data = Marshal.dump(task)
56
+ child_write.write(data)
57
+ end
58
+
59
+ def create_pipes
60
+ self.child_read, self.parent_write = IO.pipe
61
+ self.parent_read, self.child_write = IO.pipe
62
+ end
63
+
64
+ def close_parent_pipes
65
+ parent_read.close unless parent_read.closed?
66
+ parent_write.close unless parent_write.closed?
67
+ end
68
+
69
+ def close_child_pipes
70
+ child_read.close unless child_read.closed?
71
+ child_write.close unless child_write.closed?
72
+ end
73
+
74
+ def redirect_std(path)
75
+ logger.reopen($stdout)
76
+
77
+ @saved_stdout_instance = $stdout.clone
78
+ @saved_stderr_instance = $stderr.clone
79
+ @saved_stdout_constant = STDOUT.clone
80
+ @saved_stderr_constant = STDERR.clone
81
+
82
+ FileUtils.mkdir_p File.dirname(path)
83
+ @redirect_log = File.new path, 'a'
84
+ @redirect_log.sync = true
85
+
86
+ $stdout.reopen @redirect_log
87
+ $stderr.reopen @redirect_log
88
+ STDOUT.reopen @redirect_log
89
+ STDERR.reopen @redirect_log
90
+ end
91
+
92
+ def restore_std
93
+ oldverbose = $VERBOSE
94
+ $VERBOSE = nil
95
+
96
+ $stdout = @saved_stdout_instance
97
+ $stderr = @saved_stderr_instance
98
+ Object.const_set :STDOUT, @saved_stdout_constant
99
+ Object.const_set :STDERR, @saved_stderr_constant
100
+
101
+ @redirect_log.close
102
+ ensure
103
+ $VERBOSE = oldverbose
104
+ end
105
+ end
106
+ end
107
+ end
@@ -0,0 +1,57 @@
1
+ require 'binding_of_caller'
2
+ require 'fileutils'
3
+ require 'logger'
4
+
5
+ module Fasten
6
+ class << self
7
+ attr_accessor :logger
8
+ end
9
+
10
+ module Support
11
+ module Logger
12
+ attr_accessor :log_file, :logger
13
+
14
+ %w[debug info error].each do |method|
15
+ define_method "log_#{method}" do |msg|
16
+ dest_logger = logger || Fasten.logger
17
+ return unless dest_logger.respond_to?(method)
18
+
19
+ caller_name = name if respond_to? :name
20
+ caller_name ||= Kernel.binding.of_caller(1).eval('self').class
21
+ dest_logger.send(method, caller_name) { msg }
22
+ end
23
+ end
24
+
25
+ def initialize_logger(log_file: nil)
26
+ if log_file
27
+ self.log_file = log_file
28
+ else
29
+ log_path ||= "#{fasten_dir}/log/#{kind}/#{name}.log"
30
+ FileUtils.mkdir_p File.dirname(log_path)
31
+ self.log_file = File.new(log_path, 'a')
32
+ self.log_file.sync = true
33
+ end
34
+ self.logger = ::Logger.new self.log_file, level: Fasten.logger.level, progname: Fasten.logger.progname
35
+ end
36
+
37
+ def log_ini(object, message = nil)
38
+ object.ini ||= Time.new
39
+ log_info "Ini #{object.state} #{object.class} #{object} #{message}"
40
+ end
41
+
42
+ def log_fin(object, message = nil)
43
+ object.fin ||= Time.new
44
+ object.dif = object.fin - object.ini
45
+
46
+ log_info "Fin #{object.state} #{object.class} #{object} #{message} in #{object.dif}"
47
+ end
48
+ end
49
+ end
50
+ end
51
+
52
+ Fasten.logger ||=
53
+ begin
54
+ Logger.new STDOUT, level: Logger::DEBUG, progname: $PROGRAM_NAME
55
+ rescue StandardError
56
+ nil
57
+ end
@@ -0,0 +1,32 @@
1
+ module Fasten
2
+ module Support
3
+ module State
4
+ attr_accessor :error, :ini, :fin, :dif, :last
5
+ attr_writer :state
6
+
7
+ def state
8
+ @state || :IDLE
9
+ end
10
+
11
+ def running?
12
+ state == :RUNNING
13
+ end
14
+
15
+ def idle?
16
+ state == :IDLE
17
+ end
18
+
19
+ def pausing?
20
+ state == :PAUSING
21
+ end
22
+
23
+ def paused?
24
+ state == :PAUSED
25
+ end
26
+
27
+ def quitting?
28
+ state == :QUITTING
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,137 @@
1
+ require 'csv'
2
+ require 'hirb'
3
+ require 'fileutils'
4
+
5
+ module Fasten
6
+ module Support
7
+ module Stats
8
+ attr_writer :stats_data, :stats_entries
9
+ attr_reader :stats_path
10
+
11
+ def initialize_stats
12
+ return unless stats
13
+
14
+ @stats_path = "#{ENV['HOME']}/.fasten/stats/#{name}.csv" if ENV['HOME']
15
+ FileUtils.mkdir_p File.dirname(@stats_path)
16
+ rescue StandardError
17
+ @stats_path = nil
18
+ end
19
+
20
+ def load_stats
21
+ return unless @stats_path && File.exist?(@stats_path)
22
+
23
+ self.stats_data = []
24
+ CSV.foreach(@stats_path, headers: true) do |row|
25
+ stats_data << row.to_h
26
+ end
27
+
28
+ @task_waiting_list = nil
29
+ rescue StandardError
30
+ nil
31
+ end
32
+
33
+ def save_stats
34
+ return unless @stats_path && stats_data
35
+
36
+ keys = %w[state kind name run cnt avg std err]
37
+
38
+ CSV.open(@stats_path, 'wb') do |csv|
39
+ csv << keys
40
+
41
+ stats_data.each do |data|
42
+ csv << keys.map { |i| data[i] }
43
+ end
44
+ end
45
+ end
46
+
47
+ def stats_create_entry(state, target)
48
+ { 'state' => state.to_s,
49
+ 'kind' => target.kind,
50
+ 'name' => target.name,
51
+ 'ini' => target.ini.to_f,
52
+ 'fin' => target.fin.to_f,
53
+ 'run' => target.fin - target.ini,
54
+ 'worker' => target.respond_to?(:worker) ? target.worker.name : nil }
55
+ end
56
+
57
+ def stats_data
58
+ @stats_data ||= []
59
+ end
60
+
61
+ def stats_entries
62
+ @stats_entries ||= []
63
+ end
64
+
65
+ def stats_add_entry(state, target)
66
+ return unless target.ini && target.fin
67
+
68
+ entry = stats_create_entry(state, target)
69
+ stats_data << entry
70
+ stats_entries << entry
71
+
72
+ history = stats_history(entry)
73
+
74
+ update_stats(history, entry)
75
+ end
76
+
77
+ FLOAT_FORMATTER = ->(f) { format('%7.3f', f) }
78
+
79
+ def stats_table_run
80
+ sub = stats_entries.select { |x| x['kind'] == 'task' }.map { |x| x['run'] }.sum
81
+ tot = stats_entries.select { |x| x['kind'] == 'runner' }.map { |x| x['run'] }.sum
82
+
83
+ [sub, tot]
84
+ end
85
+
86
+ def split_time(time)
87
+ sign = time.negative? ? '-' : ''
88
+ time = -time if time.negative?
89
+
90
+ hours, seconds = time.divmod(3600)
91
+ minutes, seconds = seconds.divmod(60)
92
+ seconds, decimal = seconds.divmod(1)
93
+ milliseconds, _ignored = (decimal.round(4) * 1000).divmod(1)
94
+
95
+ [sign, hours, minutes, seconds, milliseconds]
96
+ end
97
+
98
+ def hformat(time, total = nil)
99
+ sign, hours, mins, secs, msecs = split_time time
100
+
101
+ str = hours.zero? ? format('%.1s%02d:%02d.%03d', sign, mins, secs, msecs) : format('%.1s%02d:%02d:%02d.%03d', sign, hours, mins, secs, msecs)
102
+ str += format(' (%.1f%%)', 100.0 * time / total) if total
103
+
104
+ str
105
+ end
106
+
107
+ def stats_table
108
+ sub, tot = stats_table_run
109
+
110
+ Hirb::Console.render_output(stats_entries,
111
+ fields: %w[state kind name run cnt avg std err worker], unicode: true, class: 'Hirb::Helpers::AutoTable',
112
+ filters: { 'run' => FLOAT_FORMATTER, 'avg' => FLOAT_FORMATTER, 'std' => FLOAT_FORMATTER, 'err' => FLOAT_FORMATTER },
113
+ description: false)
114
+
115
+ puts format('∑tasks: %<task>s runner: %<runner>s saved: %<saved>s workers: %<workers>s',
116
+ task: hformat(sub), runner: hformat(tot, sub), saved: hformat(sub - tot, sub), workers: workers.to_s)
117
+ end
118
+
119
+ def stats_history(entry)
120
+ stats_data.select { |e| e['state'] == entry['state'] && e['kind'] == entry['kind'] && e['name'] == entry['name'] }.map { |x| x['run'].to_f }
121
+ end
122
+
123
+ def stats_last(item)
124
+ return item.last if item.last
125
+
126
+ item.last = stats_data.select { |e| e['kind'] == item.kind && e['name'] == item.name }.last || {}
127
+ end
128
+
129
+ def update_stats(history, entry)
130
+ entry['cnt'] = count = history.size
131
+ entry['avg'] = avg = history.sum.to_f / count
132
+ entry['std'] = std = Math.sqrt(history.inject(0.0) { |v, x| v + (x - avg)**2 })
133
+ entry['err'] = std / Math.sqrt(count) if count.positive?
134
+ end
135
+ end
136
+ end
137
+ end