rbbt-util 5.25.25 → 5.25.26
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/util/cmd.rb +12 -52
- data/lib/rbbt/util/log.rb +11 -3
- data/lib/rbbt/util/open.rb +14 -1
- data/lib/rbbt/workflow/step/run.rb +4 -2
- data/lib/rbbt/workflow/usage.rb +23 -0
- data/share/rbbt_commands/workflow/prov +5 -4
- data/share/rbbt_commands/workflow/task +1 -14
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 332e52f2862068fad1da1a881a2d481423e709ca
|
4
|
+
data.tar.gz: c27385464c185c1d726251a7b67668b48b3d94f0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4eef47ef223160fc7b9b4211d632df7ce43782cb1808f9c6d393bc6f60e278efb4f09f789cb0add14ffbbe8dd03ecc21d4ef2f17bce1efbe7248e3d33cc890db
|
7
|
+
data.tar.gz: 816afdb536c064923a881d9cb15590615be70cc54320929aa06d06d5133e2f7611c25d42d6296cb49cf0d60da34d2403ae7efb57e155da3b320cf999d14266bd
|
data/lib/rbbt/util/cmd.rb
CHANGED
@@ -54,50 +54,6 @@ module CMD
|
|
54
54
|
|
55
55
|
in_content = StringIO.new in_content if String === in_content
|
56
56
|
|
57
|
-
#sout, serr, sin = Misc.pipe, Misc.pipe, Misc.pipe
|
58
|
-
|
59
|
-
#pid = fork {
|
60
|
-
# begin
|
61
|
-
# Misc.purge_pipes(sin.last,sout.last,serr.last)
|
62
|
-
|
63
|
-
# sin.last.close
|
64
|
-
# sout.first.close
|
65
|
-
# serr.first.close
|
66
|
-
|
67
|
-
# if IO === in_content
|
68
|
-
# in_content.close if in_content.respond_to?(:close) and not in_content.closed?
|
69
|
-
# end
|
70
|
-
|
71
|
-
# STDERR.reopen serr.last
|
72
|
-
# serr.last.close
|
73
|
-
|
74
|
-
# STDIN.reopen sin.first
|
75
|
-
# sin.first.close
|
76
|
-
|
77
|
-
# STDOUT.reopen sout.last
|
78
|
-
# sout.last.close
|
79
|
-
|
80
|
-
# STDOUT.sync = STDERR.sync = true
|
81
|
-
|
82
|
-
# exec(ENV, cmd)
|
83
|
-
|
84
|
-
# exit(-1)
|
85
|
-
# rescue Exception
|
86
|
-
# Log.debug{ "ProcessFailed: #{$!.message}" } if log
|
87
|
-
# Log.debug{ "Backtrace: \n" + $!.backtrace * "\n" } if log
|
88
|
-
# raise ProcessFailed, $!.message
|
89
|
-
# end
|
90
|
-
#}
|
91
|
-
|
92
|
-
#sin.first.close
|
93
|
-
#sout.last.close
|
94
|
-
#serr.last.close
|
95
|
-
|
96
|
-
|
97
|
-
#sin = sin.last
|
98
|
-
#sout = sout.first
|
99
|
-
#serr = serr.first
|
100
|
-
|
101
57
|
sin, sout, serr, wait_thr = begin
|
102
58
|
Open3.popen3(ENV, cmd)
|
103
59
|
rescue
|
@@ -139,8 +95,8 @@ module CMD
|
|
139
95
|
if pipe
|
140
96
|
err_thread = Thread.new do
|
141
97
|
while line = serr.gets
|
142
|
-
Log.log "STDERR [#{pid}]: " + line, stderr
|
143
|
-
end
|
98
|
+
Log.log "STDERR [#{pid}]: " + line, stderr
|
99
|
+
end if Integer === stderr and log
|
144
100
|
serr.close
|
145
101
|
end
|
146
102
|
|
@@ -181,13 +137,17 @@ module CMD
|
|
181
137
|
|
182
138
|
io = cmd(*all_args)
|
183
139
|
pid = io.pids.first
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
140
|
+
|
141
|
+
while c = io.getc
|
142
|
+
STDERR << c
|
143
|
+
if c == "\n"
|
144
|
+
if pid
|
145
|
+
Log.logn "STDOUT [#{pid}]: ", 0
|
146
|
+
else
|
147
|
+
Log.logn "STDOUT: ", 0
|
148
|
+
end
|
189
149
|
end
|
190
|
-
end
|
150
|
+
end
|
191
151
|
io.join
|
192
152
|
|
193
153
|
nil
|
data/lib/rbbt/util/log.rb
CHANGED
@@ -160,7 +160,7 @@ module Log
|
|
160
160
|
end
|
161
161
|
|
162
162
|
LAST = "log"
|
163
|
-
def self.
|
163
|
+
def self.logn(message = nil, severity = MEDIUM, &block)
|
164
164
|
return if severity < self.severity
|
165
165
|
message ||= block.call if block_given?
|
166
166
|
return if message.nil?
|
@@ -174,13 +174,21 @@ module Log
|
|
174
174
|
str = prefix << " " << message.to_s
|
175
175
|
|
176
176
|
LOG_MUTEX.synchronize do
|
177
|
-
STDERR.
|
177
|
+
STDERR.write str
|
178
178
|
Log::LAST.replace "log"
|
179
|
-
logfile.
|
179
|
+
logfile.write str unless logfile.nil?
|
180
180
|
nil
|
181
181
|
end
|
182
182
|
end
|
183
183
|
|
184
|
+
def self.log(message = nil, severity = MEDIUM, &block)
|
185
|
+
return if severity < self.severity
|
186
|
+
message ||= block.call if block_given?
|
187
|
+
return if message.nil?
|
188
|
+
message = message + "\n" unless message[-1] == "\n"
|
189
|
+
self.logn message, severity, &block
|
190
|
+
end
|
191
|
+
|
184
192
|
def self.log_obj_inspect(obj, level, file = $stdout)
|
185
193
|
stack = caller
|
186
194
|
|
data/lib/rbbt/util/open.rb
CHANGED
@@ -750,7 +750,7 @@ module Open
|
|
750
750
|
begin
|
751
751
|
if File.symlink?(file) || File.stat(file).nlink > 1
|
752
752
|
if File.exists?(file + '.info')
|
753
|
-
|
753
|
+
return Step::INFO_SERIALIAZER.load(Open.open(file + '.info'))[:done]
|
754
754
|
else
|
755
755
|
file = Pathname.new(file).realpath.to_s
|
756
756
|
end
|
@@ -762,6 +762,19 @@ module Open
|
|
762
762
|
end
|
763
763
|
end
|
764
764
|
|
765
|
+
def self.update_mtime(path, target)
|
766
|
+
if File.symlink?(target) || File.stat(target).nlink > 1
|
767
|
+
if File.exists?(target + '.info')
|
768
|
+
target = target + '.info'
|
769
|
+
else
|
770
|
+
target = Pathname.new(target).realpath.to_s
|
771
|
+
end
|
772
|
+
end
|
773
|
+
|
774
|
+
CMD.cmd("touch -r '#{path}' '#{target}'")
|
775
|
+
CMD.cmd("touch -r '#{path}.info' '#{target}'") if File.exists?(path + '.info')
|
776
|
+
end
|
777
|
+
|
765
778
|
def self.atime(file)
|
766
779
|
if (dir_sub_path = find_repo_dir(file))
|
767
780
|
get_atime_from_repo(*dir_sub_path)
|
@@ -154,7 +154,7 @@ class Step
|
|
154
154
|
dep_done = dep.done?
|
155
155
|
|
156
156
|
begin
|
157
|
-
if this_mtime && dep_done && Open.exists?(dep.path) && (Open.mtime(dep.path) > this_mtime)
|
157
|
+
if this_mtime && dep_done && Open.exists?(dep.path) && (Open.mtime(dep.path) > this_mtime + 1)
|
158
158
|
outdated_time << dep
|
159
159
|
end
|
160
160
|
rescue
|
@@ -209,7 +209,9 @@ class Step
|
|
209
209
|
no_load = :stream if no_load
|
210
210
|
|
211
211
|
Open.write(pid_file, Process.pid.to_s) unless Open.exists?(path) or Open.exists?(pid_file)
|
212
|
-
|
212
|
+
result_type = @task.result_type
|
213
|
+
result_type = info[:result_type] if result_type.nil?
|
214
|
+
result = Persist.persist "Job", result_type, :file => path, :check => persist_checks, :no_load => no_load do
|
213
215
|
if Step === Step.log_relay_step and not self == Step.log_relay_step
|
214
216
|
relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
|
215
217
|
end
|
data/lib/rbbt/workflow/usage.rb
CHANGED
@@ -130,4 +130,27 @@ module Workflow
|
|
130
130
|
end
|
131
131
|
end
|
132
132
|
end
|
133
|
+
|
134
|
+
def SOPT_str(task)
|
135
|
+
sopt_options = []
|
136
|
+
self.rec_inputs(task.name).each do |name|
|
137
|
+
short = name.to_s.chars.first
|
138
|
+
boolean = self.rec_input_types(task.name)[name].to_sym == :boolean
|
139
|
+
|
140
|
+
sopt_options << "-#{short}--#{name}#{boolean ? "" : "*"}"
|
141
|
+
end
|
142
|
+
|
143
|
+
sopt_options * ":"
|
144
|
+
end
|
145
|
+
|
146
|
+
def get_SOPT(task)
|
147
|
+
sopt_option_string = self.SOPT_str(task)
|
148
|
+
SOPT.get sopt_option_string
|
149
|
+
end
|
150
|
+
|
151
|
+
def self.get_SOPT(workflow, task)
|
152
|
+
workflow = Workflow.require_workflow workflow if String === workflow
|
153
|
+
task = workflow.tasks[task.to_sym] if String === task || Symbol === task
|
154
|
+
workflow.get_SOPT(task)
|
155
|
+
end
|
133
156
|
end
|
@@ -76,7 +76,8 @@ def report_msg(status, name, path, info = nil)
|
|
76
76
|
rescue
|
77
77
|
nil
|
78
78
|
end
|
79
|
-
str = if not Open.remote?(path) and (Open.exists?(path) and $main_mtime and path_mtime and ($main_mtime - path_mtime) <
|
79
|
+
str = if not Open.remote?(path) and (Open.exists?(path) and $main_mtime and path_mtime and ($main_mtime - path_mtime) < -2)
|
80
|
+
iii [path, path_mtime, $main_mtime, $main_mtime - path_mtime]
|
80
81
|
status_msg(status.to_s) << " " << [workflow, task, path].compact * " " << " (#{Log.color(:red, "Mtime out of sync") })"
|
81
82
|
else
|
82
83
|
status_msg(status.to_s) << " " << [workflow, task, path].compact * " "
|
@@ -111,11 +112,11 @@ def touch(step)
|
|
111
112
|
return unless File.exists?(step.path)
|
112
113
|
step.dependencies.each do |dep|
|
113
114
|
next unless Open.exists?(dep.path)
|
114
|
-
|
115
|
-
if Open.mtime(step.path) < Open.mtime(dep.path)
|
115
|
+
if Open.mtime(dep.path) > Open.mtime(step.path) + 1
|
116
116
|
Log.debug("Updating #{step.path} to #{dep.path}")
|
117
|
-
|
117
|
+
Open.update_mtime(dep.path, step.path)
|
118
118
|
end
|
119
|
+
touch(dep)
|
119
120
|
end if step.dependencies
|
120
121
|
end
|
121
122
|
|
@@ -48,18 +48,6 @@ def usage(workflow = nil, task = nil, exception=nil)
|
|
48
48
|
true
|
49
49
|
end
|
50
50
|
|
51
|
-
def SOPT_options(workflow, task)
|
52
|
-
sopt_options = []
|
53
|
-
workflow.rec_inputs(task.name).each do |name|
|
54
|
-
short = name.to_s.chars.first
|
55
|
-
boolean = workflow.rec_input_types(task.name)[name].to_sym == :boolean
|
56
|
-
|
57
|
-
sopt_options << "-#{short}--#{name}#{boolean ? "" : "*"}"
|
58
|
-
end
|
59
|
-
|
60
|
-
sopt_options * ":"
|
61
|
-
end
|
62
|
-
|
63
51
|
def get_value_stream(value)
|
64
52
|
if value == "-"
|
65
53
|
io = Misc.open_pipe do |sin|
|
@@ -289,8 +277,7 @@ usage workflow, task and exit 0 if help
|
|
289
277
|
name = options.delete(:jobname)
|
290
278
|
|
291
279
|
# get job args
|
292
|
-
|
293
|
-
job_options = SOPT.get sopt_option_string
|
280
|
+
job_options = workflow.get_SOPT(task)
|
294
281
|
|
295
282
|
if options[:load_inputs]
|
296
283
|
task_info = workflow.task_info(task_name)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.25.
|
4
|
+
version: 5.25.26
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-02-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|