rbbt-util 5.5.58 → 5.5.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/lib/rbbt/persist/tsv.rb +1 -1
- data/lib/rbbt/tsv/accessor.rb +6 -3
- data/lib/rbbt/tsv/manipulate.rb +1 -1
- data/lib/rbbt/util/color.rb +19 -0
- data/lib/rbbt/util/log.rb +15 -8
- data/lib/rbbt/util/simpleopt.rb +113 -107
- data/lib/rbbt/workflow.rb +22 -23
- data/lib/rbbt/workflow/accessor.rb +17 -15
- data/lib/rbbt/workflow/definition.rb +0 -1
- data/lib/rbbt/workflow/step.rb +10 -5
- data/test/rbbt/workflow/test_step.rb +12 -12
- metadata +15 -1
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
Y2JmMzk4OWUzYjZkNTE2NTZkODI4NzQ3OGNmZTA3YTg0Y2NmMTMyMg==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
NDNlMTU0Y2JhMjcwMzlmYjkwMmY0NWU0OGJmODVlZDRmODdkZjVkZg==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
YmI5ZGEyN2M4YjE0Mjg0YzVmMTM4NTk0NzhkZDJiODE1MTE5Zjk0NjQyM2Yy
|
10
|
+
MmY5MDIyNzRiMTJhMGI1OTM1MGFiZThmZDI1M2IyMDc2OGQ4OGRhMjVhNTEz
|
11
|
+
NWViMGU3MTlkMDA3N2QwNWVlNGEyNmQxNWM3ZWY4ZDA0ZmY1Yzc=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
NGNkMWM5OTFkY2ZiZTQyMDY1ZDhjZDFmMTM0YjQ2YTAwNGJhMTU5ZjM5ZWUy
|
14
|
+
YzBmMjM2ZjRjMTFkMTEwMDg0NmU2Mjk5MmY2NmFhNzMwMGNmNTUzOTljZmEx
|
15
|
+
ZjhkODIwN2Y4YzJiMDMxODVlZWZiZTIxMjcwOGZiMTM5MmU0MDQ=
|
data/lib/rbbt/persist/tsv.rb
CHANGED
@@ -8,7 +8,7 @@ module Persist
|
|
8
8
|
TC_CONNECTIONS = {}
|
9
9
|
|
10
10
|
def self.open_tokyocabinet(path, write, serializer = nil, tokyocabinet_class = TokyoCabinet::HDB)
|
11
|
-
write = true
|
11
|
+
write = true unless File.exists? path
|
12
12
|
|
13
13
|
tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB"
|
14
14
|
tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
|
data/lib/rbbt/tsv/accessor.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'yaml'
|
2
|
+
require 'rbbt/annotations'
|
2
3
|
module TSV
|
3
4
|
|
4
5
|
TSV_SERIALIZER = YAML
|
@@ -248,6 +249,7 @@ module TSV
|
|
248
249
|
|
249
250
|
def sort_by(field = nil, just_keys = false, &block)
|
250
251
|
field = :all if field.nil?
|
252
|
+
|
251
253
|
if field == :all
|
252
254
|
elems = collect
|
253
255
|
else
|
@@ -298,9 +300,8 @@ module TSV
|
|
298
300
|
end
|
299
301
|
end
|
300
302
|
if just_keys
|
301
|
-
#keys = elems.sort_by{|key, value| value }.collect{|key, value| key}
|
302
303
|
keys = sorted.collect{|key, value| key}
|
303
|
-
keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true))
|
304
|
+
keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true)) unless @unnamed
|
304
305
|
keys
|
305
306
|
else
|
306
307
|
sorted.collect{|key, value| [key, self[key]]}
|
@@ -308,7 +309,9 @@ module TSV
|
|
308
309
|
end
|
309
310
|
else
|
310
311
|
if just_keys
|
311
|
-
elems.sort_by(&block).collect{|key, value| key}
|
312
|
+
keys = elems.sort_by(&block).collect{|key, value| key}
|
313
|
+
keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true)) unless @unnamed
|
314
|
+
keys
|
312
315
|
else
|
313
316
|
elems.sort_by(&block).collect{|key, value| [key, self[key]]}
|
314
317
|
end
|
data/lib/rbbt/tsv/manipulate.rb
CHANGED
@@ -414,7 +414,7 @@ module TSV
|
|
414
414
|
case
|
415
415
|
when (Array === method and (key == :key or key_field == key))
|
416
416
|
with_unnamed do
|
417
|
-
Annotated.purge(method).
|
417
|
+
Annotated.purge(method).uniq.each{|key|
|
418
418
|
new[key] = self[key] if invert ^ (self.include? key)
|
419
419
|
}
|
420
420
|
end
|
data/lib/rbbt/util/color.rb
CHANGED
@@ -30,6 +30,25 @@ class Color
|
|
30
30
|
# Default value for #darken, #lighten etc.
|
31
31
|
BRIGHTNESS_DEFAULT = 0.2
|
32
32
|
|
33
|
+
SOLARIZED = {
|
34
|
+
:base03 => '#002b36',
|
35
|
+
:base02 => '#073642',
|
36
|
+
:base01 => '#586e75',
|
37
|
+
:base00 => '#657b83',
|
38
|
+
:base0 => '#839496',
|
39
|
+
:base1 => '#93a1a1',
|
40
|
+
:base2 => '#eee8d5',
|
41
|
+
:base3 => '#fdf6e3',
|
42
|
+
:yellow => '#b58900',
|
43
|
+
:orange => '#cb4b16',
|
44
|
+
:red => '#dc322f',
|
45
|
+
:magenta => '#d33682',
|
46
|
+
:violet => '#6c71c4',
|
47
|
+
:blue => '#268bd2',
|
48
|
+
:cyan => '#2aa198',
|
49
|
+
:green => '#859900',
|
50
|
+
}
|
51
|
+
|
33
52
|
# Constructor. Inits to white (#FFFFFF) by default, or accepts any params
|
34
53
|
# supported by #parse.
|
35
54
|
def initialize(*args)
|
data/lib/rbbt/util/log.rb
CHANGED
@@ -1,4 +1,8 @@
|
|
1
|
+
require 'term/ansicolor'
|
2
|
+
require 'rbbt/util/color'
|
3
|
+
|
1
4
|
module Log
|
5
|
+
extend Term::ANSIColor
|
2
6
|
|
3
7
|
DEBUG = 0
|
4
8
|
LOW = 1
|
@@ -17,21 +21,24 @@ module Log
|
|
17
21
|
@logfile = nil
|
18
22
|
end
|
19
23
|
|
20
|
-
WHITE, DARK, GREEN, YELLOW, RED =
|
24
|
+
WHITE, DARK, GREEN, YELLOW, RED = Color::SOLARIZED.values_at :base0, :base00, :green, :yellow, :magenta
|
25
|
+
|
26
|
+
SEVERITY_COLOR = [reset, green, yellow, red, green, yellow, red] #.collect{|e| "\033[#{e}"}
|
21
27
|
|
22
|
-
|
28
|
+
HIGHLIGHT = "\033[1m"
|
23
29
|
|
24
30
|
def self.log(message = nil, severity = MEDIUM, &block)
|
25
|
-
|
31
|
+
return if severity < self.severity
|
32
|
+
message ||= block.call if block_given?
|
26
33
|
return if message.nil?
|
34
|
+
|
27
35
|
severity_color = SEVERITY_COLOR[severity]
|
28
|
-
|
36
|
+
time = Time.now.strftime("%m/%d/%y-%H:%M:%S")
|
29
37
|
|
30
|
-
|
31
|
-
message = message
|
32
|
-
|
38
|
+
prefix = time << "[" << severity_color << severity.to_s << SEVERITY_COLOR[0] << "]"
|
39
|
+
message = "" << HIGHLIGHT << message << SEVERITY_COLOR[0] if severity >= INFO
|
40
|
+
str = prefix << " " << message
|
33
41
|
|
34
|
-
str = "\033[0;37m#{Time.now.strftime("[%m/%d/%y-%H:%M:%S]")}#{severity_color}[#{severity.to_s}]\033[0m:#{font_color} " << message.strip << "\033[0m"
|
35
42
|
STDERR.puts str
|
36
43
|
logfile.puts str unless logfile.nil?
|
37
44
|
end
|
data/lib/rbbt/util/simpleopt.rb
CHANGED
@@ -1,74 +1,77 @@
|
|
1
1
|
module SOPT
|
2
|
+
|
3
|
+
#{{{ ACCESSORS
|
4
|
+
|
2
5
|
class << self
|
3
6
|
attr_accessor :command, :summary, :synopsys, :description
|
4
7
|
attr_accessor :inputs, :input_shortcuts, :input_types, :input_descriptions, :input_defaults
|
8
|
+
end
|
5
9
|
|
6
|
-
|
7
|
-
|
8
|
-
|
10
|
+
def self.command
|
11
|
+
@command ||= File.basename($0)
|
12
|
+
end
|
9
13
|
|
10
|
-
|
11
|
-
|
12
|
-
|
14
|
+
def self.summary
|
15
|
+
@summary ||= ""
|
16
|
+
end
|
13
17
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
18
|
+
def self.synopsys
|
19
|
+
@synopsys ||= begin
|
20
|
+
"#{command} " <<
|
21
|
+
inputs.collect{|name|
|
22
|
+
"[" << input_format(name, input_types[name] || :string, input_defaults[name], input_shortcuts[name]).sub(/:$/,'') << "]"
|
23
|
+
} * " "
|
24
|
+
end
|
25
|
+
end
|
22
26
|
|
23
|
-
|
24
|
-
|
25
|
-
|
27
|
+
def self.description
|
28
|
+
@description ||= "Missing"
|
29
|
+
end
|
26
30
|
|
27
31
|
|
28
|
-
|
29
|
-
|
30
|
-
|
32
|
+
def self.shortcuts
|
33
|
+
@shortcuts ||= []
|
34
|
+
end
|
31
35
|
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
def inputs
|
37
|
-
@inputs ||= []
|
38
|
-
end
|
36
|
+
def self.all
|
37
|
+
@all ||= {}
|
38
|
+
end
|
39
39
|
|
40
|
-
|
41
|
-
|
42
|
-
|
40
|
+
def self.inputs
|
41
|
+
@inputs ||= []
|
42
|
+
end
|
43
43
|
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
def input_descriptions
|
49
|
-
@input_descriptions ||= {}
|
50
|
-
end
|
51
|
-
|
52
|
-
def input_defaults
|
53
|
-
@input_defaults ||= {}
|
54
|
-
end
|
44
|
+
def self.input_shortcuts
|
45
|
+
@input_shortcuts ||= {}
|
46
|
+
end
|
55
47
|
|
48
|
+
def self.input_types
|
49
|
+
@input_types ||= {}
|
50
|
+
end
|
56
51
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
end
|
52
|
+
def self.input_descriptions
|
53
|
+
@input_descriptions ||= {}
|
54
|
+
end
|
61
55
|
|
62
|
-
|
63
|
-
|
64
|
-
inputs << input
|
65
|
-
input_types[input] = info[:arg] ? :string : :boolean
|
66
|
-
input_descriptions[input] = info[:description]
|
67
|
-
input_defaults[input] = info[:default]
|
68
|
-
input_shortcuts[input] = info[:short]? info[:short].sub("-",'') : nil
|
69
|
-
end
|
56
|
+
def self.input_defaults
|
57
|
+
@input_defaults ||= {}
|
70
58
|
end
|
71
59
|
|
60
|
+
def self.reset
|
61
|
+
@shortcuts = []
|
62
|
+
@all = {}
|
63
|
+
end
|
64
|
+
|
65
|
+
#{{{ PARSING
|
66
|
+
|
67
|
+
def self.record(info)
|
68
|
+
input = info[:long].sub("--", '')
|
69
|
+
inputs << input
|
70
|
+
input_types[input] = info[:arg] ? :string : :boolean
|
71
|
+
input_descriptions[input] = info[:description]
|
72
|
+
input_defaults[input] = info[:default]
|
73
|
+
input_shortcuts[input] = info[:short]? info[:short].sub("-",'') : nil
|
74
|
+
end
|
72
75
|
|
73
76
|
def self.short_for(name)
|
74
77
|
short = []
|
@@ -84,60 +87,6 @@ module SOPT
|
|
84
87
|
short * ""
|
85
88
|
end
|
86
89
|
|
87
|
-
def self.input_format(name, type = nil, default = nil, short = "")
|
88
|
-
short = short_for(name) if not short.nil? and short.empty?
|
89
|
-
|
90
|
-
input_str = short.nil? ? "--#{name}" : "-#{short}, --#{name}"
|
91
|
-
input_str << case type
|
92
|
-
when nil
|
93
|
-
"#{default != nil ? " (default '#{default}')" : ""}:"
|
94
|
-
when :boolean
|
95
|
-
"[=false]#{default != nil ? " (default '#{default}')" : ""}:"
|
96
|
-
when :tsv, :text
|
97
|
-
"=<filename.#{type}|->#{default != nil ? " (default '#{default}')" : ""}; Use '-' for STDIN:"
|
98
|
-
when :array
|
99
|
-
"=<string[,string]*|filename.list|->#{default != nil ? " (default '#{default}')" : ""}; Use '-' for STDIN:"
|
100
|
-
else
|
101
|
-
"=<#{ type }>#{default != nil ? " (default '#{default}')" : ""}:"
|
102
|
-
end
|
103
|
-
end
|
104
|
-
|
105
|
-
def self.input_doc(inputs, input_types = nil, input_descriptions = nil, input_defaults = nil, input_shortcuts = nil)
|
106
|
-
type = description = default = nil
|
107
|
-
shortcut = ""
|
108
|
-
inputs.collect do |name|
|
109
|
-
|
110
|
-
type = input_types[name] unless input_types.nil?
|
111
|
-
description = input_descriptions[name] unless input_descriptions.nil?
|
112
|
-
default = input_defaults[name] unless input_defaults.nil?
|
113
|
-
shortcut = input_shortcuts[name] unless input_shortcuts.nil?
|
114
|
-
|
115
|
-
type = :string if type.nil?
|
116
|
-
|
117
|
-
str = " * " << SOPT.input_format(name, type.to_sym, default, shortcut) << "\n"
|
118
|
-
str << " " << description << "\n" if description and not description.empty?
|
119
|
-
str
|
120
|
-
end * "\n"
|
121
|
-
end
|
122
|
-
|
123
|
-
def self.doc
|
124
|
-
doc = <<-EOF
|
125
|
-
#{command}(1) -- #{summary}
|
126
|
-
#{"=" * (command.length + summary.length + 7)}
|
127
|
-
|
128
|
-
## SYNOPSYS
|
129
|
-
|
130
|
-
#{synopsys}
|
131
|
-
|
132
|
-
## DESCRIPTION
|
133
|
-
|
134
|
-
#{description}
|
135
|
-
|
136
|
-
## OPTIONS
|
137
|
-
|
138
|
-
#{input_doc(inputs, input_types, input_descriptions, input_defaults, input_shortcuts)}
|
139
|
-
EOF
|
140
|
-
end
|
141
90
|
|
142
91
|
def self.name(info)
|
143
92
|
(info[:long] || info[:short]).sub(/^-*/,'')
|
@@ -225,4 +174,61 @@ module SOPT
|
|
225
174
|
|
226
175
|
options
|
227
176
|
end
|
177
|
+
|
178
|
+
#{{{ DOCUMENTATION
|
179
|
+
|
180
|
+
def self.input_format(name, type = nil, default = nil, short = "")
|
181
|
+
short = short_for(name) if not short.nil? and short.empty?
|
182
|
+
|
183
|
+
input_str = short.nil? ? "--#{name}" : "-#{short}, --#{name}"
|
184
|
+
input_str << case type
|
185
|
+
when nil
|
186
|
+
"#{default != nil ? " (default '#{default}')" : ""}:"
|
187
|
+
when :boolean
|
188
|
+
"[=false]#{default != nil ? " (default '#{default}')" : ""}:"
|
189
|
+
when :tsv, :text
|
190
|
+
"=<filename.#{type}|->#{default != nil ? " (default '#{default}')" : ""}; Use '-' for STDIN:"
|
191
|
+
when :array
|
192
|
+
"=<string[,string]*|filename.list|->#{default != nil ? " (default '#{default}')" : ""}; Use '-' for STDIN:"
|
193
|
+
else
|
194
|
+
"=<#{ type }>#{default != nil ? " (default '#{default}')" : ""}:"
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
def self.input_doc(inputs, input_types = nil, input_descriptions = nil, input_defaults = nil, input_shortcuts = nil)
|
199
|
+
type = description = default = nil
|
200
|
+
shortcut = ""
|
201
|
+
inputs.collect do |name|
|
202
|
+
|
203
|
+
type = input_types[name] unless input_types.nil?
|
204
|
+
description = input_descriptions[name] unless input_descriptions.nil?
|
205
|
+
default = input_defaults[name] unless input_defaults.nil?
|
206
|
+
shortcut = input_shortcuts[name] unless input_shortcuts.nil?
|
207
|
+
|
208
|
+
type = :string if type.nil?
|
209
|
+
|
210
|
+
str = " * " << SOPT.input_format(name, type.to_sym, default, shortcut) << "\n"
|
211
|
+
str << " " << description << "\n" if description and not description.empty?
|
212
|
+
str
|
213
|
+
end * "\n"
|
214
|
+
end
|
215
|
+
|
216
|
+
def self.doc
|
217
|
+
doc = <<-EOF
|
218
|
+
#{command}(1) -- #{summary}
|
219
|
+
#{"=" * (command.length + summary.length + 7)}
|
220
|
+
|
221
|
+
## SYNOPSYS
|
222
|
+
|
223
|
+
#{synopsys}
|
224
|
+
|
225
|
+
## DESCRIPTION
|
226
|
+
|
227
|
+
#{description}
|
228
|
+
|
229
|
+
## OPTIONS
|
230
|
+
|
231
|
+
#{input_doc(inputs, input_types, input_descriptions, input_defaults, input_shortcuts)}
|
232
|
+
EOF
|
233
|
+
end
|
228
234
|
end
|
data/lib/rbbt/workflow.rb
CHANGED
@@ -5,33 +5,16 @@ require 'rbbt/workflow/accessor'
|
|
5
5
|
|
6
6
|
module Workflow
|
7
7
|
|
8
|
-
class << self
|
9
|
-
attr_accessor :autoinstall
|
10
|
-
|
11
|
-
def autoinstall
|
12
|
-
@autoload ||= ENV["RBBT_WORKFLOW_AUTOINSTALL"] == "true"
|
13
|
-
end
|
14
|
-
end
|
15
|
-
|
16
|
-
def self.resolve_locals(inputs)
|
17
|
-
inputs.each do |name, value|
|
18
|
-
if value =~ /^local:(.*?):(.*)/ or
|
19
|
-
(Array === value and value.length == 1 and value.first =~ /^local:(.*?):(.*)/) or
|
20
|
-
(TSV === value and value.size == 1 and value.keys.first =~ /^local:(.*?):(.*)/)
|
21
|
-
task_name = $1
|
22
|
-
jobname = $2
|
23
|
-
value = load_id(File.join(task_name, jobname)).load
|
24
|
-
end
|
25
|
-
inputs[name] = value
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
8
|
#{{{ WORKFLOW MANAGEMENT
|
30
9
|
class << self
|
31
|
-
attr_accessor :workflows
|
10
|
+
attr_accessor :workflows, :autoinstall
|
32
11
|
end
|
12
|
+
|
33
13
|
self.workflows = []
|
34
14
|
|
15
|
+
def self.autoinstall
|
16
|
+
@autoload ||= ENV["RBBT_WORKFLOW_AUTOINSTALL"] == "true"
|
17
|
+
end
|
35
18
|
def self.extended(base)
|
36
19
|
self.workflows << base
|
37
20
|
base.libdir = Path.caller_lib_dir.tap{|p| p.resource = base}
|
@@ -185,10 +168,25 @@ module Workflow
|
|
185
168
|
end
|
186
169
|
|
187
170
|
# {{{ JOB MANAGEMENT
|
171
|
+
DEFAULT_NAME="Default"
|
172
|
+
|
173
|
+
def self.resolve_locals(inputs)
|
174
|
+
inputs.each do |name, value|
|
175
|
+
if value =~ /^local:(.*?):(.*)/ or
|
176
|
+
(Array === value and value.length == 1 and value.first =~ /^local:(.*?):(.*)/) or
|
177
|
+
(TSV === value and value.size == 1 and value.keys.first =~ /^local:(.*?):(.*)/)
|
178
|
+
task_name = $1
|
179
|
+
jobname = $2
|
180
|
+
value = load_id(File.join(task_name, jobname)).load
|
181
|
+
inputs[name] = value
|
182
|
+
end
|
183
|
+
end
|
184
|
+
end
|
188
185
|
|
189
186
|
def job(taskname, jobname = nil, inputs = {})
|
190
187
|
taskname = taskname.to_sym
|
191
|
-
jobname =
|
188
|
+
jobname = DEFAULT_NAME if jobname.nil? or jobname.empty?
|
189
|
+
|
192
190
|
task = tasks[taskname]
|
193
191
|
raise "Task not found: #{ taskname }" if task.nil?
|
194
192
|
|
@@ -244,6 +242,7 @@ module Workflow
|
|
244
242
|
}
|
245
243
|
end
|
246
244
|
|
245
|
+
#{{{ Make workflow resources local
|
247
246
|
def local_persist_setup
|
248
247
|
class << self
|
249
248
|
include LocalPersist
|
@@ -6,7 +6,7 @@ class Step
|
|
6
6
|
INFO_SERIALIAZER = Marshal
|
7
7
|
|
8
8
|
def name
|
9
|
-
|
9
|
+
path.sub(/.*\/#{Regexp.quote task.name.to_s}\/(.*)/, '\1')
|
10
10
|
end
|
11
11
|
|
12
12
|
def clean_name
|
@@ -21,7 +21,7 @@ class Step
|
|
21
21
|
|
22
22
|
def info_file
|
23
23
|
@info_file ||= begin
|
24
|
-
|
24
|
+
path.nil? ? nil : path + '.info'
|
25
25
|
end
|
26
26
|
end
|
27
27
|
|
@@ -110,7 +110,7 @@ class Step
|
|
110
110
|
# {{{ INFO
|
111
111
|
|
112
112
|
def files_dir
|
113
|
-
|
113
|
+
path + '.files'
|
114
114
|
end
|
115
115
|
|
116
116
|
def files
|
@@ -275,19 +275,21 @@ module Workflow
|
|
275
275
|
|
276
276
|
TAG = :hash
|
277
277
|
def step_path(taskname, jobname, inputs, dependencies)
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
278
|
+
Proc.new{
|
279
|
+
raise "Jobname makes an invalid path: #{ jobname }" if jobname =~ /\.\./
|
280
|
+
if inputs.any? or dependencies.any?
|
281
|
+
tagged_jobname = case TAG
|
282
|
+
when :hash
|
283
|
+
jobname + '_' + Misc.digest((inputs * "\n" + ";" + dependencies.collect{|dep| dep.name} * "\n"))
|
284
|
+
else
|
285
|
+
jobname
|
286
|
+
end
|
287
|
+
else
|
288
|
+
tagged_jobname = jobname
|
289
|
+
end
|
289
290
|
|
290
|
-
|
291
|
+
workdir[taskname][tagged_jobname].find
|
292
|
+
}
|
291
293
|
end
|
292
294
|
|
293
295
|
|
data/lib/rbbt/workflow/step.rb
CHANGED
@@ -14,8 +14,8 @@ class Step
|
|
14
14
|
class Aborted < Exception; end
|
15
15
|
|
16
16
|
def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil)
|
17
|
-
path = Misc.sanitize_filename path
|
18
|
-
@path =
|
17
|
+
path = Path.setup(Misc.sanitize_filename(path)) if String === path
|
18
|
+
@path = path
|
19
19
|
@task = task
|
20
20
|
@bindings = bindings
|
21
21
|
@dependencies = case
|
@@ -29,6 +29,11 @@ class Step
|
|
29
29
|
@inputs = inputs || []
|
30
30
|
end
|
31
31
|
|
32
|
+
def path
|
33
|
+
@path = Misc.sanitize_filename(Path.setup(@path.call)) if Proc === @path
|
34
|
+
@path
|
35
|
+
end
|
36
|
+
|
32
37
|
class << self
|
33
38
|
attr_accessor :log_relay_step
|
34
39
|
end
|
@@ -88,7 +93,7 @@ class Step
|
|
88
93
|
end
|
89
94
|
|
90
95
|
def run(no_load = false)
|
91
|
-
result = Persist.persist "Job", @task.result_type, :file =>
|
96
|
+
result = Persist.persist "Job", @task.result_type, :file => path, :check => rec_dependencies.collect{|dependency| dependency.path }.uniq, :no_load => no_load do
|
92
97
|
@exec = false
|
93
98
|
if Step === Step.log_relay_step and not self == Step.log_relay_step
|
94
99
|
relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
|
@@ -177,11 +182,11 @@ class Step
|
|
177
182
|
begin
|
178
183
|
run(true)
|
179
184
|
rescue Step::Aborted
|
180
|
-
Log.debug{"Forked process aborted: #{
|
185
|
+
Log.debug{"Forked process aborted: #{path}"}
|
181
186
|
log :aborted, "Aborted"
|
182
187
|
raise $!
|
183
188
|
rescue Exception
|
184
|
-
Log.debug("Exception caught on forked process: #{
|
189
|
+
Log.debug("Exception caught on forked process: #{path}")
|
185
190
|
raise $!
|
186
191
|
end
|
187
192
|
|
@@ -1,11 +1,12 @@
|
|
1
1
|
require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
|
2
2
|
require 'rbbt/workflow/task'
|
3
3
|
require 'rbbt/workflow/step'
|
4
|
+
require 'rbbt/tsv'
|
4
5
|
require 'rbbt'
|
5
6
|
|
6
7
|
class TestStep < Test::Unit::TestCase
|
7
8
|
|
8
|
-
def
|
9
|
+
def _test_step
|
9
10
|
task = Task.setup do "TEST" end
|
10
11
|
task2 = Task.setup do raise "Persistence ignored" end
|
11
12
|
TmpFile.with_file do |tmp|
|
@@ -17,7 +18,7 @@ class TestStep < Test::Unit::TestCase
|
|
17
18
|
end
|
18
19
|
end
|
19
20
|
|
20
|
-
def
|
21
|
+
def _test_dependency
|
21
22
|
str = "TEST"
|
22
23
|
str2 = "TEST2"
|
23
24
|
TmpFile.with_file do |tmpfile|
|
@@ -46,7 +47,7 @@ class TestStep < Test::Unit::TestCase
|
|
46
47
|
end
|
47
48
|
end
|
48
49
|
|
49
|
-
def
|
50
|
+
def _test_dependency_log_relay
|
50
51
|
str = "TEST"
|
51
52
|
TmpFile.with_file do |tmpfile|
|
52
53
|
task1 = Task.setup :result_type => nil, :name => :task1 do
|
@@ -65,7 +66,7 @@ class TestStep < Test::Unit::TestCase
|
|
65
66
|
end
|
66
67
|
end
|
67
68
|
|
68
|
-
def
|
69
|
+
def _test_log_relay_step
|
69
70
|
str = "TEST"
|
70
71
|
TmpFile.with_file do |tmpfile|
|
71
72
|
task1 = Task.setup :result_type => nil, :name => :task1 do
|
@@ -87,7 +88,7 @@ class TestStep < Test::Unit::TestCase
|
|
87
88
|
end
|
88
89
|
|
89
90
|
|
90
|
-
def
|
91
|
+
def _test_exec
|
91
92
|
TmpFile.with_file do |lock|
|
92
93
|
task = Task.setup do "TEST" end
|
93
94
|
TmpFile.with_file do |tmp|
|
@@ -98,7 +99,7 @@ class TestStep < Test::Unit::TestCase
|
|
98
99
|
end
|
99
100
|
|
100
101
|
|
101
|
-
def
|
102
|
+
def _test_fork
|
102
103
|
TmpFile.with_file do |lock|
|
103
104
|
task = Task.setup do while not File.exists?(lock) do sleep 1; end; "TEST" end
|
104
105
|
TmpFile.with_file do |tmp|
|
@@ -114,7 +115,7 @@ class TestStep < Test::Unit::TestCase
|
|
114
115
|
end
|
115
116
|
end
|
116
117
|
|
117
|
-
def
|
118
|
+
def _test_abort
|
118
119
|
TmpFile.with_file do |lock|
|
119
120
|
task = Task.setup do while not File.exists?(lock) do sleep 1; end; "TEST" end
|
120
121
|
TmpFile.with_file do |tmp|
|
@@ -131,7 +132,7 @@ class TestStep < Test::Unit::TestCase
|
|
131
132
|
end
|
132
133
|
end
|
133
134
|
|
134
|
-
def
|
135
|
+
def _test_files
|
135
136
|
TmpFile.with_file do |lock|
|
136
137
|
task = Task.setup do
|
137
138
|
Open.write(file("test"),"TEST")
|
@@ -145,7 +146,7 @@ class TestStep < Test::Unit::TestCase
|
|
145
146
|
end
|
146
147
|
end
|
147
148
|
|
148
|
-
def
|
149
|
+
def _test_messages
|
149
150
|
TmpFile.with_file do |lock|
|
150
151
|
task = Task.setup do
|
151
152
|
message "WRITE"
|
@@ -161,7 +162,7 @@ class TestStep < Test::Unit::TestCase
|
|
161
162
|
end
|
162
163
|
end
|
163
164
|
|
164
|
-
def
|
165
|
+
def _test_subdir
|
165
166
|
TmpFile.with_file do |lock|
|
166
167
|
task = Task.setup do
|
167
168
|
message "WRITE"
|
@@ -178,7 +179,7 @@ class TestStep < Test::Unit::TestCase
|
|
178
179
|
end
|
179
180
|
end
|
180
181
|
|
181
|
-
def
|
182
|
+
def _test_semaphore
|
182
183
|
TmpFile.with_file do |semaphore|
|
183
184
|
begin
|
184
185
|
semaphore = "/" << semaphore.gsub('/','_')
|
@@ -210,5 +211,4 @@ class TestStep < Test::Unit::TestCase
|
|
210
211
|
|
211
212
|
end
|
212
213
|
|
213
|
-
|
214
214
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.5.
|
4
|
+
version: 5.5.59
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
@@ -94,6 +94,20 @@ dependencies:
|
|
94
94
|
- - ! '>='
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: term-ansicolor
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ! '>='
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ! '>='
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
97
111
|
- !ruby/object:Gem::Dependency
|
98
112
|
name: ZenTest
|
99
113
|
requirement: !ruby/object:Gem::Requirement
|