rbbt-util 4.0.0 → 4.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/run_workflow.rb +1 -1
- data/lib/rbbt/annotations.rb +17 -6
- data/lib/rbbt/persist.rb +7 -1
- data/lib/rbbt/tsv.rb +14 -1
- data/lib/rbbt/tsv/accessor.rb +13 -2
- data/lib/rbbt/tsv/attach.rb +2 -0
- data/lib/rbbt/tsv/manipulate.rb +54 -26
- data/lib/rbbt/tsv/parser.rb +37 -9
- data/lib/rbbt/util/chain_methods.rb +11 -1
- data/lib/rbbt/util/cmd.rb +1 -1
- data/lib/rbbt/util/misc.rb +44 -2
- data/lib/rbbt/workflow.rb +104 -6
- data/lib/rbbt/workflow/accessor.rb +8 -2
- data/lib/rbbt/workflow/annotate.rb +9 -1
- data/lib/rbbt/workflow/step.rb +5 -3
- data/lib/rbbt/workflow/task.rb +1 -1
- data/test/rbbt/test_fix_width_table.rb +1 -0
- data/test/rbbt/test_tsv.rb +31 -0
- data/test/rbbt/tsv/test_accessor.rb +10 -10
- metadata +7 -7
data/bin/run_workflow.rb
CHANGED
@@ -115,7 +115,7 @@ else
|
|
115
115
|
|
116
116
|
if remote_workflows.include? workflow
|
117
117
|
require 'rbbt/workflow/rest/client'
|
118
|
-
workflow = RbbtRestClient.new remote_workflows[workflow]
|
118
|
+
workflow = RbbtRestClient.new remote_workflows[workflow], workflow
|
119
119
|
else
|
120
120
|
Workflow.require_workflow workflow
|
121
121
|
workflow = Workflow.workflows.last
|
data/lib/rbbt/annotations.rb
CHANGED
@@ -128,7 +128,12 @@ module Annotation
|
|
128
128
|
base.all_inheritance = []
|
129
129
|
|
130
130
|
base.module_eval do
|
131
|
+
class << self
|
132
|
+
alias prev_annotation_extended extended
|
133
|
+
end
|
134
|
+
|
131
135
|
def self.extended(object)
|
136
|
+
self.send(:prev_annotation_extended, object)
|
132
137
|
object.extend Annotated
|
133
138
|
if not object.annotation_types.include? self
|
134
139
|
object.annotation_types.concat self.inheritance
|
@@ -178,6 +183,7 @@ module Annotation
|
|
178
183
|
|
179
184
|
object
|
180
185
|
end
|
186
|
+
|
181
187
|
end
|
182
188
|
|
183
189
|
module AnnotatedArray
|
@@ -193,23 +199,28 @@ module AnnotatedArray
|
|
193
199
|
end
|
194
200
|
|
195
201
|
def annotated_array_each
|
196
|
-
|
202
|
+
annotated_array_clean_each do |value|
|
197
203
|
annotation_types.each do |mod|
|
198
|
-
mod.setup(value,
|
204
|
+
mod.setup(value, *info.values_at(*mod.annotations))
|
199
205
|
end
|
200
206
|
yield value
|
201
207
|
end
|
202
208
|
end
|
203
209
|
|
204
210
|
def annotated_array_collect
|
205
|
-
|
211
|
+
annotated_array_clean_collect do |value|
|
206
212
|
annotation_types.each do |mod|
|
207
|
-
mod.setup(value,
|
213
|
+
mod.setup(value, *info.values_at(*mod.annotations))
|
208
214
|
end
|
209
215
|
yield value
|
210
216
|
end
|
211
217
|
end
|
212
218
|
|
219
|
+
def annotated_array_subset(list)
|
220
|
+
value = (self & list)
|
221
|
+
annotation_types.each do |mod|
|
222
|
+
mod.setup(value, *info.values_at(*mod.annotations))
|
223
|
+
end
|
224
|
+
value
|
225
|
+
end
|
213
226
|
end
|
214
|
-
|
215
|
-
|
data/lib/rbbt/persist.rb
CHANGED
@@ -4,6 +4,7 @@ require 'rbbt/util/misc'
|
|
4
4
|
require 'rbbt/util/open'
|
5
5
|
|
6
6
|
require 'rbbt/persist/tsv'
|
7
|
+
require 'set'
|
7
8
|
|
8
9
|
module Persist
|
9
10
|
CACHEDIR="/tmp/tsv_persistent_cache"
|
@@ -67,10 +68,13 @@ module Persist
|
|
67
68
|
File.join(persistence_dir, filename)
|
68
69
|
end
|
69
70
|
|
71
|
+
TRUE_STRINGS = Set.new ["true", "True", "TRUE", "t", "T", "1", "yes", "Yes", "YES", "y", "Y", "ON", "on"]
|
70
72
|
def self.load_file(path, type)
|
71
73
|
case (type || "nil").to_sym
|
72
74
|
when :nil
|
73
75
|
nil
|
76
|
+
when :boolean
|
77
|
+
TRUE_STRINGS.include? Open.read(path).chomp.strip
|
74
78
|
when :tsv
|
75
79
|
TSV.open(path)
|
76
80
|
when :marshal_tsv
|
@@ -105,6 +109,8 @@ module Persist
|
|
105
109
|
case (type || "nil").to_sym
|
106
110
|
when :nil
|
107
111
|
nil
|
112
|
+
when :boolean
|
113
|
+
Open.write(path, content ? "true" : "false")
|
108
114
|
when :fwt
|
109
115
|
content.file.seek 0
|
110
116
|
Open.write(path, content.file.read)
|
@@ -113,7 +119,7 @@ module Persist
|
|
113
119
|
when :string, :text
|
114
120
|
Open.write(path, content)
|
115
121
|
when :array
|
116
|
-
Open.write(path, content * "\n"
|
122
|
+
Open.write(path, content * "\n")
|
117
123
|
when :marshal_tsv
|
118
124
|
Open.write(path, Marshal.dump(content.dup))
|
119
125
|
when :marshal
|
data/lib/rbbt/tsv.rb
CHANGED
@@ -86,7 +86,18 @@ module TSV
|
|
86
86
|
data.extend TSV unless TSV === data
|
87
87
|
data.unnamed = true
|
88
88
|
|
89
|
-
|
89
|
+
if data.serializer == :type
|
90
|
+
data.serializer = case
|
91
|
+
when parser.cast.nil?
|
92
|
+
data.serializer = parser.type
|
93
|
+
when (parser.cast == :to_i and parser.type == :list)
|
94
|
+
data.serializer = :integer_array
|
95
|
+
when (parser.cast == :to_i and parser.type == :single)
|
96
|
+
data.serializer = :integer
|
97
|
+
when (parser.cast == :to_f and parser.type == :single)
|
98
|
+
data.serializer = :float
|
99
|
+
end
|
100
|
+
end
|
90
101
|
|
91
102
|
if monitor and (stream.respond_to?(:size) or (stream.respond_to?(:stat) and stream.stat.respond_to? :size)) and stream.respond_to?(:pos)
|
92
103
|
size = case
|
@@ -120,6 +131,8 @@ module TSV
|
|
120
131
|
rescue Parser::SKIP_LINE
|
121
132
|
line = stream.gets
|
122
133
|
next
|
134
|
+
rescue Parser::END_PARSING
|
135
|
+
break
|
123
136
|
rescue IOError
|
124
137
|
break
|
125
138
|
end
|
data/lib/rbbt/tsv/accessor.rb
CHANGED
@@ -145,8 +145,19 @@ module TSV
|
|
145
145
|
elems = collect
|
146
146
|
else
|
147
147
|
elems = []
|
148
|
-
|
149
|
-
|
148
|
+
case type
|
149
|
+
when :single
|
150
|
+
through :key, field do |key, field|
|
151
|
+
elems << [key, field]
|
152
|
+
end
|
153
|
+
when :list, :flat
|
154
|
+
through :key, field do |key, fields|
|
155
|
+
elems << [key, fields.first]
|
156
|
+
end
|
157
|
+
when :double
|
158
|
+
through :key, field do |key, fields|
|
159
|
+
elems << [key, fields.first]
|
160
|
+
end
|
150
161
|
end
|
151
162
|
end
|
152
163
|
|
data/lib/rbbt/tsv/attach.rb
CHANGED
data/lib/rbbt/tsv/manipulate.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'progress-bar'
|
2
2
|
require 'rbbt/persist'
|
3
3
|
require 'rbbt/tsv/util'
|
4
|
+
require 'set'
|
4
5
|
|
5
6
|
module TSV
|
6
7
|
|
@@ -57,27 +58,27 @@ module TSV
|
|
57
58
|
|
58
59
|
raise "Key field #{ new_key_field } not found" if @new_key_field.nil?
|
59
60
|
@new_fields = case
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
61
|
+
when new_fields.nil?
|
62
|
+
case
|
63
|
+
when @new_key_field == :key
|
64
|
+
:all
|
65
|
+
when fields.nil?
|
66
|
+
- @new_key_field
|
67
|
+
else
|
68
|
+
new = (0..fields.length - 1).to_a
|
69
|
+
new.delete_at(@new_key_field)
|
70
|
+
new.unshift :key
|
71
|
+
new
|
72
|
+
end
|
73
|
+
when Array === new_fields
|
74
|
+
new_fields.collect do |field|
|
75
|
+
TSV.identify_field(key_field, fields, field)
|
76
|
+
end
|
77
|
+
when (String === new_fields or Symbol === new_fields)
|
78
|
+
[TSV.identify_field(key_field, fields, new_fields)]
|
79
|
+
else
|
80
|
+
raise "Unknown format for new_fields (should be nil, Array or String): #{new_fields.inspect}"
|
81
|
+
end
|
81
82
|
|
82
83
|
@new_key_field_name = case
|
83
84
|
when @new_key_field == :key
|
@@ -235,9 +236,21 @@ module TSV
|
|
235
236
|
new[key] = values if yield key, values
|
236
237
|
end
|
237
238
|
when Array === method
|
239
|
+
method = Set.new method
|
238
240
|
with_unnamed do
|
239
|
-
|
240
|
-
|
241
|
+
case type
|
242
|
+
when :single
|
243
|
+
through do |key, value|
|
244
|
+
new[key] = values if method.include? key or method.include? value
|
245
|
+
end
|
246
|
+
when :list, :flat
|
247
|
+
through do |key, values|
|
248
|
+
new[key] = values if method.include? key or (method & values).any?
|
249
|
+
end
|
250
|
+
else
|
251
|
+
through do |key, values|
|
252
|
+
new[key] = values if method.include? key or (method & values.flatten).any?
|
253
|
+
end
|
241
254
|
end
|
242
255
|
end
|
243
256
|
when Regexp === method
|
@@ -273,9 +286,24 @@ module TSV
|
|
273
286
|
end
|
274
287
|
when Array === method
|
275
288
|
with_unnamed do
|
276
|
-
|
277
|
-
|
278
|
-
|
289
|
+
method = Set.new method unless Set === method
|
290
|
+
case type
|
291
|
+
when :single
|
292
|
+
through :key, key do |key, value|
|
293
|
+
new[key] = self[key] if method.include? value
|
294
|
+
end
|
295
|
+
when :list
|
296
|
+
through :key, key do |key, values|
|
297
|
+
new[key] = self[key] if method.include? value.first
|
298
|
+
end
|
299
|
+
when :flat #untested
|
300
|
+
through :key, key do |key, values|
|
301
|
+
new[key] = self[key] if (method & values.flatten).any?
|
302
|
+
end
|
303
|
+
else
|
304
|
+
through :key, key do |key, values|
|
305
|
+
new[key] = self[key] if (method & values.first).any?
|
306
|
+
end
|
279
307
|
end
|
280
308
|
end
|
281
309
|
when Regexp === method
|
data/lib/rbbt/tsv/parser.rb
CHANGED
@@ -4,6 +4,7 @@ module TSV
|
|
4
4
|
attr_accessor :header_hash, :sep, :sep2, :type, :key_position, :field_positions, :cast, :key_field, :fields, :fix, :select, :serializer, :straight
|
5
5
|
|
6
6
|
class SKIP_LINE < Exception; end
|
7
|
+
class END_PARSING < Exception; end
|
7
8
|
|
8
9
|
def all_fields
|
9
10
|
all = [key_field] + fields
|
@@ -49,6 +50,7 @@ module TSV
|
|
49
50
|
l = line.chomp
|
50
51
|
raise Parser::SKIP_LINE if Proc === @select and not @select.call l
|
51
52
|
l = @fix.call l if Proc === @fix
|
53
|
+
raise Parser::END_PARSING unless l
|
52
54
|
l
|
53
55
|
end
|
54
56
|
|
@@ -85,6 +87,22 @@ module TSV
|
|
85
87
|
data[key] = values unless data.include? key
|
86
88
|
end
|
87
89
|
|
90
|
+
def add_to_data_flat(data, keys, values)
|
91
|
+
keys.each do |key|
|
92
|
+
data[key] = values.flatten unless data.include? key
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
def add_to_data_flat_merge(data, keys, values)
|
97
|
+
keys.each do |key|
|
98
|
+
if data.include? key
|
99
|
+
data[key] = data[key].concat values.flatten
|
100
|
+
else
|
101
|
+
data[key] = values.flatten
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
88
106
|
def add_to_data_no_merge_double(data, keys, values)
|
89
107
|
keys.each do |key|
|
90
108
|
data[key] = values unless data.include? key
|
@@ -168,7 +186,9 @@ module TSV
|
|
168
186
|
when Integer === field
|
169
187
|
field
|
170
188
|
when String === field
|
171
|
-
@fields.dup.unshift(@key_field).index field
|
189
|
+
pos = @fields.dup.unshift(@key_field).index field
|
190
|
+
raise "Field not identified: #{ field }" if pos.nil?
|
191
|
+
pos
|
172
192
|
else
|
173
193
|
raise "Format of fields not understood: #{fields.inspect}"
|
174
194
|
end
|
@@ -196,7 +216,8 @@ module TSV
|
|
196
216
|
@fix = Misc.process_options(options, :fix)
|
197
217
|
@select= Misc.process_options options, :select
|
198
218
|
|
199
|
-
|
219
|
+
case @type
|
220
|
+
when :double
|
200
221
|
self.instance_eval do alias get_values get_values_double end
|
201
222
|
self.instance_eval do alias cast_values cast_values_double end
|
202
223
|
if merge
|
@@ -204,15 +225,22 @@ module TSV
|
|
204
225
|
else
|
205
226
|
self.instance_eval do alias add_to_data add_to_data_no_merge_double end
|
206
227
|
end
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
228
|
+
when :single
|
229
|
+
self.instance_eval do alias get_values get_values_single end
|
230
|
+
self.instance_eval do alias cast_values cast_values_single end
|
231
|
+
self.instance_eval do alias add_to_data add_to_data_no_merge_list end
|
232
|
+
when :list
|
233
|
+
self.instance_eval do alias get_values get_values_list end
|
234
|
+
self.instance_eval do alias cast_values cast_values_list end
|
235
|
+
self.instance_eval do alias add_to_data add_to_data_no_merge_list end
|
236
|
+
when :flat
|
237
|
+
self.instance_eval do alias get_values get_values_double end
|
238
|
+
self.instance_eval do alias cast_values cast_values_double end
|
239
|
+
if merge
|
240
|
+
self.instance_eval do alias add_to_data add_to_data_flat_merge end
|
211
241
|
else
|
212
|
-
self.instance_eval do alias
|
213
|
-
self.instance_eval do alias cast_values cast_values_list end
|
242
|
+
self.instance_eval do alias add_to_data add_to_data_flat end
|
214
243
|
end
|
215
|
-
self.instance_eval do alias add_to_data add_to_data_no_merge_list end
|
216
244
|
end
|
217
245
|
|
218
246
|
fix_fields(options)
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'rbbt/util/log'
|
2
2
|
|
3
3
|
module ChainMethods
|
4
|
-
def self.
|
4
|
+
def self.chain_methods_extended(base)
|
5
5
|
if not base.respond_to? :chain_prefix
|
6
6
|
metaclass = class << base
|
7
7
|
attr_accessor :chain_prefix, :chained_methods
|
@@ -52,7 +52,10 @@ module ChainMethods
|
|
52
52
|
|
53
53
|
if not metaclass.respond_to? :extended
|
54
54
|
metaclass.module_eval do
|
55
|
+
alias prev_chain_methods_extended extended
|
56
|
+
|
55
57
|
def extended(base)
|
58
|
+
prev_chain_methods_extended(base)
|
56
59
|
setup_chains(base)
|
57
60
|
end
|
58
61
|
end
|
@@ -61,4 +64,11 @@ module ChainMethods
|
|
61
64
|
|
62
65
|
base.chain_prefix = base.to_s.downcase.to_sym
|
63
66
|
end
|
67
|
+
|
68
|
+
def self.extended(base)
|
69
|
+
chain_methods_extended(base)
|
70
|
+
end
|
71
|
+
|
72
|
+
|
73
|
+
|
64
74
|
end
|
data/lib/rbbt/util/cmd.rb
CHANGED
data/lib/rbbt/util/misc.rb
CHANGED
@@ -1,10 +1,18 @@
|
|
1
1
|
require 'lockfile'
|
2
2
|
require 'rbbt/util/chain_methods'
|
3
3
|
require 'rbbt/resource/path'
|
4
|
+
require 'rbbt/annotations'
|
4
5
|
|
5
6
|
module Misc
|
6
7
|
class FieldNotFoundError < StandardError;end
|
7
8
|
|
9
|
+
def self.counts(array)
|
10
|
+
counts = Hash.new 0
|
11
|
+
array.each do |e|
|
12
|
+
counts[e] += 1
|
13
|
+
end
|
14
|
+
counts
|
15
|
+
end
|
8
16
|
|
9
17
|
IUPAC2BASE = {
|
10
18
|
"A" => ["A"],
|
@@ -261,7 +269,7 @@ module Misc
|
|
261
269
|
when String === options
|
262
270
|
new_options = string2hash options
|
263
271
|
else
|
264
|
-
raise "Format of '#{options.inspect}' not understood"
|
272
|
+
raise "Format of '#{options.inspect}' not understood. It should be a hash"
|
265
273
|
end
|
266
274
|
defaults.each do |key, value|
|
267
275
|
next unless new_options[key].nil?
|
@@ -444,7 +452,41 @@ module NamedArray
|
|
444
452
|
end
|
445
453
|
|
446
454
|
def named_array_get_brackets(key)
|
447
|
-
|
455
|
+
if defined? Entity
|
456
|
+
entity = (defined?(Entity) and Entity.respond_to?(:formats)) ? Entity.formats[key] : nil
|
457
|
+
if entity
|
458
|
+
if entity.annotations.first == :format
|
459
|
+
entity.setup(named_array_clean_get_brackets(Misc.field_position(fields, key)), key)
|
460
|
+
else
|
461
|
+
entity.setup(named_array_clean_get_brackets(Misc.field_position(fields, key)))
|
462
|
+
end
|
463
|
+
else
|
464
|
+
named_array_clean_get_brackets(Misc.field_position(fields, key))
|
465
|
+
end
|
466
|
+
else
|
467
|
+
named_array_clean_get_brackets(Misc.field_position(fields, key))
|
468
|
+
end
|
469
|
+
end
|
470
|
+
|
471
|
+
def named_array_each(&block)
|
472
|
+
if defined?(Entity) and not fields.nil? and not fields.empty?
|
473
|
+
fields.zip(self) do |field,elem|
|
474
|
+
entity = (defined?(Entity) and Entity.respond_to?(:formats)) ? Entity.formats[field] : nil
|
475
|
+
if entity
|
476
|
+
elem = elem.dup if elem.frozen?
|
477
|
+
if entity.annotations.first == :format
|
478
|
+
elem = entity.setup(elem, field)
|
479
|
+
else
|
480
|
+
elem = entity.setup(elem)
|
481
|
+
end
|
482
|
+
else
|
483
|
+
end
|
484
|
+
yield(elem)
|
485
|
+
elem
|
486
|
+
end
|
487
|
+
else
|
488
|
+
named_array_clean_each &block
|
489
|
+
end
|
448
490
|
end
|
449
491
|
|
450
492
|
def named_array_set_brackets(key,value)
|
data/lib/rbbt/workflow.rb
CHANGED
@@ -9,7 +9,7 @@ module Workflow
|
|
9
9
|
end
|
10
10
|
self.workflows = []
|
11
11
|
|
12
|
-
def self.
|
12
|
+
def self.require_local_workflow2(wf_name, wf_dir = nil)
|
13
13
|
require 'rbbt/resource/path'
|
14
14
|
|
15
15
|
if File.exists?(wf_name) or File.exists?(wf_name + '.rb')
|
@@ -46,6 +46,103 @@ module Workflow
|
|
46
46
|
require wf_dir["workflow.rb"].find
|
47
47
|
end
|
48
48
|
|
49
|
+
def self.require_remote_workflow(wf_name, url)
|
50
|
+
require 'rbbt/workflow/rest/client'
|
51
|
+
eval "Object::#{wf_name} = RbbtRestClient.new '#{ url }', '#{wf_name}'"
|
52
|
+
end
|
53
|
+
|
54
|
+
def self.require_local_workflow(wf_name)
|
55
|
+
if Path === wf_name
|
56
|
+
case
|
57
|
+
|
58
|
+
# Points to workflow file
|
59
|
+
when ((File.exists?(wf_name.find) and not File.directory?(wf_name.find)) or File.exists?(wf_name.find + '.rb'))
|
60
|
+
$LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(wf_name.find)), 'lib'))
|
61
|
+
require wf_name.find
|
62
|
+
Log.debug "Workflow loaded from file: #{ wf_name }"
|
63
|
+
return true
|
64
|
+
|
65
|
+
# Points to workflow dir
|
66
|
+
when (File.exists?(wf_name.find) and File.directory?(wf_name.find) and File.exists?(File.join(wf_name.find, 'workflow.rb')))
|
67
|
+
$LOAD_PATH.unshift(File.join(File.expand_path(wf_name.find), 'lib'))
|
68
|
+
require File.join(wf_name.find, 'workflow.rb')
|
69
|
+
Log.debug "Workflow loaded from directory: #{ wf_name }"
|
70
|
+
return true
|
71
|
+
|
72
|
+
else
|
73
|
+
raise "Workflow path was not resolved: #{ wf_name } (#{wf_name.find})"
|
74
|
+
end
|
75
|
+
|
76
|
+
else
|
77
|
+
case
|
78
|
+
|
79
|
+
# Points to workflow file
|
80
|
+
when ((File.exists?(wf_name) and not File.directory?(wf_name)) or File.exists?(wf_name + '.rb'))
|
81
|
+
$LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(wf_name)), 'lib'))
|
82
|
+
require wf_name
|
83
|
+
Log.debug "Workflow loaded from file: #{ wf_name }"
|
84
|
+
return true
|
85
|
+
|
86
|
+
when (defined?(Rbbt) and Rbbt.etc.workflow_dir.exists?)
|
87
|
+
dir = Rbbt.etc.workflow_dir.read.strip
|
88
|
+
dir = File.join(dir, wf_name)
|
89
|
+
$LOAD_PATH.unshift(File.join(File.expand_path(dir), 'lib'))
|
90
|
+
require File.join(dir, 'workflow.rb')
|
91
|
+
Log.debug "Workflow #{wf_name} loaded from workflow_dir: #{ dir }"
|
92
|
+
return true
|
93
|
+
|
94
|
+
when defined?(Rbbt)
|
95
|
+
path = Rbbt.workflows[wf_name].find
|
96
|
+
$LOAD_PATH.unshift(File.join(File.expand_path(path), 'lib'))
|
97
|
+
require File.join(path, 'workflow.rb')
|
98
|
+
Log.debug "Workflow #{wf_name} loaded from Rbbt.workflows: #{ path }"
|
99
|
+
return true
|
100
|
+
|
101
|
+
else
|
102
|
+
path = File.join(ENV['HOME'], '.workflows', wf_name)
|
103
|
+
$LOAD_PATH.unshift(File.join(File.expand_path(path), 'lib'))
|
104
|
+
require File.join(path, 'workflow.rb')
|
105
|
+
Log.debug "Workflow #{wf_name} loaded from .workflows: #{ path }"
|
106
|
+
return true
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
raise "Workflow not found: #{ wf_name }"
|
111
|
+
end
|
112
|
+
|
113
|
+
def self.require_workflow(wf_name)
|
114
|
+
begin
|
115
|
+
Misc.string2const wf_name
|
116
|
+
Log.debug "Workflow #{ wf_name } already loaded"
|
117
|
+
return true
|
118
|
+
rescue Exception
|
119
|
+
end
|
120
|
+
|
121
|
+
if Rbbt.etc.remote_workflows.exists?
|
122
|
+
remote_workflows = Rbbt.etc.remote_workflows.yaml
|
123
|
+
if remote_workflows.include? wf_name
|
124
|
+
url = remote_workflows[wf_name]
|
125
|
+
require_remote_workflow(wf_name, url)
|
126
|
+
Log.debug "Workflow #{ wf_name } loaded remotely: #{ url }"
|
127
|
+
return
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
begin
|
132
|
+
require_local_workflow(wf_name)
|
133
|
+
rescue Exception
|
134
|
+
Log.debug $!.message
|
135
|
+
raise "Workflow not found: #{ wf_name }" if wf_name == wf_name.downcase
|
136
|
+
Log.debug "Trying with downcase: '#{wf_name.downcase}'"
|
137
|
+
begin
|
138
|
+
require_local_workflow(wf_name.downcase)
|
139
|
+
rescue Exception
|
140
|
+
Log.debug $!.message
|
141
|
+
raise "Workflow not found: #{ wf_name }"
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
49
146
|
def self.extended(base)
|
50
147
|
if not base.respond_to? :workdir
|
51
148
|
base.extend AnnotatedModule
|
@@ -59,7 +156,7 @@ module Workflow
|
|
59
156
|
def task_dependencies
|
60
157
|
IndiferentHash.setup(@task_dependencies || {})
|
61
158
|
end
|
62
|
-
|
159
|
+
|
63
160
|
def tasks
|
64
161
|
IndiferentHash.setup(@tasks || {})
|
65
162
|
end
|
@@ -83,7 +180,7 @@ module Workflow
|
|
83
180
|
end
|
84
181
|
|
85
182
|
# {{{ Task definition helpers
|
86
|
-
|
183
|
+
|
87
184
|
def task(name, &block)
|
88
185
|
if Hash === name
|
89
186
|
result_type = name.first.last
|
@@ -104,7 +201,8 @@ module Workflow
|
|
104
201
|
:input_types => input_types,
|
105
202
|
:result_type => Array == result_type ? result_type.to_sym : result_type,
|
106
203
|
:input_defaults => input_defaults,
|
107
|
-
:input_descriptions => input_descriptions
|
204
|
+
:input_descriptions => input_descriptions,
|
205
|
+
:result_description => result_description
|
108
206
|
}, &block)
|
109
207
|
|
110
208
|
@last_task = task
|
@@ -125,7 +223,7 @@ module Workflow
|
|
125
223
|
end
|
126
224
|
|
127
225
|
# {{{ Job management
|
128
|
-
|
226
|
+
|
129
227
|
def resolve_locals(inputs)
|
130
228
|
inputs.each do |name, value|
|
131
229
|
if value =~ /^local:(.*?):(.*)/ or
|
@@ -172,7 +270,7 @@ module Workflow
|
|
172
270
|
step = Step.new path, tasks[task]
|
173
271
|
if step.info.include? :dependencies
|
174
272
|
step.dependencies = step.info[:dependencies].collect do |task, job|
|
175
|
-
|
273
|
+
load_id(File.join(task.to_s, job))
|
176
274
|
end
|
177
275
|
end
|
178
276
|
step
|
@@ -58,7 +58,8 @@ class Step
|
|
58
58
|
end
|
59
59
|
|
60
60
|
def done?
|
61
|
-
|
61
|
+
status = info[:status]
|
62
|
+
status == :done or status == :error
|
62
63
|
end
|
63
64
|
|
64
65
|
def error?
|
@@ -72,7 +73,10 @@ class Step
|
|
72
73
|
end
|
73
74
|
|
74
75
|
def files
|
75
|
-
Dir.glob(File.join(files_dir, '*')).collect do |path|
|
76
|
+
files = Dir.glob(File.join(files_dir, '**', '*')).reject{|path| File.directory? path}.collect do |path|
|
77
|
+
Misc.path_relative_to(files_dir, path)
|
78
|
+
end
|
79
|
+
files
|
76
80
|
end
|
77
81
|
|
78
82
|
def file(name)
|
@@ -139,6 +143,7 @@ module Workflow
|
|
139
143
|
def task_info(name)
|
140
144
|
task = tasks[name]
|
141
145
|
description = task.description
|
146
|
+
result_description = task.result_description
|
142
147
|
result_type = task.result_type
|
143
148
|
inputs = rec_inputs(name)
|
144
149
|
input_types = rec_input_types(name)
|
@@ -165,6 +170,7 @@ module Workflow
|
|
165
170
|
:input_descriptions => input_descriptions,
|
166
171
|
:input_defaults => input_defaults,
|
167
172
|
:result_type => result_type,
|
173
|
+
:result_description => result_description,
|
168
174
|
:dependencies => dependencies
|
169
175
|
}
|
170
176
|
end
|
@@ -2,7 +2,7 @@ module AnnotatedModule
|
|
2
2
|
def self.extended(base)
|
3
3
|
if not base.respond_to? :inputs
|
4
4
|
class << base
|
5
|
-
attr_accessor :description, :inputs, :input_types, :input_descriptions, :input_defaults
|
5
|
+
attr_accessor :description, :inputs, :input_types, :input_descriptions, :input_defaults, :result_description
|
6
6
|
|
7
7
|
def description
|
8
8
|
i = @description; @description = ""; i
|
@@ -27,6 +27,10 @@ module AnnotatedModule
|
|
27
27
|
def description
|
28
28
|
i = @description; @description = ""; i
|
29
29
|
end
|
30
|
+
|
31
|
+
def result_description
|
32
|
+
i = @result_description; @result_description = nil; i
|
33
|
+
end
|
30
34
|
end
|
31
35
|
|
32
36
|
base.description = ""
|
@@ -38,6 +42,10 @@ module AnnotatedModule
|
|
38
42
|
end
|
39
43
|
end
|
40
44
|
|
45
|
+
def returns(text)
|
46
|
+
@result_description = text
|
47
|
+
end
|
48
|
+
|
41
49
|
def desc(description)
|
42
50
|
@description = description
|
43
51
|
end
|
data/lib/rbbt/workflow/step.rb
CHANGED
@@ -86,11 +86,13 @@ class Step
|
|
86
86
|
def clean
|
87
87
|
if File.exists?(path) or File.exists?(info_file)
|
88
88
|
begin
|
89
|
-
FileUtils.rm info_file
|
90
|
-
FileUtils.rm path
|
91
|
-
FileUtils.
|
89
|
+
FileUtils.rm info_file if File.exists? info_file
|
90
|
+
FileUtils.rm path if File.exists? path
|
91
|
+
FileUtils.rm path + '.lock' if File.exists? path + '.lock'
|
92
|
+
FileUtils.rm_rf files_dir if File.exists? files_dir
|
92
93
|
end
|
93
94
|
end
|
95
|
+
self
|
94
96
|
end
|
95
97
|
|
96
98
|
def rec_dependencies
|
data/lib/rbbt/workflow/task.rb
CHANGED
@@ -2,7 +2,7 @@ require 'rbbt/util/misc'
|
|
2
2
|
require 'rbbt/persist'
|
3
3
|
|
4
4
|
module Task
|
5
|
-
attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :description, :name
|
5
|
+
attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :description, :name, :result_description
|
6
6
|
|
7
7
|
def self.setup(options = {}, &block)
|
8
8
|
block.extend Task
|
data/test/rbbt/test_tsv.rb
CHANGED
@@ -329,4 +329,35 @@ b 2
|
|
329
329
|
|
330
330
|
end
|
331
331
|
|
332
|
+
def test_flat_no_merge
|
333
|
+
content =<<-EOF
|
334
|
+
#Id ValueA ValueB OtherID
|
335
|
+
row1 a|aa|aaa b Id1|Id2
|
336
|
+
row2 A B Id3
|
337
|
+
EOF
|
338
|
+
|
339
|
+
TmpFile.with_file(content) do |filename|
|
340
|
+
tsv = TSV.open(filename, :sep => /\s+/, :type => :flat, :fields => ["ValueA"])
|
341
|
+
assert_equal ["a", "aa", "aaa"], tsv["row1"]
|
342
|
+
assert_equal ["ValueA"], tsv.fields
|
343
|
+
assert_equal :flat, tsv.type
|
344
|
+
assert_equal "Id", tsv.key_field
|
345
|
+
end
|
346
|
+
end
|
347
|
+
|
348
|
+
def test_flat_merge
|
349
|
+
content =<<-EOF
|
350
|
+
#Id ValueA ValueB OtherID
|
351
|
+
row1 a|aa|aaa b Id1|Id2
|
352
|
+
row1 aaaa b Id1|Id2
|
353
|
+
row2 A B Id3
|
354
|
+
EOF
|
355
|
+
|
356
|
+
TmpFile.with_file(content) do |filename|
|
357
|
+
tsv = TSV.open(filename, :sep => /\s+/, :merge => true, :type => :flat, :fields => ["ValueA"])
|
358
|
+
assert_equal ["a", "aa", "aaa", "aaaa"], tsv["row1"]
|
359
|
+
end
|
360
|
+
end
|
361
|
+
|
362
|
+
|
332
363
|
end
|
@@ -5,7 +5,7 @@ require 'test/unit'
|
|
5
5
|
|
6
6
|
class TestTSV < Test::Unit::TestCase
|
7
7
|
|
8
|
-
def
|
8
|
+
def test_tsv
|
9
9
|
content =<<-EOF
|
10
10
|
#Id ValueA ValueB OtherID
|
11
11
|
row1 a|aa|aaa b Id1|Id2
|
@@ -20,7 +20,7 @@ row2 A B Id3
|
|
20
20
|
end
|
21
21
|
end
|
22
22
|
|
23
|
-
def
|
23
|
+
def test_named_values
|
24
24
|
content =<<-EOF
|
25
25
|
#Id ValueA ValueB OtherID
|
26
26
|
row1 a|aa|aaa b Id1|Id2
|
@@ -33,7 +33,7 @@ row2 A B Id3
|
|
33
33
|
end
|
34
34
|
end
|
35
35
|
|
36
|
-
def
|
36
|
+
def test_to_s
|
37
37
|
content =<<-EOF
|
38
38
|
#Id ValueA ValueB OtherID
|
39
39
|
row1 a|aa|aaa b Id1|Id2
|
@@ -47,7 +47,7 @@ row2 A B Id3
|
|
47
47
|
end
|
48
48
|
end
|
49
49
|
|
50
|
-
def
|
50
|
+
def test_entries
|
51
51
|
content =<<-EOF
|
52
52
|
#Id ValueA ValueB OtherID
|
53
53
|
row1 a|aa|aaa b Id1|Id2
|
@@ -61,7 +61,7 @@ row2 A B Id3
|
|
61
61
|
|
62
62
|
end
|
63
63
|
|
64
|
-
def
|
64
|
+
def test_marshal
|
65
65
|
content =<<-EOF
|
66
66
|
#Id ValueA ValueB OtherID
|
67
67
|
row1 a|aa|aaa b Id1|Id2
|
@@ -74,12 +74,12 @@ row2 A B Id3
|
|
74
74
|
end
|
75
75
|
end
|
76
76
|
|
77
|
-
def
|
77
|
+
def test_zip_fields
|
78
78
|
a = [%w(1 2), %w(a b)]
|
79
79
|
assert_equal a, TSV.zip_fields(TSV.zip_fields(a))
|
80
80
|
end
|
81
81
|
|
82
|
-
def
|
82
|
+
def test_indentify_fields
|
83
83
|
content =<<-EOF
|
84
84
|
#ID ValueA ValueB Comment
|
85
85
|
row1 a b c
|
@@ -92,7 +92,7 @@ row2 A B C
|
|
92
92
|
end
|
93
93
|
end
|
94
94
|
|
95
|
-
def
|
95
|
+
def test_sort
|
96
96
|
content =<<-EOF
|
97
97
|
#ID ValueA ValueB Comment
|
98
98
|
row1 a B c
|
@@ -110,7 +110,7 @@ row2 A b C
|
|
110
110
|
end
|
111
111
|
end
|
112
112
|
|
113
|
-
def
|
113
|
+
def test_sort_by
|
114
114
|
content =<<-EOF
|
115
115
|
#ID ValueA ValueB Comment
|
116
116
|
row1 a B c
|
@@ -146,7 +146,7 @@ row6 A b a
|
|
146
146
|
end
|
147
147
|
|
148
148
|
|
149
|
-
def
|
149
|
+
def test_sort_by_with_proc
|
150
150
|
content =<<-EOF
|
151
151
|
#Id ValueA ValueB OtherID Pos
|
152
152
|
row1 a|aa|aaa b Id1|Id2 2
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
5
|
-
prerelease:
|
4
|
+
hash: 61
|
5
|
+
prerelease:
|
6
6
|
segments:
|
7
7
|
- 4
|
8
8
|
- 0
|
9
|
-
-
|
10
|
-
version: 4.0.
|
9
|
+
- 1
|
10
|
+
version: 4.0.1
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Miguel Vazquez
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2011-
|
18
|
+
date: 2011-10-03 00:00:00 +02:00
|
19
19
|
default_executable:
|
20
20
|
dependencies:
|
21
21
|
- !ruby/object:Gem::Dependency
|
@@ -117,7 +117,7 @@ dependencies:
|
|
117
117
|
type: :runtime
|
118
118
|
version_requirements: *id007
|
119
119
|
description: Utilities for handling tsv files, caches, etc
|
120
|
-
email: miguel.vazquez@
|
120
|
+
email: miguel.vazquez@cnio.es
|
121
121
|
executables:
|
122
122
|
- tsv.rb
|
123
123
|
- tchash.rb
|
@@ -237,7 +237,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
237
237
|
requirements: []
|
238
238
|
|
239
239
|
rubyforge_project:
|
240
|
-
rubygems_version: 1.
|
240
|
+
rubygems_version: 1.6.2
|
241
241
|
signing_key:
|
242
242
|
specification_version: 3
|
243
243
|
summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
|