rbbt-util 4.0.0 → 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/run_workflow.rb +1 -1
- data/lib/rbbt/annotations.rb +17 -6
- data/lib/rbbt/persist.rb +7 -1
- data/lib/rbbt/tsv.rb +14 -1
- data/lib/rbbt/tsv/accessor.rb +13 -2
- data/lib/rbbt/tsv/attach.rb +2 -0
- data/lib/rbbt/tsv/manipulate.rb +54 -26
- data/lib/rbbt/tsv/parser.rb +37 -9
- data/lib/rbbt/util/chain_methods.rb +11 -1
- data/lib/rbbt/util/cmd.rb +1 -1
- data/lib/rbbt/util/misc.rb +44 -2
- data/lib/rbbt/workflow.rb +104 -6
- data/lib/rbbt/workflow/accessor.rb +8 -2
- data/lib/rbbt/workflow/annotate.rb +9 -1
- data/lib/rbbt/workflow/step.rb +5 -3
- data/lib/rbbt/workflow/task.rb +1 -1
- data/test/rbbt/test_fix_width_table.rb +1 -0
- data/test/rbbt/test_tsv.rb +31 -0
- data/test/rbbt/tsv/test_accessor.rb +10 -10
- metadata +7 -7
data/bin/run_workflow.rb
CHANGED
@@ -115,7 +115,7 @@ else
|
|
115
115
|
|
116
116
|
if remote_workflows.include? workflow
|
117
117
|
require 'rbbt/workflow/rest/client'
|
118
|
-
workflow = RbbtRestClient.new remote_workflows[workflow]
|
118
|
+
workflow = RbbtRestClient.new remote_workflows[workflow], workflow
|
119
119
|
else
|
120
120
|
Workflow.require_workflow workflow
|
121
121
|
workflow = Workflow.workflows.last
|
data/lib/rbbt/annotations.rb
CHANGED
@@ -128,7 +128,12 @@ module Annotation
|
|
128
128
|
base.all_inheritance = []
|
129
129
|
|
130
130
|
base.module_eval do
|
131
|
+
class << self
|
132
|
+
alias prev_annotation_extended extended
|
133
|
+
end
|
134
|
+
|
131
135
|
def self.extended(object)
|
136
|
+
self.send(:prev_annotation_extended, object)
|
132
137
|
object.extend Annotated
|
133
138
|
if not object.annotation_types.include? self
|
134
139
|
object.annotation_types.concat self.inheritance
|
@@ -178,6 +183,7 @@ module Annotation
|
|
178
183
|
|
179
184
|
object
|
180
185
|
end
|
186
|
+
|
181
187
|
end
|
182
188
|
|
183
189
|
module AnnotatedArray
|
@@ -193,23 +199,28 @@ module AnnotatedArray
|
|
193
199
|
end
|
194
200
|
|
195
201
|
def annotated_array_each
|
196
|
-
|
202
|
+
annotated_array_clean_each do |value|
|
197
203
|
annotation_types.each do |mod|
|
198
|
-
mod.setup(value,
|
204
|
+
mod.setup(value, *info.values_at(*mod.annotations))
|
199
205
|
end
|
200
206
|
yield value
|
201
207
|
end
|
202
208
|
end
|
203
209
|
|
204
210
|
def annotated_array_collect
|
205
|
-
|
211
|
+
annotated_array_clean_collect do |value|
|
206
212
|
annotation_types.each do |mod|
|
207
|
-
mod.setup(value,
|
213
|
+
mod.setup(value, *info.values_at(*mod.annotations))
|
208
214
|
end
|
209
215
|
yield value
|
210
216
|
end
|
211
217
|
end
|
212
218
|
|
219
|
+
def annotated_array_subset(list)
|
220
|
+
value = (self & list)
|
221
|
+
annotation_types.each do |mod|
|
222
|
+
mod.setup(value, *info.values_at(*mod.annotations))
|
223
|
+
end
|
224
|
+
value
|
225
|
+
end
|
213
226
|
end
|
214
|
-
|
215
|
-
|
data/lib/rbbt/persist.rb
CHANGED
@@ -4,6 +4,7 @@ require 'rbbt/util/misc'
|
|
4
4
|
require 'rbbt/util/open'
|
5
5
|
|
6
6
|
require 'rbbt/persist/tsv'
|
7
|
+
require 'set'
|
7
8
|
|
8
9
|
module Persist
|
9
10
|
CACHEDIR="/tmp/tsv_persistent_cache"
|
@@ -67,10 +68,13 @@ module Persist
|
|
67
68
|
File.join(persistence_dir, filename)
|
68
69
|
end
|
69
70
|
|
71
|
+
TRUE_STRINGS = Set.new ["true", "True", "TRUE", "t", "T", "1", "yes", "Yes", "YES", "y", "Y", "ON", "on"]
|
70
72
|
def self.load_file(path, type)
|
71
73
|
case (type || "nil").to_sym
|
72
74
|
when :nil
|
73
75
|
nil
|
76
|
+
when :boolean
|
77
|
+
TRUE_STRINGS.include? Open.read(path).chomp.strip
|
74
78
|
when :tsv
|
75
79
|
TSV.open(path)
|
76
80
|
when :marshal_tsv
|
@@ -105,6 +109,8 @@ module Persist
|
|
105
109
|
case (type || "nil").to_sym
|
106
110
|
when :nil
|
107
111
|
nil
|
112
|
+
when :boolean
|
113
|
+
Open.write(path, content ? "true" : "false")
|
108
114
|
when :fwt
|
109
115
|
content.file.seek 0
|
110
116
|
Open.write(path, content.file.read)
|
@@ -113,7 +119,7 @@ module Persist
|
|
113
119
|
when :string, :text
|
114
120
|
Open.write(path, content)
|
115
121
|
when :array
|
116
|
-
Open.write(path, content * "\n"
|
122
|
+
Open.write(path, content * "\n")
|
117
123
|
when :marshal_tsv
|
118
124
|
Open.write(path, Marshal.dump(content.dup))
|
119
125
|
when :marshal
|
data/lib/rbbt/tsv.rb
CHANGED
@@ -86,7 +86,18 @@ module TSV
|
|
86
86
|
data.extend TSV unless TSV === data
|
87
87
|
data.unnamed = true
|
88
88
|
|
89
|
-
|
89
|
+
if data.serializer == :type
|
90
|
+
data.serializer = case
|
91
|
+
when parser.cast.nil?
|
92
|
+
data.serializer = parser.type
|
93
|
+
when (parser.cast == :to_i and parser.type == :list)
|
94
|
+
data.serializer = :integer_array
|
95
|
+
when (parser.cast == :to_i and parser.type == :single)
|
96
|
+
data.serializer = :integer
|
97
|
+
when (parser.cast == :to_f and parser.type == :single)
|
98
|
+
data.serializer = :float
|
99
|
+
end
|
100
|
+
end
|
90
101
|
|
91
102
|
if monitor and (stream.respond_to?(:size) or (stream.respond_to?(:stat) and stream.stat.respond_to? :size)) and stream.respond_to?(:pos)
|
92
103
|
size = case
|
@@ -120,6 +131,8 @@ module TSV
|
|
120
131
|
rescue Parser::SKIP_LINE
|
121
132
|
line = stream.gets
|
122
133
|
next
|
134
|
+
rescue Parser::END_PARSING
|
135
|
+
break
|
123
136
|
rescue IOError
|
124
137
|
break
|
125
138
|
end
|
data/lib/rbbt/tsv/accessor.rb
CHANGED
@@ -145,8 +145,19 @@ module TSV
|
|
145
145
|
elems = collect
|
146
146
|
else
|
147
147
|
elems = []
|
148
|
-
|
149
|
-
|
148
|
+
case type
|
149
|
+
when :single
|
150
|
+
through :key, field do |key, field|
|
151
|
+
elems << [key, field]
|
152
|
+
end
|
153
|
+
when :list, :flat
|
154
|
+
through :key, field do |key, fields|
|
155
|
+
elems << [key, fields.first]
|
156
|
+
end
|
157
|
+
when :double
|
158
|
+
through :key, field do |key, fields|
|
159
|
+
elems << [key, fields.first]
|
160
|
+
end
|
150
161
|
end
|
151
162
|
end
|
152
163
|
|
data/lib/rbbt/tsv/attach.rb
CHANGED
data/lib/rbbt/tsv/manipulate.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'progress-bar'
|
2
2
|
require 'rbbt/persist'
|
3
3
|
require 'rbbt/tsv/util'
|
4
|
+
require 'set'
|
4
5
|
|
5
6
|
module TSV
|
6
7
|
|
@@ -57,27 +58,27 @@ module TSV
|
|
57
58
|
|
58
59
|
raise "Key field #{ new_key_field } not found" if @new_key_field.nil?
|
59
60
|
@new_fields = case
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
61
|
+
when new_fields.nil?
|
62
|
+
case
|
63
|
+
when @new_key_field == :key
|
64
|
+
:all
|
65
|
+
when fields.nil?
|
66
|
+
- @new_key_field
|
67
|
+
else
|
68
|
+
new = (0..fields.length - 1).to_a
|
69
|
+
new.delete_at(@new_key_field)
|
70
|
+
new.unshift :key
|
71
|
+
new
|
72
|
+
end
|
73
|
+
when Array === new_fields
|
74
|
+
new_fields.collect do |field|
|
75
|
+
TSV.identify_field(key_field, fields, field)
|
76
|
+
end
|
77
|
+
when (String === new_fields or Symbol === new_fields)
|
78
|
+
[TSV.identify_field(key_field, fields, new_fields)]
|
79
|
+
else
|
80
|
+
raise "Unknown format for new_fields (should be nil, Array or String): #{new_fields.inspect}"
|
81
|
+
end
|
81
82
|
|
82
83
|
@new_key_field_name = case
|
83
84
|
when @new_key_field == :key
|
@@ -235,9 +236,21 @@ module TSV
|
|
235
236
|
new[key] = values if yield key, values
|
236
237
|
end
|
237
238
|
when Array === method
|
239
|
+
method = Set.new method
|
238
240
|
with_unnamed do
|
239
|
-
|
240
|
-
|
241
|
+
case type
|
242
|
+
when :single
|
243
|
+
through do |key, value|
|
244
|
+
new[key] = values if method.include? key or method.include? value
|
245
|
+
end
|
246
|
+
when :list, :flat
|
247
|
+
through do |key, values|
|
248
|
+
new[key] = values if method.include? key or (method & values).any?
|
249
|
+
end
|
250
|
+
else
|
251
|
+
through do |key, values|
|
252
|
+
new[key] = values if method.include? key or (method & values.flatten).any?
|
253
|
+
end
|
241
254
|
end
|
242
255
|
end
|
243
256
|
when Regexp === method
|
@@ -273,9 +286,24 @@ module TSV
|
|
273
286
|
end
|
274
287
|
when Array === method
|
275
288
|
with_unnamed do
|
276
|
-
|
277
|
-
|
278
|
-
|
289
|
+
method = Set.new method unless Set === method
|
290
|
+
case type
|
291
|
+
when :single
|
292
|
+
through :key, key do |key, value|
|
293
|
+
new[key] = self[key] if method.include? value
|
294
|
+
end
|
295
|
+
when :list
|
296
|
+
through :key, key do |key, values|
|
297
|
+
new[key] = self[key] if method.include? value.first
|
298
|
+
end
|
299
|
+
when :flat #untested
|
300
|
+
through :key, key do |key, values|
|
301
|
+
new[key] = self[key] if (method & values.flatten).any?
|
302
|
+
end
|
303
|
+
else
|
304
|
+
through :key, key do |key, values|
|
305
|
+
new[key] = self[key] if (method & values.first).any?
|
306
|
+
end
|
279
307
|
end
|
280
308
|
end
|
281
309
|
when Regexp === method
|
data/lib/rbbt/tsv/parser.rb
CHANGED
@@ -4,6 +4,7 @@ module TSV
|
|
4
4
|
attr_accessor :header_hash, :sep, :sep2, :type, :key_position, :field_positions, :cast, :key_field, :fields, :fix, :select, :serializer, :straight
|
5
5
|
|
6
6
|
class SKIP_LINE < Exception; end
|
7
|
+
class END_PARSING < Exception; end
|
7
8
|
|
8
9
|
def all_fields
|
9
10
|
all = [key_field] + fields
|
@@ -49,6 +50,7 @@ module TSV
|
|
49
50
|
l = line.chomp
|
50
51
|
raise Parser::SKIP_LINE if Proc === @select and not @select.call l
|
51
52
|
l = @fix.call l if Proc === @fix
|
53
|
+
raise Parser::END_PARSING unless l
|
52
54
|
l
|
53
55
|
end
|
54
56
|
|
@@ -85,6 +87,22 @@ module TSV
|
|
85
87
|
data[key] = values unless data.include? key
|
86
88
|
end
|
87
89
|
|
90
|
+
def add_to_data_flat(data, keys, values)
|
91
|
+
keys.each do |key|
|
92
|
+
data[key] = values.flatten unless data.include? key
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
def add_to_data_flat_merge(data, keys, values)
|
97
|
+
keys.each do |key|
|
98
|
+
if data.include? key
|
99
|
+
data[key] = data[key].concat values.flatten
|
100
|
+
else
|
101
|
+
data[key] = values.flatten
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
88
106
|
def add_to_data_no_merge_double(data, keys, values)
|
89
107
|
keys.each do |key|
|
90
108
|
data[key] = values unless data.include? key
|
@@ -168,7 +186,9 @@ module TSV
|
|
168
186
|
when Integer === field
|
169
187
|
field
|
170
188
|
when String === field
|
171
|
-
@fields.dup.unshift(@key_field).index field
|
189
|
+
pos = @fields.dup.unshift(@key_field).index field
|
190
|
+
raise "Field not identified: #{ field }" if pos.nil?
|
191
|
+
pos
|
172
192
|
else
|
173
193
|
raise "Format of fields not understood: #{fields.inspect}"
|
174
194
|
end
|
@@ -196,7 +216,8 @@ module TSV
|
|
196
216
|
@fix = Misc.process_options(options, :fix)
|
197
217
|
@select= Misc.process_options options, :select
|
198
218
|
|
199
|
-
|
219
|
+
case @type
|
220
|
+
when :double
|
200
221
|
self.instance_eval do alias get_values get_values_double end
|
201
222
|
self.instance_eval do alias cast_values cast_values_double end
|
202
223
|
if merge
|
@@ -204,15 +225,22 @@ module TSV
|
|
204
225
|
else
|
205
226
|
self.instance_eval do alias add_to_data add_to_data_no_merge_double end
|
206
227
|
end
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
228
|
+
when :single
|
229
|
+
self.instance_eval do alias get_values get_values_single end
|
230
|
+
self.instance_eval do alias cast_values cast_values_single end
|
231
|
+
self.instance_eval do alias add_to_data add_to_data_no_merge_list end
|
232
|
+
when :list
|
233
|
+
self.instance_eval do alias get_values get_values_list end
|
234
|
+
self.instance_eval do alias cast_values cast_values_list end
|
235
|
+
self.instance_eval do alias add_to_data add_to_data_no_merge_list end
|
236
|
+
when :flat
|
237
|
+
self.instance_eval do alias get_values get_values_double end
|
238
|
+
self.instance_eval do alias cast_values cast_values_double end
|
239
|
+
if merge
|
240
|
+
self.instance_eval do alias add_to_data add_to_data_flat_merge end
|
211
241
|
else
|
212
|
-
self.instance_eval do alias
|
213
|
-
self.instance_eval do alias cast_values cast_values_list end
|
242
|
+
self.instance_eval do alias add_to_data add_to_data_flat end
|
214
243
|
end
|
215
|
-
self.instance_eval do alias add_to_data add_to_data_no_merge_list end
|
216
244
|
end
|
217
245
|
|
218
246
|
fix_fields(options)
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'rbbt/util/log'
|
2
2
|
|
3
3
|
module ChainMethods
|
4
|
-
def self.
|
4
|
+
def self.chain_methods_extended(base)
|
5
5
|
if not base.respond_to? :chain_prefix
|
6
6
|
metaclass = class << base
|
7
7
|
attr_accessor :chain_prefix, :chained_methods
|
@@ -52,7 +52,10 @@ module ChainMethods
|
|
52
52
|
|
53
53
|
if not metaclass.respond_to? :extended
|
54
54
|
metaclass.module_eval do
|
55
|
+
alias prev_chain_methods_extended extended
|
56
|
+
|
55
57
|
def extended(base)
|
58
|
+
prev_chain_methods_extended(base)
|
56
59
|
setup_chains(base)
|
57
60
|
end
|
58
61
|
end
|
@@ -61,4 +64,11 @@ module ChainMethods
|
|
61
64
|
|
62
65
|
base.chain_prefix = base.to_s.downcase.to_sym
|
63
66
|
end
|
67
|
+
|
68
|
+
def self.extended(base)
|
69
|
+
chain_methods_extended(base)
|
70
|
+
end
|
71
|
+
|
72
|
+
|
73
|
+
|
64
74
|
end
|
data/lib/rbbt/util/cmd.rb
CHANGED
data/lib/rbbt/util/misc.rb
CHANGED
@@ -1,10 +1,18 @@
|
|
1
1
|
require 'lockfile'
|
2
2
|
require 'rbbt/util/chain_methods'
|
3
3
|
require 'rbbt/resource/path'
|
4
|
+
require 'rbbt/annotations'
|
4
5
|
|
5
6
|
module Misc
|
6
7
|
class FieldNotFoundError < StandardError;end
|
7
8
|
|
9
|
+
def self.counts(array)
|
10
|
+
counts = Hash.new 0
|
11
|
+
array.each do |e|
|
12
|
+
counts[e] += 1
|
13
|
+
end
|
14
|
+
counts
|
15
|
+
end
|
8
16
|
|
9
17
|
IUPAC2BASE = {
|
10
18
|
"A" => ["A"],
|
@@ -261,7 +269,7 @@ module Misc
|
|
261
269
|
when String === options
|
262
270
|
new_options = string2hash options
|
263
271
|
else
|
264
|
-
raise "Format of '#{options.inspect}' not understood"
|
272
|
+
raise "Format of '#{options.inspect}' not understood. It should be a hash"
|
265
273
|
end
|
266
274
|
defaults.each do |key, value|
|
267
275
|
next unless new_options[key].nil?
|
@@ -444,7 +452,41 @@ module NamedArray
|
|
444
452
|
end
|
445
453
|
|
446
454
|
def named_array_get_brackets(key)
|
447
|
-
|
455
|
+
if defined? Entity
|
456
|
+
entity = (defined?(Entity) and Entity.respond_to?(:formats)) ? Entity.formats[key] : nil
|
457
|
+
if entity
|
458
|
+
if entity.annotations.first == :format
|
459
|
+
entity.setup(named_array_clean_get_brackets(Misc.field_position(fields, key)), key)
|
460
|
+
else
|
461
|
+
entity.setup(named_array_clean_get_brackets(Misc.field_position(fields, key)))
|
462
|
+
end
|
463
|
+
else
|
464
|
+
named_array_clean_get_brackets(Misc.field_position(fields, key))
|
465
|
+
end
|
466
|
+
else
|
467
|
+
named_array_clean_get_brackets(Misc.field_position(fields, key))
|
468
|
+
end
|
469
|
+
end
|
470
|
+
|
471
|
+
def named_array_each(&block)
|
472
|
+
if defined?(Entity) and not fields.nil? and not fields.empty?
|
473
|
+
fields.zip(self) do |field,elem|
|
474
|
+
entity = (defined?(Entity) and Entity.respond_to?(:formats)) ? Entity.formats[field] : nil
|
475
|
+
if entity
|
476
|
+
elem = elem.dup if elem.frozen?
|
477
|
+
if entity.annotations.first == :format
|
478
|
+
elem = entity.setup(elem, field)
|
479
|
+
else
|
480
|
+
elem = entity.setup(elem)
|
481
|
+
end
|
482
|
+
else
|
483
|
+
end
|
484
|
+
yield(elem)
|
485
|
+
elem
|
486
|
+
end
|
487
|
+
else
|
488
|
+
named_array_clean_each &block
|
489
|
+
end
|
448
490
|
end
|
449
491
|
|
450
492
|
def named_array_set_brackets(key,value)
|
data/lib/rbbt/workflow.rb
CHANGED
@@ -9,7 +9,7 @@ module Workflow
|
|
9
9
|
end
|
10
10
|
self.workflows = []
|
11
11
|
|
12
|
-
def self.
|
12
|
+
def self.require_local_workflow2(wf_name, wf_dir = nil)
|
13
13
|
require 'rbbt/resource/path'
|
14
14
|
|
15
15
|
if File.exists?(wf_name) or File.exists?(wf_name + '.rb')
|
@@ -46,6 +46,103 @@ module Workflow
|
|
46
46
|
require wf_dir["workflow.rb"].find
|
47
47
|
end
|
48
48
|
|
49
|
+
def self.require_remote_workflow(wf_name, url)
|
50
|
+
require 'rbbt/workflow/rest/client'
|
51
|
+
eval "Object::#{wf_name} = RbbtRestClient.new '#{ url }', '#{wf_name}'"
|
52
|
+
end
|
53
|
+
|
54
|
+
def self.require_local_workflow(wf_name)
|
55
|
+
if Path === wf_name
|
56
|
+
case
|
57
|
+
|
58
|
+
# Points to workflow file
|
59
|
+
when ((File.exists?(wf_name.find) and not File.directory?(wf_name.find)) or File.exists?(wf_name.find + '.rb'))
|
60
|
+
$LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(wf_name.find)), 'lib'))
|
61
|
+
require wf_name.find
|
62
|
+
Log.debug "Workflow loaded from file: #{ wf_name }"
|
63
|
+
return true
|
64
|
+
|
65
|
+
# Points to workflow dir
|
66
|
+
when (File.exists?(wf_name.find) and File.directory?(wf_name.find) and File.exists?(File.join(wf_name.find, 'workflow.rb')))
|
67
|
+
$LOAD_PATH.unshift(File.join(File.expand_path(wf_name.find), 'lib'))
|
68
|
+
require File.join(wf_name.find, 'workflow.rb')
|
69
|
+
Log.debug "Workflow loaded from directory: #{ wf_name }"
|
70
|
+
return true
|
71
|
+
|
72
|
+
else
|
73
|
+
raise "Workflow path was not resolved: #{ wf_name } (#{wf_name.find})"
|
74
|
+
end
|
75
|
+
|
76
|
+
else
|
77
|
+
case
|
78
|
+
|
79
|
+
# Points to workflow file
|
80
|
+
when ((File.exists?(wf_name) and not File.directory?(wf_name)) or File.exists?(wf_name + '.rb'))
|
81
|
+
$LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(wf_name)), 'lib'))
|
82
|
+
require wf_name
|
83
|
+
Log.debug "Workflow loaded from file: #{ wf_name }"
|
84
|
+
return true
|
85
|
+
|
86
|
+
when (defined?(Rbbt) and Rbbt.etc.workflow_dir.exists?)
|
87
|
+
dir = Rbbt.etc.workflow_dir.read.strip
|
88
|
+
dir = File.join(dir, wf_name)
|
89
|
+
$LOAD_PATH.unshift(File.join(File.expand_path(dir), 'lib'))
|
90
|
+
require File.join(dir, 'workflow.rb')
|
91
|
+
Log.debug "Workflow #{wf_name} loaded from workflow_dir: #{ dir }"
|
92
|
+
return true
|
93
|
+
|
94
|
+
when defined?(Rbbt)
|
95
|
+
path = Rbbt.workflows[wf_name].find
|
96
|
+
$LOAD_PATH.unshift(File.join(File.expand_path(path), 'lib'))
|
97
|
+
require File.join(path, 'workflow.rb')
|
98
|
+
Log.debug "Workflow #{wf_name} loaded from Rbbt.workflows: #{ path }"
|
99
|
+
return true
|
100
|
+
|
101
|
+
else
|
102
|
+
path = File.join(ENV['HOME'], '.workflows', wf_name)
|
103
|
+
$LOAD_PATH.unshift(File.join(File.expand_path(path), 'lib'))
|
104
|
+
require File.join(path, 'workflow.rb')
|
105
|
+
Log.debug "Workflow #{wf_name} loaded from .workflows: #{ path }"
|
106
|
+
return true
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
raise "Workflow not found: #{ wf_name }"
|
111
|
+
end
|
112
|
+
|
113
|
+
def self.require_workflow(wf_name)
|
114
|
+
begin
|
115
|
+
Misc.string2const wf_name
|
116
|
+
Log.debug "Workflow #{ wf_name } already loaded"
|
117
|
+
return true
|
118
|
+
rescue Exception
|
119
|
+
end
|
120
|
+
|
121
|
+
if Rbbt.etc.remote_workflows.exists?
|
122
|
+
remote_workflows = Rbbt.etc.remote_workflows.yaml
|
123
|
+
if remote_workflows.include? wf_name
|
124
|
+
url = remote_workflows[wf_name]
|
125
|
+
require_remote_workflow(wf_name, url)
|
126
|
+
Log.debug "Workflow #{ wf_name } loaded remotely: #{ url }"
|
127
|
+
return
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
begin
|
132
|
+
require_local_workflow(wf_name)
|
133
|
+
rescue Exception
|
134
|
+
Log.debug $!.message
|
135
|
+
raise "Workflow not found: #{ wf_name }" if wf_name == wf_name.downcase
|
136
|
+
Log.debug "Trying with downcase: '#{wf_name.downcase}'"
|
137
|
+
begin
|
138
|
+
require_local_workflow(wf_name.downcase)
|
139
|
+
rescue Exception
|
140
|
+
Log.debug $!.message
|
141
|
+
raise "Workflow not found: #{ wf_name }"
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
49
146
|
def self.extended(base)
|
50
147
|
if not base.respond_to? :workdir
|
51
148
|
base.extend AnnotatedModule
|
@@ -59,7 +156,7 @@ module Workflow
|
|
59
156
|
def task_dependencies
|
60
157
|
IndiferentHash.setup(@task_dependencies || {})
|
61
158
|
end
|
62
|
-
|
159
|
+
|
63
160
|
def tasks
|
64
161
|
IndiferentHash.setup(@tasks || {})
|
65
162
|
end
|
@@ -83,7 +180,7 @@ module Workflow
|
|
83
180
|
end
|
84
181
|
|
85
182
|
# {{{ Task definition helpers
|
86
|
-
|
183
|
+
|
87
184
|
def task(name, &block)
|
88
185
|
if Hash === name
|
89
186
|
result_type = name.first.last
|
@@ -104,7 +201,8 @@ module Workflow
|
|
104
201
|
:input_types => input_types,
|
105
202
|
:result_type => Array == result_type ? result_type.to_sym : result_type,
|
106
203
|
:input_defaults => input_defaults,
|
107
|
-
:input_descriptions => input_descriptions
|
204
|
+
:input_descriptions => input_descriptions,
|
205
|
+
:result_description => result_description
|
108
206
|
}, &block)
|
109
207
|
|
110
208
|
@last_task = task
|
@@ -125,7 +223,7 @@ module Workflow
|
|
125
223
|
end
|
126
224
|
|
127
225
|
# {{{ Job management
|
128
|
-
|
226
|
+
|
129
227
|
def resolve_locals(inputs)
|
130
228
|
inputs.each do |name, value|
|
131
229
|
if value =~ /^local:(.*?):(.*)/ or
|
@@ -172,7 +270,7 @@ module Workflow
|
|
172
270
|
step = Step.new path, tasks[task]
|
173
271
|
if step.info.include? :dependencies
|
174
272
|
step.dependencies = step.info[:dependencies].collect do |task, job|
|
175
|
-
|
273
|
+
load_id(File.join(task.to_s, job))
|
176
274
|
end
|
177
275
|
end
|
178
276
|
step
|
@@ -58,7 +58,8 @@ class Step
|
|
58
58
|
end
|
59
59
|
|
60
60
|
def done?
|
61
|
-
|
61
|
+
status = info[:status]
|
62
|
+
status == :done or status == :error
|
62
63
|
end
|
63
64
|
|
64
65
|
def error?
|
@@ -72,7 +73,10 @@ class Step
|
|
72
73
|
end
|
73
74
|
|
74
75
|
def files
|
75
|
-
Dir.glob(File.join(files_dir, '*')).collect do |path|
|
76
|
+
files = Dir.glob(File.join(files_dir, '**', '*')).reject{|path| File.directory? path}.collect do |path|
|
77
|
+
Misc.path_relative_to(files_dir, path)
|
78
|
+
end
|
79
|
+
files
|
76
80
|
end
|
77
81
|
|
78
82
|
def file(name)
|
@@ -139,6 +143,7 @@ module Workflow
|
|
139
143
|
def task_info(name)
|
140
144
|
task = tasks[name]
|
141
145
|
description = task.description
|
146
|
+
result_description = task.result_description
|
142
147
|
result_type = task.result_type
|
143
148
|
inputs = rec_inputs(name)
|
144
149
|
input_types = rec_input_types(name)
|
@@ -165,6 +170,7 @@ module Workflow
|
|
165
170
|
:input_descriptions => input_descriptions,
|
166
171
|
:input_defaults => input_defaults,
|
167
172
|
:result_type => result_type,
|
173
|
+
:result_description => result_description,
|
168
174
|
:dependencies => dependencies
|
169
175
|
}
|
170
176
|
end
|
@@ -2,7 +2,7 @@ module AnnotatedModule
|
|
2
2
|
def self.extended(base)
|
3
3
|
if not base.respond_to? :inputs
|
4
4
|
class << base
|
5
|
-
attr_accessor :description, :inputs, :input_types, :input_descriptions, :input_defaults
|
5
|
+
attr_accessor :description, :inputs, :input_types, :input_descriptions, :input_defaults, :result_description
|
6
6
|
|
7
7
|
def description
|
8
8
|
i = @description; @description = ""; i
|
@@ -27,6 +27,10 @@ module AnnotatedModule
|
|
27
27
|
def description
|
28
28
|
i = @description; @description = ""; i
|
29
29
|
end
|
30
|
+
|
31
|
+
def result_description
|
32
|
+
i = @result_description; @result_description = nil; i
|
33
|
+
end
|
30
34
|
end
|
31
35
|
|
32
36
|
base.description = ""
|
@@ -38,6 +42,10 @@ module AnnotatedModule
|
|
38
42
|
end
|
39
43
|
end
|
40
44
|
|
45
|
+
def returns(text)
|
46
|
+
@result_description = text
|
47
|
+
end
|
48
|
+
|
41
49
|
def desc(description)
|
42
50
|
@description = description
|
43
51
|
end
|
data/lib/rbbt/workflow/step.rb
CHANGED
@@ -86,11 +86,13 @@ class Step
|
|
86
86
|
def clean
|
87
87
|
if File.exists?(path) or File.exists?(info_file)
|
88
88
|
begin
|
89
|
-
FileUtils.rm info_file
|
90
|
-
FileUtils.rm path
|
91
|
-
FileUtils.
|
89
|
+
FileUtils.rm info_file if File.exists? info_file
|
90
|
+
FileUtils.rm path if File.exists? path
|
91
|
+
FileUtils.rm path + '.lock' if File.exists? path + '.lock'
|
92
|
+
FileUtils.rm_rf files_dir if File.exists? files_dir
|
92
93
|
end
|
93
94
|
end
|
95
|
+
self
|
94
96
|
end
|
95
97
|
|
96
98
|
def rec_dependencies
|
data/lib/rbbt/workflow/task.rb
CHANGED
@@ -2,7 +2,7 @@ require 'rbbt/util/misc'
|
|
2
2
|
require 'rbbt/persist'
|
3
3
|
|
4
4
|
module Task
|
5
|
-
attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :description, :name
|
5
|
+
attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :description, :name, :result_description
|
6
6
|
|
7
7
|
def self.setup(options = {}, &block)
|
8
8
|
block.extend Task
|
data/test/rbbt/test_tsv.rb
CHANGED
@@ -329,4 +329,35 @@ b 2
|
|
329
329
|
|
330
330
|
end
|
331
331
|
|
332
|
+
def test_flat_no_merge
|
333
|
+
content =<<-EOF
|
334
|
+
#Id ValueA ValueB OtherID
|
335
|
+
row1 a|aa|aaa b Id1|Id2
|
336
|
+
row2 A B Id3
|
337
|
+
EOF
|
338
|
+
|
339
|
+
TmpFile.with_file(content) do |filename|
|
340
|
+
tsv = TSV.open(filename, :sep => /\s+/, :type => :flat, :fields => ["ValueA"])
|
341
|
+
assert_equal ["a", "aa", "aaa"], tsv["row1"]
|
342
|
+
assert_equal ["ValueA"], tsv.fields
|
343
|
+
assert_equal :flat, tsv.type
|
344
|
+
assert_equal "Id", tsv.key_field
|
345
|
+
end
|
346
|
+
end
|
347
|
+
|
348
|
+
def test_flat_merge
|
349
|
+
content =<<-EOF
|
350
|
+
#Id ValueA ValueB OtherID
|
351
|
+
row1 a|aa|aaa b Id1|Id2
|
352
|
+
row1 aaaa b Id1|Id2
|
353
|
+
row2 A B Id3
|
354
|
+
EOF
|
355
|
+
|
356
|
+
TmpFile.with_file(content) do |filename|
|
357
|
+
tsv = TSV.open(filename, :sep => /\s+/, :merge => true, :type => :flat, :fields => ["ValueA"])
|
358
|
+
assert_equal ["a", "aa", "aaa", "aaaa"], tsv["row1"]
|
359
|
+
end
|
360
|
+
end
|
361
|
+
|
362
|
+
|
332
363
|
end
|
@@ -5,7 +5,7 @@ require 'test/unit'
|
|
5
5
|
|
6
6
|
class TestTSV < Test::Unit::TestCase
|
7
7
|
|
8
|
-
def
|
8
|
+
def test_tsv
|
9
9
|
content =<<-EOF
|
10
10
|
#Id ValueA ValueB OtherID
|
11
11
|
row1 a|aa|aaa b Id1|Id2
|
@@ -20,7 +20,7 @@ row2 A B Id3
|
|
20
20
|
end
|
21
21
|
end
|
22
22
|
|
23
|
-
def
|
23
|
+
def test_named_values
|
24
24
|
content =<<-EOF
|
25
25
|
#Id ValueA ValueB OtherID
|
26
26
|
row1 a|aa|aaa b Id1|Id2
|
@@ -33,7 +33,7 @@ row2 A B Id3
|
|
33
33
|
end
|
34
34
|
end
|
35
35
|
|
36
|
-
def
|
36
|
+
def test_to_s
|
37
37
|
content =<<-EOF
|
38
38
|
#Id ValueA ValueB OtherID
|
39
39
|
row1 a|aa|aaa b Id1|Id2
|
@@ -47,7 +47,7 @@ row2 A B Id3
|
|
47
47
|
end
|
48
48
|
end
|
49
49
|
|
50
|
-
def
|
50
|
+
def test_entries
|
51
51
|
content =<<-EOF
|
52
52
|
#Id ValueA ValueB OtherID
|
53
53
|
row1 a|aa|aaa b Id1|Id2
|
@@ -61,7 +61,7 @@ row2 A B Id3
|
|
61
61
|
|
62
62
|
end
|
63
63
|
|
64
|
-
def
|
64
|
+
def test_marshal
|
65
65
|
content =<<-EOF
|
66
66
|
#Id ValueA ValueB OtherID
|
67
67
|
row1 a|aa|aaa b Id1|Id2
|
@@ -74,12 +74,12 @@ row2 A B Id3
|
|
74
74
|
end
|
75
75
|
end
|
76
76
|
|
77
|
-
def
|
77
|
+
def test_zip_fields
|
78
78
|
a = [%w(1 2), %w(a b)]
|
79
79
|
assert_equal a, TSV.zip_fields(TSV.zip_fields(a))
|
80
80
|
end
|
81
81
|
|
82
|
-
def
|
82
|
+
def test_indentify_fields
|
83
83
|
content =<<-EOF
|
84
84
|
#ID ValueA ValueB Comment
|
85
85
|
row1 a b c
|
@@ -92,7 +92,7 @@ row2 A B C
|
|
92
92
|
end
|
93
93
|
end
|
94
94
|
|
95
|
-
def
|
95
|
+
def test_sort
|
96
96
|
content =<<-EOF
|
97
97
|
#ID ValueA ValueB Comment
|
98
98
|
row1 a B c
|
@@ -110,7 +110,7 @@ row2 A b C
|
|
110
110
|
end
|
111
111
|
end
|
112
112
|
|
113
|
-
def
|
113
|
+
def test_sort_by
|
114
114
|
content =<<-EOF
|
115
115
|
#ID ValueA ValueB Comment
|
116
116
|
row1 a B c
|
@@ -146,7 +146,7 @@ row6 A b a
|
|
146
146
|
end
|
147
147
|
|
148
148
|
|
149
|
-
def
|
149
|
+
def test_sort_by_with_proc
|
150
150
|
content =<<-EOF
|
151
151
|
#Id ValueA ValueB OtherID Pos
|
152
152
|
row1 a|aa|aaa b Id1|Id2 2
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
5
|
-
prerelease:
|
4
|
+
hash: 61
|
5
|
+
prerelease:
|
6
6
|
segments:
|
7
7
|
- 4
|
8
8
|
- 0
|
9
|
-
-
|
10
|
-
version: 4.0.
|
9
|
+
- 1
|
10
|
+
version: 4.0.1
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Miguel Vazquez
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2011-
|
18
|
+
date: 2011-10-03 00:00:00 +02:00
|
19
19
|
default_executable:
|
20
20
|
dependencies:
|
21
21
|
- !ruby/object:Gem::Dependency
|
@@ -117,7 +117,7 @@ dependencies:
|
|
117
117
|
type: :runtime
|
118
118
|
version_requirements: *id007
|
119
119
|
description: Utilities for handling tsv files, caches, etc
|
120
|
-
email: miguel.vazquez@
|
120
|
+
email: miguel.vazquez@cnio.es
|
121
121
|
executables:
|
122
122
|
- tsv.rb
|
123
123
|
- tchash.rb
|
@@ -237,7 +237,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
237
237
|
requirements: []
|
238
238
|
|
239
239
|
rubyforge_project:
|
240
|
-
rubygems_version: 1.
|
240
|
+
rubygems_version: 1.6.2
|
241
241
|
signing_key:
|
242
242
|
specification_version: 3
|
243
243
|
summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
|