rbbt-util 6.0.3 → 6.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE +1 -1
  3. data/bin/rbbt +1 -1
  4. data/bin/rbbt_exec.rb +2 -2
  5. data/lib/rbbt/hpc/batch.rb +1 -1
  6. data/lib/rbbt/knowledge_base/enrichment.rb +9 -9
  7. data/lib/rbbt/knowledge_base/entity.rb +128 -128
  8. data/lib/rbbt/knowledge_base/query.rb +94 -94
  9. data/lib/rbbt/knowledge_base/registry.rb +189 -189
  10. data/lib/rbbt/knowledge_base/syndicate.rb +26 -26
  11. data/lib/rbbt/knowledge_base/traverse.rb +315 -315
  12. data/lib/rbbt/knowledge_base.rb +37 -34
  13. data/lib/rbbt/tsv/excel.rb +1 -1
  14. data/lib/rbbt/util/filecache.rb +1 -1
  15. data/lib/rbbt/util/migrate.rb +4 -4
  16. data/lib/rbbt/util/misc/system.rb +92 -90
  17. data/lib/rbbt/workflow/refactor/export.rb +66 -66
  18. data/lib/rbbt/workflow/refactor/recursive.rb +64 -64
  19. data/lib/rbbt/workflow/refactor/task_info.rb +66 -65
  20. data/lib/rbbt/workflow/refactor.rb +0 -3
  21. data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +1 -2
  22. data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb +11 -11
  23. data/lib/rbbt/workflow/remote_workflow/remote_step.rb +1 -1
  24. data/lib/rbbt/workflow/remote_workflow.rb +2 -1
  25. data/lib/rbbt-util.rb +2 -2
  26. data/lib/rbbt.rb +1 -1
  27. data/python/rbbt/'/Users/miki/config/tmp/undodir'/%Users%miki%git%rbbt-util%python%rbbt%__init__.py +0 -0
  28. data/python/rbbt/__init__.py +78 -4
  29. data/python/rbbt/workflow/remote.py +104 -0
  30. data/python/rbbt/workflow.py +64 -0
  31. data/python/test.py +10 -0
  32. data/share/rbbt_commands/hpc/list +1 -1
  33. data/share/rbbt_commands/lsf/list +1 -1
  34. data/share/rbbt_commands/pbs/list +1 -1
  35. data/share/rbbt_commands/resource/find +1 -1
  36. data/share/rbbt_commands/slurm/list +1 -1
  37. data/share/rbbt_commands/system/status +2 -2
  38. data/share/rbbt_commands/workflow/info +1 -1
  39. data/share/rbbt_commands/workflow/retry +43 -0
  40. data/share/rbbt_commands/workflow/server +1 -1
  41. data/share/rbbt_commands/workflow/task +4 -2
  42. metadata +8 -6
@@ -1,315 +1,315 @@
1
- require 'rbbt-util'
2
- require 'rbbt/knowledge_base'
3
-
4
- class KnowledgeBase
5
-
6
- class Traverser
7
- attr_accessor :rules, :assignments, :matches, :kb
8
-
9
- def initialize(kb, rules = [])
10
- @kb = kb
11
- @rules = rules
12
- @assignments = {}
13
- @matches = {}
14
- end
15
-
16
- def wildcard(name)
17
- return name unless is_wildcard?(name)
18
- assignments[name] || name
19
- end
20
-
21
- def is_wildcard?(name)
22
- name[0] == '?'
23
- end
24
-
25
- def identify(db, source, target)
26
- source_entities = if is_wildcard? source
27
- assignments[source] || :all
28
- else
29
- kb.identify_source db, source
30
- end
31
-
32
- target_entities = if is_wildcard? target
33
- assignments[target] || :all
34
- else
35
- kb.identify_target db, target
36
- end
37
-
38
- source_entities = [source_entities] unless Array === source_entities or source_entities == :all
39
- target_entities = [target_entities] unless Array === target_entities or target_entities == :all
40
-
41
- [source_entities, target_entities]
42
- end
43
-
44
- def reassign(matches, source, target)
45
- assignments[source] = (matches.any? ? matches.collect{|m|m.partition("~").first}.uniq : nil) if is_wildcard? source
46
- assignments[target] = (matches.any? ? matches.collect{|m|m.partition("~").last}.uniq : nil) if is_wildcard? target
47
- end
48
-
49
- def clean_matches(rules, all_matches, assignments)
50
- paths = {}
51
-
52
- rules.zip(all_matches).each do |rule, matches|
53
- source, db, target = rule.split /\s+/
54
- next if matches.nil?
55
-
56
- if is_wildcard? source
57
- assigned = assignments[source] || []
58
- matches = matches.select{|m| assigned.include? m.partition("~").first }
59
- end
60
-
61
- if is_wildcard? target
62
- assigned = assignments[target] || []
63
- matches = matches.select{|m| assigned.include? m.partition("~").last }
64
- end
65
-
66
- paths[rule] = matches
67
- end
68
-
69
- paths
70
- end
71
-
72
- def _fp(rules, clean_matches, assignments)
73
- return true if rules.empty?
74
-
75
- rule, *rest = rules
76
- source, db, target = rule.split /\s+/
77
-
78
- wildcard_source = is_wildcard? source
79
- wildcard_target = is_wildcard? target
80
-
81
- paths = {}
82
- matches = clean_matches[rule]
83
- Annotation.purge(matches).each do |match|
84
- new_assignments = nil
85
- match_source, _sep, match_target = match.partition "~"
86
-
87
- if wildcard_source
88
- next if assignments[source] and assignments[source] != match_source
89
- new_assignments ||= assignments.dup
90
- new_assignments[source] = match_source
91
- end
92
-
93
- if wildcard_target
94
- next if assignments[target] and assignments[target] != match_target
95
- new_assignments ||= assignments.dup
96
- new_assignments[target] = match_target
97
- end
98
-
99
- new_paths = _fp(rest, clean_matches, new_assignments)
100
- next unless new_paths
101
- paths[match] = new_paths
102
- end
103
-
104
- return false if paths.empty?
105
-
106
- paths
107
- end
108
-
109
- def _ep(paths)
110
- found = []
111
- paths.each do |match,_next|
112
- case _next
113
- when TrueClass
114
- found << [match]
115
- when FalseClass
116
- next
117
- else
118
- _ep(_next).each do |_n|
119
- found << [match] + _n
120
- end
121
- end
122
- end
123
- found
124
- end
125
-
126
- def find_paths(rules, all_matches, assignments)
127
- clean_matches = clean_matches(rules, all_matches, assignments)
128
-
129
- path_hash = _fp(rules, clean_matches, {})
130
-
131
- return [] unless path_hash
132
- _ep(path_hash).collect do |path|
133
- path.zip(clean_matches.values_at(*rules)).collect do |item, matches|
134
- matches.select{|m| m == item}.first
135
- end
136
- end
137
- end
138
-
139
- def traverse_db(db, source, target, conditions)
140
- source_entities, target_entities = identify db, source, target
141
-
142
- options = {:source => source_entities, :target => target_entities}
143
- Log.debug "Traversing #{ db }: #{Misc.fingerprint options}"
144
- matches = kb.subset(db, options)
145
-
146
- if conditions
147
- Misc.tokenize(conditions).each do |condition|
148
- if condition.index "="
149
- key, value = condition.split("=")
150
- matches = matches.select{|m| Misc.match_value(m.info[key.strip], value)}
151
- else
152
- matches = matches.select{|m| m.info[condition.strip].to_s =~ /\btrue\b/}
153
- end
154
- end
155
- end
156
-
157
- matches
158
- end
159
-
160
- def id_dbs(db)
161
- # ToDo: Revise this, I'm not sure what id does anymore
162
- # I think it deals with syndication
163
- if db.include? '?'
164
- all_dbs = kb.registry.keys.collect{|k| k.to_s }
165
- _name, _sep, _kb = db.partition("@")
166
- case
167
- when _name[0] == '?'
168
- dbs = all_dbs.select{|_db|
169
- n,_s,d=_db.partition("@");
170
- d.nil? or d.empty? or (d == _kb and assignments[_name].include?(n))
171
- }
172
- when _kb[0] == '?'
173
- dbs = all_dbs.select{|_db| n,_s,d=_db.partition("@"); n == _name and assignments[_kb].include?(d) }
174
- end
175
- else
176
- dbs = [db]
177
- end
178
-
179
- dbs
180
- end
181
-
182
- def traverse
183
- all_matches = []
184
- path_rules = []
185
- acc_var = nil
186
- pre_acc_var_assignments = nil
187
- rules.each do |rule|
188
- rule = rule.strip
189
- next if rule.empty?
190
-
191
- if m = rule.match(/([^\s]+)\s+([^\s=]+)\s+([^\s]+)(?:\s+-\s+(.*))?/)
192
- Log.debug "Traverse rule: #{rule}"
193
- path_rules << rule
194
-
195
- source, db, target, conditions = m.captures
196
-
197
- dbs = id_dbs(db)
198
-
199
- rule_matches = []
200
- dbs.each do |_db|
201
- matches = traverse_db(_db, source, target, conditions)
202
-
203
- next if matches.nil? or matches.empty?
204
-
205
- # ToDo: Revise this, I'm not sure what id does anymore
206
- #
207
- #if db.include? '?'
208
- # _name, _sep, _kb = db.partition("@")
209
- # case
210
- # when _kb[0] == '?'
211
- # assignments[_kb] ||= []
212
- # assignments[_kb] << _db.partition("@").reject{|p| p.empty?}.last
213
- # when _name[0] == '?'
214
- # assignments[_name] ||= []
215
- # assignments[_name] << _db.partition("@").first
216
- # end
217
- #end
218
-
219
- matches.each do |m|
220
- rule_matches << m
221
- end
222
-
223
- assignments.each{|k,v| v.uniq! if v}
224
- end
225
-
226
- reassign rule_matches, source, target
227
-
228
- all_matches << rule_matches
229
-
230
- elsif m = rule.match(/([^\s=]+)\s*=([^\s]*)\s*(.*)/)
231
- Log.debug "Assign rule: #{rule}"
232
- var, db, value_str = m.captures
233
- names = value_str.split(",").collect{|v| v.strip}
234
- if db.empty?
235
- ids = names
236
- else
237
- dbs = id_dbs(db)
238
- ids = names.collect{|name|
239
- id = nil
240
- dbs.each do |db|
241
- sid, tid = identify db, name, name
242
- id = (sid + tid).compact.first
243
- break if id
244
- end
245
- id
246
- }
247
- end
248
- assignments[var] = ids
249
-
250
- elsif m = rule.match(/(\?[^\s{]+)\s*{/)
251
- acc_var = m.captures.first
252
- pre_acc_var_assignments = assignments.dup
253
- Log.debug "Start assign block: #{acc_var}"
254
- elsif m = rule.match(/^\s*}\s*$/)
255
- Log.debug "Close assign block: #{acc_var}"
256
- saved_assign = assignments[acc_var]
257
- assignments.clear
258
- assignments.merge!(pre_acc_var_assignments)
259
- pre_acc_var_assignments = nil
260
- assignments[acc_var] = saved_assign
261
- all_matches = []
262
- path_rules = []
263
- else
264
- raise "Rule not understood: #{rule}"
265
- end
266
- end
267
-
268
- Log.debug "Finding paths: #{all_matches.length}"
269
- paths = find_paths path_rules, all_matches, assignments
270
- Log.debug "Found paths: #{paths.length}"
271
-
272
- [assignments, paths]
273
- end
274
-
275
- #def traverse
276
- # all_matches = []
277
-
278
- # rules.each do |rule|
279
- # rule = rule.strip
280
- # next if rule.empty?
281
- # source, db, target, conditions = rule.match(/([^\s]+)\s+([^\s]+)\s+([^\s]+)(?:\s+-\s+([^\s]+))?/).captures
282
-
283
- # source_entities, target_entities = identify db, source, target
284
-
285
- # matches = kb.subset(db, :source => source_entities, :target => target_entities)
286
-
287
- # if conditions
288
- # conditions.split(/\s+/).each do |condition|
289
- # if condition.index "="
290
- # key, value = conditions.split("=")
291
- # matches = matches.select{|m| m.info[key.strip].to_s =~ /\b#{value.strip}\b/}
292
- # else
293
- # matches = matches.select{|m| m.info[condition.strip].to_s =~ /\btrue\b/}
294
- # end
295
- # end
296
- # end
297
-
298
- # reassign matches, source, target
299
-
300
- # all_matches << matches
301
- # end
302
-
303
- # paths = find_paths rules, all_matches, assignments
304
-
305
- # [assignments, paths]
306
- #end
307
-
308
- end
309
-
310
- def traverse(rules)
311
- traverser = KnowledgeBase::Traverser.new self, rules
312
- traverser.traverse
313
- end
314
-
315
- end
1
+ #require 'rbbt-util'
2
+ #require 'rbbt/knowledge_base'
3
+ #
4
+ #class KnowledgeBase
5
+ #
6
+ # class Traverser
7
+ # attr_accessor :rules, :assignments, :matches, :kb
8
+ #
9
+ # def initialize(kb, rules = [])
10
+ # @kb = kb
11
+ # @rules = rules
12
+ # @assignments = {}
13
+ # @matches = {}
14
+ # end
15
+ #
16
+ # def wildcard(name)
17
+ # return name unless is_wildcard?(name)
18
+ # assignments[name] || name
19
+ # end
20
+ #
21
+ # def is_wildcard?(name)
22
+ # name[0] == '?'
23
+ # end
24
+ #
25
+ # def identify(db, source, target)
26
+ # source_entities = if is_wildcard? source
27
+ # assignments[source] || :all
28
+ # else
29
+ # kb.identify_source db, source
30
+ # end
31
+ #
32
+ # target_entities = if is_wildcard? target
33
+ # assignments[target] || :all
34
+ # else
35
+ # kb.identify_target db, target
36
+ # end
37
+ #
38
+ # source_entities = [source_entities] unless Array === source_entities or source_entities == :all
39
+ # target_entities = [target_entities] unless Array === target_entities or target_entities == :all
40
+ #
41
+ # [source_entities, target_entities]
42
+ # end
43
+ #
44
+ # def reassign(matches, source, target)
45
+ # assignments[source] = (matches.any? ? matches.collect{|m|m.partition("~").first}.uniq : nil) if is_wildcard? source
46
+ # assignments[target] = (matches.any? ? matches.collect{|m|m.partition("~").last}.uniq : nil) if is_wildcard? target
47
+ # end
48
+ #
49
+ # def clean_matches(rules, all_matches, assignments)
50
+ # paths = {}
51
+ #
52
+ # rules.zip(all_matches).each do |rule, matches|
53
+ # source, db, target = rule.split /\s+/
54
+ # next if matches.nil?
55
+ #
56
+ # if is_wildcard? source
57
+ # assigned = assignments[source] || []
58
+ # matches = matches.select{|m| assigned.include? m.partition("~").first }
59
+ # end
60
+ #
61
+ # if is_wildcard? target
62
+ # assigned = assignments[target] || []
63
+ # matches = matches.select{|m| assigned.include? m.partition("~").last }
64
+ # end
65
+ #
66
+ # paths[rule] = matches
67
+ # end
68
+ #
69
+ # paths
70
+ # end
71
+ #
72
+ # def _fp(rules, clean_matches, assignments)
73
+ # return true if rules.empty?
74
+ #
75
+ # rule, *rest = rules
76
+ # source, db, target = rule.split /\s+/
77
+ #
78
+ # wildcard_source = is_wildcard? source
79
+ # wildcard_target = is_wildcard? target
80
+ #
81
+ # paths = {}
82
+ # matches = clean_matches[rule]
83
+ # Annotation.purge(matches).each do |match|
84
+ # new_assignments = nil
85
+ # match_source, _sep, match_target = match.partition "~"
86
+ #
87
+ # if wildcard_source
88
+ # next if assignments[source] and assignments[source] != match_source
89
+ # new_assignments ||= assignments.dup
90
+ # new_assignments[source] = match_source
91
+ # end
92
+ #
93
+ # if wildcard_target
94
+ # next if assignments[target] and assignments[target] != match_target
95
+ # new_assignments ||= assignments.dup
96
+ # new_assignments[target] = match_target
97
+ # end
98
+ #
99
+ # new_paths = _fp(rest, clean_matches, new_assignments)
100
+ # next unless new_paths
101
+ # paths[match] = new_paths
102
+ # end
103
+ #
104
+ # return false if paths.empty?
105
+ #
106
+ # paths
107
+ # end
108
+ #
109
+ # def _ep(paths)
110
+ # found = []
111
+ # paths.each do |match,_next|
112
+ # case _next
113
+ # when TrueClass
114
+ # found << [match]
115
+ # when FalseClass
116
+ # next
117
+ # else
118
+ # _ep(_next).each do |_n|
119
+ # found << [match] + _n
120
+ # end
121
+ # end
122
+ # end
123
+ # found
124
+ # end
125
+ #
126
+ # def find_paths(rules, all_matches, assignments)
127
+ # clean_matches = clean_matches(rules, all_matches, assignments)
128
+ #
129
+ # path_hash = _fp(rules, clean_matches, {})
130
+ #
131
+ # return [] unless path_hash
132
+ # _ep(path_hash).collect do |path|
133
+ # path.zip(clean_matches.values_at(*rules)).collect do |item, matches|
134
+ # matches.select{|m| m == item}.first
135
+ # end
136
+ # end
137
+ # end
138
+ #
139
+ # def traverse_db(db, source, target, conditions)
140
+ # source_entities, target_entities = identify db, source, target
141
+ #
142
+ # options = {:source => source_entities, :target => target_entities}
143
+ # Log.debug "Traversing #{ db }: #{Misc.fingerprint options}"
144
+ # matches = kb.subset(db, options)
145
+ #
146
+ # if conditions
147
+ # Misc.tokenize(conditions).each do |condition|
148
+ # if condition.index "="
149
+ # key, value = condition.split("=")
150
+ # matches = matches.select{|m| Misc.match_value(m.info[key.strip], value)}
151
+ # else
152
+ # matches = matches.select{|m| m.info[condition.strip].to_s =~ /\btrue\b/}
153
+ # end
154
+ # end
155
+ # end
156
+ #
157
+ # matches
158
+ # end
159
+ #
160
+ # def id_dbs(db)
161
+ # # ToDo: Revise this, I'm not sure what id does anymore
162
+ # # I think it deals with syndication
163
+ # if db.include? '?'
164
+ # all_dbs = kb.registry.keys.collect{|k| k.to_s }
165
+ # _name, _sep, _kb = db.partition("@")
166
+ # case
167
+ # when _name[0] == '?'
168
+ # dbs = all_dbs.select{|_db|
169
+ # n,_s,d=_db.partition("@");
170
+ # d.nil? or d.empty? or (d == _kb and assignments[_name].include?(n))
171
+ # }
172
+ # when _kb[0] == '?'
173
+ # dbs = all_dbs.select{|_db| n,_s,d=_db.partition("@"); n == _name and assignments[_kb].include?(d) }
174
+ # end
175
+ # else
176
+ # dbs = [db]
177
+ # end
178
+ #
179
+ # dbs
180
+ # end
181
+ #
182
+ # def traverse
183
+ # all_matches = []
184
+ # path_rules = []
185
+ # acc_var = nil
186
+ # pre_acc_var_assignments = nil
187
+ # rules.each do |rule|
188
+ # rule = rule.strip
189
+ # next if rule.empty?
190
+ #
191
+ # if m = rule.match(/([^\s]+)\s+([^\s=]+)\s+([^\s]+)(?:\s+-\s+(.*))?/)
192
+ # Log.debug "Traverse rule: #{rule}"
193
+ # path_rules << rule
194
+ #
195
+ # source, db, target, conditions = m.captures
196
+ #
197
+ # dbs = id_dbs(db)
198
+ #
199
+ # rule_matches = []
200
+ # dbs.each do |_db|
201
+ # matches = traverse_db(_db, source, target, conditions)
202
+ #
203
+ # next if matches.nil? or matches.empty?
204
+ #
205
+ # # ToDo: Revise this, I'm not sure what id does anymore
206
+ # #
207
+ # #if db.include? '?'
208
+ # # _name, _sep, _kb = db.partition("@")
209
+ # # case
210
+ # # when _kb[0] == '?'
211
+ # # assignments[_kb] ||= []
212
+ # # assignments[_kb] << _db.partition("@").reject{|p| p.empty?}.last
213
+ # # when _name[0] == '?'
214
+ # # assignments[_name] ||= []
215
+ # # assignments[_name] << _db.partition("@").first
216
+ # # end
217
+ # #end
218
+ #
219
+ # matches.each do |m|
220
+ # rule_matches << m
221
+ # end
222
+ #
223
+ # assignments.each{|k,v| v.uniq! if v}
224
+ # end
225
+ #
226
+ # reassign rule_matches, source, target
227
+ #
228
+ # all_matches << rule_matches
229
+ #
230
+ # elsif m = rule.match(/([^\s=]+)\s*=([^\s]*)\s*(.*)/)
231
+ # Log.debug "Assign rule: #{rule}"
232
+ # var, db, value_str = m.captures
233
+ # names = value_str.split(",").collect{|v| v.strip}
234
+ # if db.empty?
235
+ # ids = names
236
+ # else
237
+ # dbs = id_dbs(db)
238
+ # ids = names.collect{|name|
239
+ # id = nil
240
+ # dbs.each do |db|
241
+ # sid, tid = identify db, name, name
242
+ # id = (sid + tid).compact.first
243
+ # break if id
244
+ # end
245
+ # id
246
+ # }
247
+ # end
248
+ # assignments[var] = ids
249
+ #
250
+ # elsif m = rule.match(/(\?[^\s{]+)\s*{/)
251
+ # acc_var = m.captures.first
252
+ # pre_acc_var_assignments = assignments.dup
253
+ # Log.debug "Start assign block: #{acc_var}"
254
+ # elsif m = rule.match(/^\s*}\s*$/)
255
+ # Log.debug "Close assign block: #{acc_var}"
256
+ # saved_assign = assignments[acc_var]
257
+ # assignments.clear
258
+ # assignments.merge!(pre_acc_var_assignments)
259
+ # pre_acc_var_assignments = nil
260
+ # assignments[acc_var] = saved_assign
261
+ # all_matches = []
262
+ # path_rules = []
263
+ # else
264
+ # raise "Rule not understood: #{rule}"
265
+ # end
266
+ # end
267
+ #
268
+ # Log.debug "Finding paths: #{all_matches.length}"
269
+ # paths = find_paths path_rules, all_matches, assignments
270
+ # Log.debug "Found paths: #{paths.length}"
271
+ #
272
+ # [assignments, paths]
273
+ # end
274
+ #
275
+ # #def traverse
276
+ # # all_matches = []
277
+ #
278
+ # # rules.each do |rule|
279
+ # # rule = rule.strip
280
+ # # next if rule.empty?
281
+ # # source, db, target, conditions = rule.match(/([^\s]+)\s+([^\s]+)\s+([^\s]+)(?:\s+-\s+([^\s]+))?/).captures
282
+ #
283
+ # # source_entities, target_entities = identify db, source, target
284
+ #
285
+ # # matches = kb.subset(db, :source => source_entities, :target => target_entities)
286
+ #
287
+ # # if conditions
288
+ # # conditions.split(/\s+/).each do |condition|
289
+ # # if condition.index "="
290
+ # # key, value = conditions.split("=")
291
+ # # matches = matches.select{|m| m.info[key.strip].to_s =~ /\b#{value.strip}\b/}
292
+ # # else
293
+ # # matches = matches.select{|m| m.info[condition.strip].to_s =~ /\btrue\b/}
294
+ # # end
295
+ # # end
296
+ # # end
297
+ #
298
+ # # reassign matches, source, target
299
+ #
300
+ # # all_matches << matches
301
+ # # end
302
+ #
303
+ # # paths = find_paths rules, all_matches, assignments
304
+ #
305
+ # # [assignments, paths]
306
+ # #end
307
+ #
308
+ # end
309
+ #
310
+ # def traverse(rules)
311
+ # traverser = KnowledgeBase::Traverser.new self, rules
312
+ # traverser.traverse
313
+ # end
314
+ #
315
+ #end