rbbt-util 6.0.3 → 6.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE +1 -1
- data/bin/rbbt +1 -1
- data/bin/rbbt_exec.rb +2 -2
- data/lib/rbbt/hpc/batch.rb +1 -1
- data/lib/rbbt/knowledge_base/enrichment.rb +9 -9
- data/lib/rbbt/knowledge_base/entity.rb +128 -128
- data/lib/rbbt/knowledge_base/query.rb +94 -94
- data/lib/rbbt/knowledge_base/registry.rb +189 -189
- data/lib/rbbt/knowledge_base/syndicate.rb +26 -26
- data/lib/rbbt/knowledge_base/traverse.rb +315 -315
- data/lib/rbbt/knowledge_base.rb +37 -34
- data/lib/rbbt/tsv/excel.rb +1 -1
- data/lib/rbbt/util/filecache.rb +1 -1
- data/lib/rbbt/util/migrate.rb +4 -4
- data/lib/rbbt/util/misc/system.rb +92 -90
- data/lib/rbbt/workflow/refactor/export.rb +66 -66
- data/lib/rbbt/workflow/refactor/recursive.rb +64 -64
- data/lib/rbbt/workflow/refactor/task_info.rb +66 -65
- data/lib/rbbt/workflow/refactor.rb +0 -3
- data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +1 -2
- data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb +11 -11
- data/lib/rbbt/workflow/remote_workflow/remote_step.rb +1 -1
- data/lib/rbbt/workflow/remote_workflow.rb +2 -1
- data/lib/rbbt-util.rb +2 -2
- data/lib/rbbt.rb +1 -1
- data/python/rbbt/'/Users/miki/config/tmp/undodir'/%Users%miki%git%rbbt-util%python%rbbt%__init__.py +0 -0
- data/python/rbbt/__init__.py +78 -4
- data/python/rbbt/workflow/remote.py +104 -0
- data/python/rbbt/workflow.py +64 -0
- data/python/test.py +10 -0
- data/share/rbbt_commands/hpc/list +1 -1
- data/share/rbbt_commands/lsf/list +1 -1
- data/share/rbbt_commands/pbs/list +1 -1
- data/share/rbbt_commands/resource/find +1 -1
- data/share/rbbt_commands/slurm/list +1 -1
- data/share/rbbt_commands/system/status +2 -2
- data/share/rbbt_commands/workflow/info +1 -1
- data/share/rbbt_commands/workflow/retry +43 -0
- data/share/rbbt_commands/workflow/server +1 -1
- data/share/rbbt_commands/workflow/task +4 -2
- metadata +8 -6
data/lib/rbbt/knowledge_base.rb
CHANGED
@@ -1,35 +1,38 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
require 'rbbt/association/item'
|
4
|
-
require 'rbbt/knowledge_base/entity'
|
5
|
-
require 'rbbt/knowledge_base/query'
|
6
|
-
require 'rbbt/knowledge_base/syndicate'
|
1
|
+
require_relative 'refactor'
|
2
|
+
Rbbt.require_instead 'scout/knowledge_base'
|
7
3
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
@
|
22
|
-
@
|
23
|
-
|
24
|
-
@
|
25
|
-
@
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
4
|
+
#require 'rbbt/annotations'
|
5
|
+
#require 'rbbt/association'
|
6
|
+
#require 'rbbt/association/item'
|
7
|
+
#require 'rbbt/knowledge_base/entity'
|
8
|
+
#require 'rbbt/knowledge_base/query'
|
9
|
+
#require 'rbbt/knowledge_base/syndicate'
|
10
|
+
#
|
11
|
+
#class KnowledgeBase
|
12
|
+
#
|
13
|
+
# attr_accessor :namespace, :dir, :databases, :indices, :registry, :format, :entity_options
|
14
|
+
# def initialize(dir, namespace = nil)
|
15
|
+
# @dir = Path.setup(dir.dup)
|
16
|
+
#
|
17
|
+
# @namespace = namespace
|
18
|
+
# @format = IndiferentHash.setup({})
|
19
|
+
#
|
20
|
+
# @registry ||= IndiferentHash.setup({})
|
21
|
+
# @entity_options = IndiferentHash.setup({})
|
22
|
+
#
|
23
|
+
# @indices = IndiferentHash.setup({})
|
24
|
+
# @databases = IndiferentHash.setup({})
|
25
|
+
# @identifiers = IndiferentHash.setup({})
|
26
|
+
# @fields = {}
|
27
|
+
# @descriptions = {}
|
28
|
+
# @databases = {}
|
29
|
+
# end
|
30
|
+
#
|
31
|
+
# def self.load(dir)
|
32
|
+
# KnowledgeBase.new dir
|
33
|
+
# end
|
34
|
+
#
|
35
|
+
# def setup(name, matches, reverse = false)
|
36
|
+
# AssociationItem.setup matches, self, name, reverse
|
37
|
+
# end
|
38
|
+
#end
|
data/lib/rbbt/tsv/excel.rb
CHANGED
data/lib/rbbt/util/filecache.rb
CHANGED
data/lib/rbbt/util/migrate.rb
CHANGED
@@ -2,16 +2,16 @@ module Rbbt
|
|
2
2
|
|
3
3
|
def self.migrate_source_paths(path, resource = Rbbt, source = nil)
|
4
4
|
if source
|
5
|
-
lpath, *paths =
|
5
|
+
lpath, *paths = SSHLine.ruby(source, <<-EOF).split("\n")
|
6
6
|
require 'rbbt-util'
|
7
7
|
path = "#{path}"
|
8
8
|
if Open.exists?(path)
|
9
|
-
path = #{resource.to_s}.identify(path)
|
9
|
+
path = Path.setup(#{resource.to_s}.identify(path))
|
10
10
|
else
|
11
11
|
path = Path.setup(path)
|
12
12
|
end
|
13
13
|
puts path
|
14
|
-
puts path.
|
14
|
+
puts path.find_all.collect{|p| File.directory?(p) ? p + "/" : p } * "\n"
|
15
15
|
EOF
|
16
16
|
|
17
17
|
[path, paths.collect{|p| [source, p] * ":"}, lpath]
|
@@ -36,7 +36,7 @@ puts path.glob_all.collect{|p| File.directory?(p) ? p + "/" : p } * "\n"
|
|
36
36
|
|
37
37
|
def self.migrate_target_path(path, search_path = 'user', resource = Rbbt, target = nil)
|
38
38
|
if target
|
39
|
-
|
39
|
+
SSHLine.ruby(target, <<-EOF).split("\n").first
|
40
40
|
require 'rbbt-util'
|
41
41
|
path = "#{path}"
|
42
42
|
resource = #{resource.to_s}
|
@@ -1,90 +1,92 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
ENV[var]
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
return
|
83
|
-
return true if
|
84
|
-
return
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
1
|
+
require_relative 'refactor'
|
2
|
+
Rbbt.require_instead 'scout/misc/format'
|
3
|
+
#module Misc
|
4
|
+
#
|
5
|
+
# def self.hostname
|
6
|
+
# @hostanem ||= `hostname`.strip
|
7
|
+
# end
|
8
|
+
#
|
9
|
+
# def self.pid_exists?(pid)
|
10
|
+
# return false if pid.nil?
|
11
|
+
# begin
|
12
|
+
# Process.getpgid(pid.to_i)
|
13
|
+
# true
|
14
|
+
# rescue Errno::ESRCH
|
15
|
+
# false
|
16
|
+
# end
|
17
|
+
# end
|
18
|
+
#
|
19
|
+
# def self.env_add(var, value, sep = ":", prepend = true)
|
20
|
+
# ENV[var] ||= ""
|
21
|
+
# return if ENV[var] =~ /(#{sep}|^)#{Regexp.quote value}(#{sep}|$)/
|
22
|
+
# if prepend
|
23
|
+
# ENV[var] = value + sep + ENV[var]
|
24
|
+
# else
|
25
|
+
# ENV[var] += sep + ENV[var]
|
26
|
+
# end
|
27
|
+
# end
|
28
|
+
#
|
29
|
+
# def self.with_env(var, value, &block)
|
30
|
+
# var = var.to_s
|
31
|
+
# value = value.to_s
|
32
|
+
# current = ENV[var]
|
33
|
+
# begin
|
34
|
+
# ENV[var] = value
|
35
|
+
# yield
|
36
|
+
# ensure
|
37
|
+
# ENV[var] = current
|
38
|
+
# end
|
39
|
+
# end
|
40
|
+
#
|
41
|
+
# def self.common_path(dir, file)
|
42
|
+
# file = File.expand_path file
|
43
|
+
# dir = File.expand_path dir
|
44
|
+
#
|
45
|
+
# return true if file == dir
|
46
|
+
# while File.dirname(file) != file
|
47
|
+
# file = File.dirname(file)
|
48
|
+
# return true if file == dir
|
49
|
+
# end
|
50
|
+
#
|
51
|
+
# return false
|
52
|
+
# end
|
53
|
+
#
|
54
|
+
#
|
55
|
+
# def self.relative_link(source, target_dir)
|
56
|
+
# path = "."
|
57
|
+
# current = target_dir
|
58
|
+
# while ! Misc.common_path current, source
|
59
|
+
# current = File.dirname(current)
|
60
|
+
# path = File.join(path, '..')
|
61
|
+
# return nil if current == "/"
|
62
|
+
# end
|
63
|
+
#
|
64
|
+
# File.join(path, Misc.path_relative_to(current, source))
|
65
|
+
# end
|
66
|
+
#
|
67
|
+
# # WARN: probably not thread safe...
|
68
|
+
# def self.in_dir(dir)
|
69
|
+
# old_pwd = FileUtils.pwd
|
70
|
+
# res = nil
|
71
|
+
# begin
|
72
|
+
# FileUtils.mkdir_p dir unless File.exist?(dir)
|
73
|
+
# FileUtils.cd dir
|
74
|
+
# res = yield
|
75
|
+
# ensure
|
76
|
+
# FileUtils.cd old_pwd
|
77
|
+
# end
|
78
|
+
# res
|
79
|
+
# end
|
80
|
+
#
|
81
|
+
# def self.is_filename?(string, need_to_exists = true)
|
82
|
+
# return false if string.nil?
|
83
|
+
# return true if defined? Path and Path === string
|
84
|
+
# return true if string.respond_to? :exists
|
85
|
+
# return true if String === string and ! string.include?("\n") and string.split("/").select{|p| p.length > 265}.empty? and (! need_to_exists || File.exist?(string))
|
86
|
+
# return false
|
87
|
+
# end
|
88
|
+
#
|
89
|
+
# class << self
|
90
|
+
# alias filename? is_filename?
|
91
|
+
# end
|
92
|
+
#end
|
@@ -1,66 +1,66 @@
|
|
1
|
-
module Workflow
|
2
|
-
|
3
|
-
annotation :asynchronous_exports, :synchronous_exports, :exec_exports, :stream_exports
|
4
|
-
|
5
|
-
def asynchronous_exports
|
6
|
-
@asynchronous_exports ||= []
|
7
|
-
end
|
8
|
-
|
9
|
-
def synchronous_exports
|
10
|
-
@synchronous_exports ||= []
|
11
|
-
end
|
12
|
-
|
13
|
-
def exec_exports
|
14
|
-
@exec_exports ||= []
|
15
|
-
end
|
16
|
-
|
17
|
-
def stream_exports
|
18
|
-
@exec_exports ||= []
|
19
|
-
end
|
20
|
-
|
21
|
-
|
22
|
-
def all_exports
|
23
|
-
asynchronous_exports + synchronous_exports + exec_exports + stream_exports
|
24
|
-
end
|
25
|
-
|
26
|
-
alias task_exports all_exports
|
27
|
-
|
28
|
-
def unexport(*names)
|
29
|
-
names = names.collect{|n| n.to_s} + names.collect{|n| n.to_sym}
|
30
|
-
names.uniq!
|
31
|
-
exec_exports.replace exec_exports - names if exec_exports
|
32
|
-
synchronous_exports.replace synchronous_exports - names if synchronous_exports
|
33
|
-
asynchronous_exports.replace asynchronous_exports - names if asynchronous_exports
|
34
|
-
stream_exports.replace stream_exports - names if stream_exports
|
35
|
-
end
|
36
|
-
|
37
|
-
def export_exec(*names)
|
38
|
-
unexport *names
|
39
|
-
exec_exports.concat names
|
40
|
-
exec_exports.uniq!
|
41
|
-
exec_exports
|
42
|
-
end
|
43
|
-
|
44
|
-
def export_synchronous(*names)
|
45
|
-
unexport *names
|
46
|
-
synchronous_exports.concat names
|
47
|
-
synchronous_exports.uniq!
|
48
|
-
synchronous_exports
|
49
|
-
end
|
50
|
-
|
51
|
-
def export_asynchronous(*names)
|
52
|
-
unexport *names
|
53
|
-
asynchronous_exports.concat names
|
54
|
-
asynchronous_exports.uniq!
|
55
|
-
asynchronous_exports
|
56
|
-
end
|
57
|
-
|
58
|
-
def export_stream(*names)
|
59
|
-
unexport *names
|
60
|
-
stream_exports.concat names
|
61
|
-
stream_exports.uniq!
|
62
|
-
stream_exports
|
63
|
-
end
|
64
|
-
|
65
|
-
alias export export_asynchronous
|
66
|
-
end
|
1
|
+
#module Workflow
|
2
|
+
#
|
3
|
+
# annotation :asynchronous_exports, :synchronous_exports, :exec_exports, :stream_exports
|
4
|
+
#
|
5
|
+
# def asynchronous_exports
|
6
|
+
# @asynchronous_exports ||= []
|
7
|
+
# end
|
8
|
+
#
|
9
|
+
# def synchronous_exports
|
10
|
+
# @synchronous_exports ||= []
|
11
|
+
# end
|
12
|
+
#
|
13
|
+
# def exec_exports
|
14
|
+
# @exec_exports ||= []
|
15
|
+
# end
|
16
|
+
#
|
17
|
+
# def stream_exports
|
18
|
+
# @exec_exports ||= []
|
19
|
+
# end
|
20
|
+
#
|
21
|
+
#
|
22
|
+
# def all_exports
|
23
|
+
# asynchronous_exports + synchronous_exports + exec_exports + stream_exports
|
24
|
+
# end
|
25
|
+
#
|
26
|
+
# alias task_exports all_exports
|
27
|
+
#
|
28
|
+
# def unexport(*names)
|
29
|
+
# names = names.collect{|n| n.to_s} + names.collect{|n| n.to_sym}
|
30
|
+
# names.uniq!
|
31
|
+
# exec_exports.replace exec_exports - names if exec_exports
|
32
|
+
# synchronous_exports.replace synchronous_exports - names if synchronous_exports
|
33
|
+
# asynchronous_exports.replace asynchronous_exports - names if asynchronous_exports
|
34
|
+
# stream_exports.replace stream_exports - names if stream_exports
|
35
|
+
# end
|
36
|
+
#
|
37
|
+
# def export_exec(*names)
|
38
|
+
# unexport *names
|
39
|
+
# exec_exports.concat names
|
40
|
+
# exec_exports.uniq!
|
41
|
+
# exec_exports
|
42
|
+
# end
|
43
|
+
#
|
44
|
+
# def export_synchronous(*names)
|
45
|
+
# unexport *names
|
46
|
+
# synchronous_exports.concat names
|
47
|
+
# synchronous_exports.uniq!
|
48
|
+
# synchronous_exports
|
49
|
+
# end
|
50
|
+
#
|
51
|
+
# def export_asynchronous(*names)
|
52
|
+
# unexport *names
|
53
|
+
# asynchronous_exports.concat names
|
54
|
+
# asynchronous_exports.uniq!
|
55
|
+
# asynchronous_exports
|
56
|
+
# end
|
57
|
+
#
|
58
|
+
# def export_stream(*names)
|
59
|
+
# unexport *names
|
60
|
+
# stream_exports.concat names
|
61
|
+
# stream_exports.uniq!
|
62
|
+
# stream_exports
|
63
|
+
# end
|
64
|
+
#
|
65
|
+
# alias export export_asynchronous
|
66
|
+
#end
|
@@ -1,64 +1,64 @@
|
|
1
|
-
module Workflow
|
2
|
-
def rec_inputs(task_name)
|
3
|
-
tasks[task_name].recursive_inputs.collect{|name, _| name }
|
4
|
-
end
|
5
|
-
|
6
|
-
def rec_input_types(task_name)
|
7
|
-
tasks[task_name].recursive_inputs.inject({}) do |acc,l|
|
8
|
-
name, type, desc, default, options = l
|
9
|
-
acc.merge!(name => type) unless acc.include?(name)
|
10
|
-
acc
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
|
-
|
15
|
-
def rec_input_descriptions(task_name)
|
16
|
-
tasks[task_name].recursive_inputs.inject({}) do |acc,l|
|
17
|
-
name, type, desc, default, options = l
|
18
|
-
acc.merge!(name => desc) unless desc.nil? || acc.include?(name)
|
19
|
-
acc
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
def rec_input_defaults(task_name)
|
24
|
-
tasks[task_name].recursive_inputs.inject({}) do |acc,l|
|
25
|
-
name, type, desc, default, options = l
|
26
|
-
acc.merge!(name => default) unless default.nil? || acc.include?(name)
|
27
|
-
acc
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
def rec_input_options(task_name)
|
32
|
-
tasks[task_name].recursive_inputs.inject({}) do |acc,l|
|
33
|
-
name, type, desc, default, options = l
|
34
|
-
acc.merge!(name => options) unless options.nil? unless acc.include?(name)
|
35
|
-
acc
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
|
40
|
-
def rec_input_use(task_name)
|
41
|
-
input_use = {}
|
42
|
-
task = self.tasks[task_name]
|
43
|
-
task.inputs.each do |name,_|
|
44
|
-
input_use[name] ||= {}
|
45
|
-
input_use[name][self] ||= []
|
46
|
-
input_use[name][self] << task_name
|
47
|
-
end
|
48
|
-
|
49
|
-
task.deps.inject(input_use) do |acc,p|
|
50
|
-
workflow, task_name = p
|
51
|
-
next if task_name.nil?
|
52
|
-
workflow.rec_input_use(task_name).each do |name,uses|
|
53
|
-
acc[name] ||= {}
|
54
|
-
uses.each do |workflow, task_names|
|
55
|
-
acc[name][workflow] ||= []
|
56
|
-
acc[name][workflow].concat(task_names)
|
57
|
-
end
|
58
|
-
end
|
59
|
-
acc
|
60
|
-
end if task.deps
|
61
|
-
|
62
|
-
input_use
|
63
|
-
end
|
64
|
-
end
|
1
|
+
#module Workflow
|
2
|
+
# def rec_inputs(task_name)
|
3
|
+
# tasks[task_name].recursive_inputs.collect{|name, _| name }
|
4
|
+
# end
|
5
|
+
#
|
6
|
+
# def rec_input_types(task_name)
|
7
|
+
# tasks[task_name].recursive_inputs.inject({}) do |acc,l|
|
8
|
+
# name, type, desc, default, options = l
|
9
|
+
# acc.merge!(name => type) unless acc.include?(name)
|
10
|
+
# acc
|
11
|
+
# end
|
12
|
+
# end
|
13
|
+
#
|
14
|
+
#
|
15
|
+
# def rec_input_descriptions(task_name)
|
16
|
+
# tasks[task_name].recursive_inputs.inject({}) do |acc,l|
|
17
|
+
# name, type, desc, default, options = l
|
18
|
+
# acc.merge!(name => desc) unless desc.nil? || acc.include?(name)
|
19
|
+
# acc
|
20
|
+
# end
|
21
|
+
# end
|
22
|
+
#
|
23
|
+
# def rec_input_defaults(task_name)
|
24
|
+
# tasks[task_name].recursive_inputs.inject({}) do |acc,l|
|
25
|
+
# name, type, desc, default, options = l
|
26
|
+
# acc.merge!(name => default) unless default.nil? || acc.include?(name)
|
27
|
+
# acc
|
28
|
+
# end
|
29
|
+
# end
|
30
|
+
#
|
31
|
+
# def rec_input_options(task_name)
|
32
|
+
# tasks[task_name].recursive_inputs.inject({}) do |acc,l|
|
33
|
+
# name, type, desc, default, options = l
|
34
|
+
# acc.merge!(name => options) unless options.nil? unless acc.include?(name)
|
35
|
+
# acc
|
36
|
+
# end
|
37
|
+
# end
|
38
|
+
#
|
39
|
+
#
|
40
|
+
# def rec_input_use(task_name)
|
41
|
+
# input_use = {}
|
42
|
+
# task = self.tasks[task_name]
|
43
|
+
# task.inputs.each do |name,_|
|
44
|
+
# input_use[name] ||= {}
|
45
|
+
# input_use[name][self] ||= []
|
46
|
+
# input_use[name][self] << task_name
|
47
|
+
# end
|
48
|
+
#
|
49
|
+
# task.deps.inject(input_use) do |acc,p|
|
50
|
+
# workflow, task_name = p
|
51
|
+
# next if task_name.nil?
|
52
|
+
# workflow.rec_input_use(task_name).each do |name,uses|
|
53
|
+
# acc[name] ||= {}
|
54
|
+
# uses.each do |workflow, task_names|
|
55
|
+
# acc[name][workflow] ||= []
|
56
|
+
# acc[name][workflow].concat(task_names)
|
57
|
+
# end
|
58
|
+
# end
|
59
|
+
# acc
|
60
|
+
# end if task.deps
|
61
|
+
#
|
62
|
+
# input_use
|
63
|
+
# end
|
64
|
+
#end
|