rbbt-util 6.0.3 → 6.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE +1 -1
  3. data/bin/rbbt +1 -1
  4. data/bin/rbbt_exec.rb +2 -2
  5. data/lib/rbbt/hpc/batch.rb +1 -1
  6. data/lib/rbbt/knowledge_base/enrichment.rb +9 -9
  7. data/lib/rbbt/knowledge_base/entity.rb +128 -128
  8. data/lib/rbbt/knowledge_base/query.rb +94 -94
  9. data/lib/rbbt/knowledge_base/registry.rb +189 -189
  10. data/lib/rbbt/knowledge_base/syndicate.rb +26 -26
  11. data/lib/rbbt/knowledge_base/traverse.rb +315 -315
  12. data/lib/rbbt/knowledge_base.rb +37 -34
  13. data/lib/rbbt/tsv/excel.rb +1 -1
  14. data/lib/rbbt/util/filecache.rb +1 -1
  15. data/lib/rbbt/util/migrate.rb +4 -4
  16. data/lib/rbbt/util/misc/system.rb +92 -90
  17. data/lib/rbbt/workflow/refactor/export.rb +66 -66
  18. data/lib/rbbt/workflow/refactor/recursive.rb +64 -64
  19. data/lib/rbbt/workflow/refactor/task_info.rb +66 -65
  20. data/lib/rbbt/workflow/refactor.rb +0 -3
  21. data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +1 -2
  22. data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb +11 -11
  23. data/lib/rbbt/workflow/remote_workflow/remote_step.rb +1 -1
  24. data/lib/rbbt/workflow/remote_workflow.rb +2 -1
  25. data/lib/rbbt-util.rb +2 -2
  26. data/lib/rbbt.rb +1 -1
  27. data/python/rbbt/'/Users/miki/config/tmp/undodir'/%Users%miki%git%rbbt-util%python%rbbt%__init__.py +0 -0
  28. data/python/rbbt/__init__.py +78 -4
  29. data/python/rbbt/workflow/remote.py +104 -0
  30. data/python/rbbt/workflow.py +64 -0
  31. data/python/test.py +10 -0
  32. data/share/rbbt_commands/hpc/list +1 -1
  33. data/share/rbbt_commands/lsf/list +1 -1
  34. data/share/rbbt_commands/pbs/list +1 -1
  35. data/share/rbbt_commands/resource/find +1 -1
  36. data/share/rbbt_commands/slurm/list +1 -1
  37. data/share/rbbt_commands/system/status +2 -2
  38. data/share/rbbt_commands/workflow/info +1 -1
  39. data/share/rbbt_commands/workflow/retry +43 -0
  40. data/share/rbbt_commands/workflow/server +1 -1
  41. data/share/rbbt_commands/workflow/task +4 -2
  42. metadata +8 -6
@@ -1,35 +1,38 @@
1
- require 'rbbt/annotations'
2
- require 'rbbt/association'
3
- require 'rbbt/association/item'
4
- require 'rbbt/knowledge_base/entity'
5
- require 'rbbt/knowledge_base/query'
6
- require 'rbbt/knowledge_base/syndicate'
1
+ require_relative 'refactor'
2
+ Rbbt.require_instead 'scout/knowledge_base'
7
3
 
8
- class KnowledgeBase
9
-
10
- attr_accessor :namespace, :dir, :databases, :indices, :registry, :format, :entity_options
11
- def initialize(dir, namespace = nil)
12
- @dir = Path.setup(dir.dup)
13
-
14
- @namespace = namespace
15
- @format = IndiferentHash.setup({})
16
-
17
- @registry ||= IndiferentHash.setup({})
18
- @entity_options = IndiferentHash.setup({})
19
-
20
- @indices = IndiferentHash.setup({})
21
- @databases = IndiferentHash.setup({})
22
- @identifiers = IndiferentHash.setup({})
23
- @fields = {}
24
- @descriptions = {}
25
- @databases = {}
26
- end
27
-
28
- def self.load(dir)
29
- KnowledgeBase.new dir
30
- end
31
-
32
- def setup(name, matches, reverse = false)
33
- AssociationItem.setup matches, self, name, reverse
34
- end
35
- end
4
+ #require 'rbbt/annotations'
5
+ #require 'rbbt/association'
6
+ #require 'rbbt/association/item'
7
+ #require 'rbbt/knowledge_base/entity'
8
+ #require 'rbbt/knowledge_base/query'
9
+ #require 'rbbt/knowledge_base/syndicate'
10
+ #
11
+ #class KnowledgeBase
12
+ #
13
+ # attr_accessor :namespace, :dir, :databases, :indices, :registry, :format, :entity_options
14
+ # def initialize(dir, namespace = nil)
15
+ # @dir = Path.setup(dir.dup)
16
+ #
17
+ # @namespace = namespace
18
+ # @format = IndiferentHash.setup({})
19
+ #
20
+ # @registry ||= IndiferentHash.setup({})
21
+ # @entity_options = IndiferentHash.setup({})
22
+ #
23
+ # @indices = IndiferentHash.setup({})
24
+ # @databases = IndiferentHash.setup({})
25
+ # @identifiers = IndiferentHash.setup({})
26
+ # @fields = {}
27
+ # @descriptions = {}
28
+ # @databases = {}
29
+ # end
30
+ #
31
+ # def self.load(dir)
32
+ # KnowledgeBase.new dir
33
+ # end
34
+ #
35
+ # def setup(name, matches, reverse = false)
36
+ # AssociationItem.setup matches, self, name, reverse
37
+ # end
38
+ #end
@@ -1,5 +1,5 @@
1
- require_relative '../refactor'
2
1
  Rbbt.require_instead 'scout/tsv'
2
+ require_relative '../refactor'
3
3
  require 'spreadsheet'
4
4
  require 'rubyXL'
5
5
 
@@ -12,7 +12,7 @@ module FileCache
12
12
 
13
13
  def self.cachedir=(cachedir)
14
14
  CACHEDIR.replace cachedir
15
- FileUtils.mkdir_p CACHEDIR unless File.exist? CACHEDIR
15
+ Open.mkdir CACHEDIR unless Open.exist? CACHEDIR
16
16
  end
17
17
 
18
18
  def self.cachedir
@@ -2,16 +2,16 @@ module Rbbt
2
2
 
3
3
  def self.migrate_source_paths(path, resource = Rbbt, source = nil)
4
4
  if source
5
- lpath, *paths = Misc.ssh_run(source, <<-EOF).split("\n")
5
+ lpath, *paths = SSHLine.ruby(source, <<-EOF).split("\n")
6
6
  require 'rbbt-util'
7
7
  path = "#{path}"
8
8
  if Open.exists?(path)
9
- path = #{resource.to_s}.identify(path)
9
+ path = Path.setup(#{resource.to_s}.identify(path))
10
10
  else
11
11
  path = Path.setup(path)
12
12
  end
13
13
  puts path
14
- puts path.glob_all.collect{|p| File.directory?(p) ? p + "/" : p } * "\n"
14
+ puts path.find_all.collect{|p| File.directory?(p) ? p + "/" : p } * "\n"
15
15
  EOF
16
16
 
17
17
  [path, paths.collect{|p| [source, p] * ":"}, lpath]
@@ -36,7 +36,7 @@ puts path.glob_all.collect{|p| File.directory?(p) ? p + "/" : p } * "\n"
36
36
 
37
37
  def self.migrate_target_path(path, search_path = 'user', resource = Rbbt, target = nil)
38
38
  if target
39
- Misc.ssh_run(target, <<-EOF).split("\n").first
39
+ SSHLine.ruby(target, <<-EOF).split("\n").first
40
40
  require 'rbbt-util'
41
41
  path = "#{path}"
42
42
  resource = #{resource.to_s}
@@ -1,90 +1,92 @@
1
- module Misc
2
-
3
- def self.hostname
4
- @hostanem ||= `hostname`.strip
5
- end
6
-
7
- def self.pid_exists?(pid)
8
- return false if pid.nil?
9
- begin
10
- Process.getpgid(pid.to_i)
11
- true
12
- rescue Errno::ESRCH
13
- false
14
- end
15
- end
16
-
17
- def self.env_add(var, value, sep = ":", prepend = true)
18
- ENV[var] ||= ""
19
- return if ENV[var] =~ /(#{sep}|^)#{Regexp.quote value}(#{sep}|$)/
20
- if prepend
21
- ENV[var] = value + sep + ENV[var]
22
- else
23
- ENV[var] += sep + ENV[var]
24
- end
25
- end
26
-
27
- def self.with_env(var, value, &block)
28
- var = var.to_s
29
- value = value.to_s
30
- current = ENV[var]
31
- begin
32
- ENV[var] = value
33
- yield
34
- ensure
35
- ENV[var] = current
36
- end
37
- end
38
-
39
- def self.common_path(dir, file)
40
- file = File.expand_path file
41
- dir = File.expand_path dir
42
-
43
- return true if file == dir
44
- while File.dirname(file) != file
45
- file = File.dirname(file)
46
- return true if file == dir
47
- end
48
-
49
- return false
50
- end
51
-
52
-
53
- def self.relative_link(source, target_dir)
54
- path = "."
55
- current = target_dir
56
- while ! Misc.common_path current, source
57
- current = File.dirname(current)
58
- path = File.join(path, '..')
59
- return nil if current == "/"
60
- end
61
-
62
- File.join(path, Misc.path_relative_to(current, source))
63
- end
64
-
65
- # WARN: probably not thread safe...
66
- def self.in_dir(dir)
67
- old_pwd = FileUtils.pwd
68
- res = nil
69
- begin
70
- FileUtils.mkdir_p dir unless File.exist?(dir)
71
- FileUtils.cd dir
72
- res = yield
73
- ensure
74
- FileUtils.cd old_pwd
75
- end
76
- res
77
- end
78
-
79
- def self.is_filename?(string, need_to_exists = true)
80
- return false if string.nil?
81
- return true if defined? Path and Path === string
82
- return true if string.respond_to? :exists
83
- return true if String === string and ! string.include?("\n") and string.split("/").select{|p| p.length > 265}.empty? and (! need_to_exists || File.exist?(string))
84
- return false
85
- end
86
-
87
- class << self
88
- alias filename? is_filename?
89
- end
90
- end
1
+ require_relative 'refactor'
2
+ Rbbt.require_instead 'scout/misc/format'
3
+ #module Misc
4
+ #
5
+ # def self.hostname
6
+ # @hostanem ||= `hostname`.strip
7
+ # end
8
+ #
9
+ # def self.pid_exists?(pid)
10
+ # return false if pid.nil?
11
+ # begin
12
+ # Process.getpgid(pid.to_i)
13
+ # true
14
+ # rescue Errno::ESRCH
15
+ # false
16
+ # end
17
+ # end
18
+ #
19
+ # def self.env_add(var, value, sep = ":", prepend = true)
20
+ # ENV[var] ||= ""
21
+ # return if ENV[var] =~ /(#{sep}|^)#{Regexp.quote value}(#{sep}|$)/
22
+ # if prepend
23
+ # ENV[var] = value + sep + ENV[var]
24
+ # else
25
+ # ENV[var] += sep + ENV[var]
26
+ # end
27
+ # end
28
+ #
29
+ # def self.with_env(var, value, &block)
30
+ # var = var.to_s
31
+ # value = value.to_s
32
+ # current = ENV[var]
33
+ # begin
34
+ # ENV[var] = value
35
+ # yield
36
+ # ensure
37
+ # ENV[var] = current
38
+ # end
39
+ # end
40
+ #
41
+ # def self.common_path(dir, file)
42
+ # file = File.expand_path file
43
+ # dir = File.expand_path dir
44
+ #
45
+ # return true if file == dir
46
+ # while File.dirname(file) != file
47
+ # file = File.dirname(file)
48
+ # return true if file == dir
49
+ # end
50
+ #
51
+ # return false
52
+ # end
53
+ #
54
+ #
55
+ # def self.relative_link(source, target_dir)
56
+ # path = "."
57
+ # current = target_dir
58
+ # while ! Misc.common_path current, source
59
+ # current = File.dirname(current)
60
+ # path = File.join(path, '..')
61
+ # return nil if current == "/"
62
+ # end
63
+ #
64
+ # File.join(path, Misc.path_relative_to(current, source))
65
+ # end
66
+ #
67
+ # # WARN: probably not thread safe...
68
+ # def self.in_dir(dir)
69
+ # old_pwd = FileUtils.pwd
70
+ # res = nil
71
+ # begin
72
+ # FileUtils.mkdir_p dir unless File.exist?(dir)
73
+ # FileUtils.cd dir
74
+ # res = yield
75
+ # ensure
76
+ # FileUtils.cd old_pwd
77
+ # end
78
+ # res
79
+ # end
80
+ #
81
+ # def self.is_filename?(string, need_to_exists = true)
82
+ # return false if string.nil?
83
+ # return true if defined? Path and Path === string
84
+ # return true if string.respond_to? :exists
85
+ # return true if String === string and ! string.include?("\n") and string.split("/").select{|p| p.length > 265}.empty? and (! need_to_exists || File.exist?(string))
86
+ # return false
87
+ # end
88
+ #
89
+ # class << self
90
+ # alias filename? is_filename?
91
+ # end
92
+ #end
@@ -1,66 +1,66 @@
1
- module Workflow
2
-
3
- annotation :asynchronous_exports, :synchronous_exports, :exec_exports, :stream_exports
4
-
5
- def asynchronous_exports
6
- @asynchronous_exports ||= []
7
- end
8
-
9
- def synchronous_exports
10
- @synchronous_exports ||= []
11
- end
12
-
13
- def exec_exports
14
- @exec_exports ||= []
15
- end
16
-
17
- def stream_exports
18
- @exec_exports ||= []
19
- end
20
-
21
-
22
- def all_exports
23
- asynchronous_exports + synchronous_exports + exec_exports + stream_exports
24
- end
25
-
26
- alias task_exports all_exports
27
-
28
- def unexport(*names)
29
- names = names.collect{|n| n.to_s} + names.collect{|n| n.to_sym}
30
- names.uniq!
31
- exec_exports.replace exec_exports - names if exec_exports
32
- synchronous_exports.replace synchronous_exports - names if synchronous_exports
33
- asynchronous_exports.replace asynchronous_exports - names if asynchronous_exports
34
- stream_exports.replace stream_exports - names if stream_exports
35
- end
36
-
37
- def export_exec(*names)
38
- unexport *names
39
- exec_exports.concat names
40
- exec_exports.uniq!
41
- exec_exports
42
- end
43
-
44
- def export_synchronous(*names)
45
- unexport *names
46
- synchronous_exports.concat names
47
- synchronous_exports.uniq!
48
- synchronous_exports
49
- end
50
-
51
- def export_asynchronous(*names)
52
- unexport *names
53
- asynchronous_exports.concat names
54
- asynchronous_exports.uniq!
55
- asynchronous_exports
56
- end
57
-
58
- def export_stream(*names)
59
- unexport *names
60
- stream_exports.concat names
61
- stream_exports.uniq!
62
- stream_exports
63
- end
64
-
65
- alias export export_asynchronous
66
- end
1
+ #module Workflow
2
+ #
3
+ # annotation :asynchronous_exports, :synchronous_exports, :exec_exports, :stream_exports
4
+ #
5
+ # def asynchronous_exports
6
+ # @asynchronous_exports ||= []
7
+ # end
8
+ #
9
+ # def synchronous_exports
10
+ # @synchronous_exports ||= []
11
+ # end
12
+ #
13
+ # def exec_exports
14
+ # @exec_exports ||= []
15
+ # end
16
+ #
17
+ # def stream_exports
18
+ # @exec_exports ||= []
19
+ # end
20
+ #
21
+ #
22
+ # def all_exports
23
+ # asynchronous_exports + synchronous_exports + exec_exports + stream_exports
24
+ # end
25
+ #
26
+ # alias task_exports all_exports
27
+ #
28
+ # def unexport(*names)
29
+ # names = names.collect{|n| n.to_s} + names.collect{|n| n.to_sym}
30
+ # names.uniq!
31
+ # exec_exports.replace exec_exports - names if exec_exports
32
+ # synchronous_exports.replace synchronous_exports - names if synchronous_exports
33
+ # asynchronous_exports.replace asynchronous_exports - names if asynchronous_exports
34
+ # stream_exports.replace stream_exports - names if stream_exports
35
+ # end
36
+ #
37
+ # def export_exec(*names)
38
+ # unexport *names
39
+ # exec_exports.concat names
40
+ # exec_exports.uniq!
41
+ # exec_exports
42
+ # end
43
+ #
44
+ # def export_synchronous(*names)
45
+ # unexport *names
46
+ # synchronous_exports.concat names
47
+ # synchronous_exports.uniq!
48
+ # synchronous_exports
49
+ # end
50
+ #
51
+ # def export_asynchronous(*names)
52
+ # unexport *names
53
+ # asynchronous_exports.concat names
54
+ # asynchronous_exports.uniq!
55
+ # asynchronous_exports
56
+ # end
57
+ #
58
+ # def export_stream(*names)
59
+ # unexport *names
60
+ # stream_exports.concat names
61
+ # stream_exports.uniq!
62
+ # stream_exports
63
+ # end
64
+ #
65
+ # alias export export_asynchronous
66
+ #end
@@ -1,64 +1,64 @@
1
- module Workflow
2
- def rec_inputs(task_name)
3
- tasks[task_name].recursive_inputs.collect{|name, _| name }
4
- end
5
-
6
- def rec_input_types(task_name)
7
- tasks[task_name].recursive_inputs.inject({}) do |acc,l|
8
- name, type, desc, default, options = l
9
- acc.merge!(name => type) unless acc.include?(name)
10
- acc
11
- end
12
- end
13
-
14
-
15
- def rec_input_descriptions(task_name)
16
- tasks[task_name].recursive_inputs.inject({}) do |acc,l|
17
- name, type, desc, default, options = l
18
- acc.merge!(name => desc) unless desc.nil? || acc.include?(name)
19
- acc
20
- end
21
- end
22
-
23
- def rec_input_defaults(task_name)
24
- tasks[task_name].recursive_inputs.inject({}) do |acc,l|
25
- name, type, desc, default, options = l
26
- acc.merge!(name => default) unless default.nil? || acc.include?(name)
27
- acc
28
- end
29
- end
30
-
31
- def rec_input_options(task_name)
32
- tasks[task_name].recursive_inputs.inject({}) do |acc,l|
33
- name, type, desc, default, options = l
34
- acc.merge!(name => options) unless options.nil? unless acc.include?(name)
35
- acc
36
- end
37
- end
38
-
39
-
40
- def rec_input_use(task_name)
41
- input_use = {}
42
- task = self.tasks[task_name]
43
- task.inputs.each do |name,_|
44
- input_use[name] ||= {}
45
- input_use[name][self] ||= []
46
- input_use[name][self] << task_name
47
- end
48
-
49
- task.deps.inject(input_use) do |acc,p|
50
- workflow, task_name = p
51
- next if task_name.nil?
52
- workflow.rec_input_use(task_name).each do |name,uses|
53
- acc[name] ||= {}
54
- uses.each do |workflow, task_names|
55
- acc[name][workflow] ||= []
56
- acc[name][workflow].concat(task_names)
57
- end
58
- end
59
- acc
60
- end if task.deps
61
-
62
- input_use
63
- end
64
- end
1
+ #module Workflow
2
+ # def rec_inputs(task_name)
3
+ # tasks[task_name].recursive_inputs.collect{|name, _| name }
4
+ # end
5
+ #
6
+ # def rec_input_types(task_name)
7
+ # tasks[task_name].recursive_inputs.inject({}) do |acc,l|
8
+ # name, type, desc, default, options = l
9
+ # acc.merge!(name => type) unless acc.include?(name)
10
+ # acc
11
+ # end
12
+ # end
13
+ #
14
+ #
15
+ # def rec_input_descriptions(task_name)
16
+ # tasks[task_name].recursive_inputs.inject({}) do |acc,l|
17
+ # name, type, desc, default, options = l
18
+ # acc.merge!(name => desc) unless desc.nil? || acc.include?(name)
19
+ # acc
20
+ # end
21
+ # end
22
+ #
23
+ # def rec_input_defaults(task_name)
24
+ # tasks[task_name].recursive_inputs.inject({}) do |acc,l|
25
+ # name, type, desc, default, options = l
26
+ # acc.merge!(name => default) unless default.nil? || acc.include?(name)
27
+ # acc
28
+ # end
29
+ # end
30
+ #
31
+ # def rec_input_options(task_name)
32
+ # tasks[task_name].recursive_inputs.inject({}) do |acc,l|
33
+ # name, type, desc, default, options = l
34
+ # acc.merge!(name => options) unless options.nil? unless acc.include?(name)
35
+ # acc
36
+ # end
37
+ # end
38
+ #
39
+ #
40
+ # def rec_input_use(task_name)
41
+ # input_use = {}
42
+ # task = self.tasks[task_name]
43
+ # task.inputs.each do |name,_|
44
+ # input_use[name] ||= {}
45
+ # input_use[name][self] ||= []
46
+ # input_use[name][self] << task_name
47
+ # end
48
+ #
49
+ # task.deps.inject(input_use) do |acc,p|
50
+ # workflow, task_name = p
51
+ # next if task_name.nil?
52
+ # workflow.rec_input_use(task_name).each do |name,uses|
53
+ # acc[name] ||= {}
54
+ # uses.each do |workflow, task_names|
55
+ # acc[name][workflow] ||= []
56
+ # acc[name][workflow].concat(task_names)
57
+ # end
58
+ # end
59
+ # acc
60
+ # end if task.deps
61
+ #
62
+ # input_use
63
+ # end
64
+ #end