rbbt-util 5.28.5 → 5.28.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/entity.rb +1 -1
- data/lib/rbbt/fix_width_table.rb +5 -4
- data/lib/rbbt/persist.rb +1 -0
- data/lib/rbbt/persist/tsv/adapter.rb +0 -1
- data/lib/rbbt/persist/tsv/fix_width_table.rb +5 -3
- data/lib/rbbt/tsv/accessor.rb +10 -2
- data/lib/rbbt/tsv/dumper.rb +14 -2
- data/lib/rbbt/tsv/parallel/traverse.rb +3 -0
- data/lib/rbbt/tsv/util.rb +5 -1
- data/lib/rbbt/util/cmd.rb +1 -0
- data/lib/rbbt/util/config.rb +2 -1
- data/lib/rbbt/util/misc/bgzf.rb +1 -1
- data/lib/rbbt/util/misc/inspect.rb +1 -1
- data/lib/rbbt/util/misc/system.rb +1 -1
- data/lib/rbbt/util/named_array.rb +1 -1
- data/lib/rbbt/util/open.rb +18 -17
- data/lib/rbbt/workflow.rb +1 -0
- data/lib/rbbt/workflow/accessor.rb +94 -93
- data/lib/rbbt/workflow/definition.rb +8 -4
- data/lib/rbbt/workflow/integration/ansible.rb +53 -0
- data/lib/rbbt/workflow/integration/ansible/workflow.rb +60 -0
- data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +5 -1
- data/lib/rbbt/workflow/step.rb +22 -5
- data/lib/rbbt/workflow/step/accessor.rb +7 -5
- data/lib/rbbt/workflow/usage.rb +1 -1
- data/lib/rbbt/workflow/util/archive.rb +3 -0
- data/lib/rbbt/workflow/util/orchestrator.rb +228 -0
- data/lib/rbbt/workflow/util/trace.rb +182 -0
- data/share/rbbt_commands/ansible +55 -0
- data/share/rbbt_commands/purge_job +2 -5
- data/share/rbbt_commands/system/status +1 -1
- data/share/rbbt_commands/workflow/forget_deps +10 -3
- data/share/rbbt_commands/workflow/server +2 -0
- data/test/rbbt/association/test_index.rb +6 -6
- data/test/rbbt/knowledge_base/test_query.rb +3 -3
- data/test/rbbt/knowledge_base/test_registry.rb +1 -1
- data/test/rbbt/persist/tsv/test_cdb.rb +0 -7
- data/test/rbbt/persist/tsv/test_kyotocabinet.rb +2 -8
- data/test/rbbt/persist/tsv/test_leveldb.rb +0 -6
- data/test/rbbt/persist/tsv/test_lmdb.rb +0 -6
- data/test/rbbt/persist/tsv/test_tokyocabinet.rb +15 -14
- data/test/rbbt/test_entity.rb +0 -1
- data/test/rbbt/test_knowledge_base.rb +3 -4
- data/test/rbbt/test_persist.rb +10 -6
- data/test/rbbt/test_workflow.rb +17 -16
- data/test/rbbt/tsv/parallel/test_traverse.rb +14 -0
- data/test/rbbt/tsv/test_accessor.rb +11 -0
- data/test/rbbt/tsv/test_attach.rb +0 -2
- data/test/rbbt/tsv/test_index.rb +6 -7
- data/test/rbbt/tsv/test_manipulate.rb +22 -3
- data/test/rbbt/util/R/test_model.rb +2 -1
- data/test/rbbt/util/R/test_plot.rb +0 -2
- data/test/rbbt/util/concurrency/test_processes.rb +1 -1
- data/test/rbbt/util/misc/test_bgzf.rb +11 -7
- data/test/rbbt/util/misc/test_lock.rb +0 -1
- data/test/rbbt/util/misc/test_multipart_payload.rb +1 -1
- data/test/rbbt/util/misc/test_pipes.rb +0 -5
- data/test/rbbt/util/test_R.rb +1 -0
- data/test/rbbt/util/test_log.rb +4 -6
- data/test/rbbt/util/test_misc.rb +0 -2
- data/test/rbbt/util/test_open.rb +0 -1
- data/test/rbbt/util/test_python.rb +17 -1
- data/test/rbbt/workflow/test_remote_workflow.rb +1 -1
- data/test/rbbt/workflow/test_schedule.rb +0 -0
- data/test/rbbt/workflow/test_step.rb +8 -3
- data/test/rbbt/workflow/util/test_orchestrator.rb +273 -0
- metadata +11 -5
- data/lib/rbbt/workflow/schedule.rb +0 -238
- data/test/rbbt/workflow/remote/test_client.rb +0 -56
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.28.
|
4
|
+
version: 5.28.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-11-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -318,6 +318,8 @@ files:
|
|
318
318
|
- lib/rbbt/workflow/definition.rb
|
319
319
|
- lib/rbbt/workflow/doc.rb
|
320
320
|
- lib/rbbt/workflow/examples.rb
|
321
|
+
- lib/rbbt/workflow/integration/ansible.rb
|
322
|
+
- lib/rbbt/workflow/integration/ansible/workflow.rb
|
321
323
|
- lib/rbbt/workflow/integration/cromwell.rb
|
322
324
|
- lib/rbbt/workflow/integration/nextflow.rb
|
323
325
|
- lib/rbbt/workflow/remote_workflow.rb
|
@@ -327,7 +329,6 @@ files:
|
|
327
329
|
- lib/rbbt/workflow/remote_workflow/remote_step.rb
|
328
330
|
- lib/rbbt/workflow/remote_workflow/remote_step/rest.rb
|
329
331
|
- lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb
|
330
|
-
- lib/rbbt/workflow/schedule.rb
|
331
332
|
- lib/rbbt/workflow/soap.rb
|
332
333
|
- lib/rbbt/workflow/step.rb
|
333
334
|
- lib/rbbt/workflow/step/accessor.rb
|
@@ -337,7 +338,9 @@ files:
|
|
337
338
|
- lib/rbbt/workflow/task.rb
|
338
339
|
- lib/rbbt/workflow/usage.rb
|
339
340
|
- lib/rbbt/workflow/util/archive.rb
|
341
|
+
- lib/rbbt/workflow/util/orchestrator.rb
|
340
342
|
- lib/rbbt/workflow/util/provenance.rb
|
343
|
+
- lib/rbbt/workflow/util/trace.rb
|
341
344
|
- share/Rlib/plot.R
|
342
345
|
- share/Rlib/svg.R
|
343
346
|
- share/Rlib/util.R
|
@@ -346,6 +349,7 @@ files:
|
|
346
349
|
- share/config.ru
|
347
350
|
- share/install/software/lib/install_helpers
|
348
351
|
- share/rbbt_commands/alias
|
352
|
+
- share/rbbt_commands/ansible
|
349
353
|
- share/rbbt_commands/app/install
|
350
354
|
- share/rbbt_commands/app/start
|
351
355
|
- share/rbbt_commands/app/template
|
@@ -512,12 +516,13 @@ files:
|
|
512
516
|
- test/rbbt/util/test_simpleDSL.rb
|
513
517
|
- test/rbbt/util/test_simpleopt.rb
|
514
518
|
- test/rbbt/util/test_tmpfile.rb
|
515
|
-
- test/rbbt/workflow/remote/test_client.rb
|
516
519
|
- test/rbbt/workflow/step/test_dependencies.rb
|
517
520
|
- test/rbbt/workflow/test_doc.rb
|
518
521
|
- test/rbbt/workflow/test_remote_workflow.rb
|
522
|
+
- test/rbbt/workflow/test_schedule.rb
|
519
523
|
- test/rbbt/workflow/test_step.rb
|
520
524
|
- test/rbbt/workflow/test_task.rb
|
525
|
+
- test/rbbt/workflow/util/test_orchestrator.rb
|
521
526
|
- test/test_helper.rb
|
522
527
|
homepage: http://github.com/mikisvaz/rbbt-util
|
523
528
|
licenses:
|
@@ -545,9 +550,10 @@ summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
|
|
545
550
|
test_files:
|
546
551
|
- test/rbbt/test_entity.rb
|
547
552
|
- test/rbbt/workflow/test_remote_workflow.rb
|
553
|
+
- test/rbbt/workflow/util/test_orchestrator.rb
|
548
554
|
- test/rbbt/workflow/test_doc.rb
|
555
|
+
- test/rbbt/workflow/test_schedule.rb
|
549
556
|
- test/rbbt/workflow/test_step.rb
|
550
|
-
- test/rbbt/workflow/remote/test_client.rb
|
551
557
|
- test/rbbt/workflow/step/test_dependencies.rb
|
552
558
|
- test/rbbt/workflow/test_task.rb
|
553
559
|
- test/rbbt/resource/test_path.rb
|
@@ -1,238 +0,0 @@
|
|
1
|
-
class Step
|
2
|
-
class ResourceManager
|
3
|
-
class NotEnoughResources < Exception
|
4
|
-
end
|
5
|
-
|
6
|
-
attr_accessor :cpus, :memory
|
7
|
-
def initialize(cpus = nil, memory = nil)
|
8
|
-
@cpus = cpus
|
9
|
-
@memory = memory
|
10
|
-
@sem_file = "ResourceManager-" + rand(10000).to_s
|
11
|
-
@semaphore = RbbtSemaphore.create_semaphore(@sem_file, 1)
|
12
|
-
end
|
13
|
-
|
14
|
-
def allocate(cpus = nil, memory = nil, &block)
|
15
|
-
RbbtSemaphore.synchronize(@semaphore) do
|
16
|
-
if (@cpus && cpus && @cpus < cups) ||
|
17
|
-
(@memory && memory && @memory < memory)
|
18
|
-
raise NotEnoughResources
|
19
|
-
end
|
20
|
-
begin
|
21
|
-
@cpus -= cpus
|
22
|
-
@memory -= memory
|
23
|
-
yield
|
24
|
-
rescue
|
25
|
-
@cpus += cpus
|
26
|
-
@memory += memory
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
def finalize(manager)
|
32
|
-
RbbtSemaphore.delete_semaphore(@sem_file)
|
33
|
-
end
|
34
|
-
|
35
|
-
def self.finalize(manager)
|
36
|
-
proc { manager.finalize }
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
class Scheduler
|
41
|
-
attr_accessor :jobs, :cpus, :dep_jobs, :job_deps, :jobps
|
42
|
-
def initialize(jobs, cpus)
|
43
|
-
@jobs = jobs
|
44
|
-
@cpus = cpus
|
45
|
-
|
46
|
-
@job_deps = {}
|
47
|
-
|
48
|
-
with_deps = jobs.dup
|
49
|
-
@dep_jobs = {}
|
50
|
-
@job_deps = {}
|
51
|
-
@jobps = {}
|
52
|
-
@missing = Set.new
|
53
|
-
while with_deps.any?
|
54
|
-
job = with_deps.pop
|
55
|
-
@jobps[job.path] = job
|
56
|
-
@missing << job.path unless job.done?
|
57
|
-
|
58
|
-
jdeps = job.dependencies
|
59
|
-
jdeps += job.inputs.flatten.select{|i| Step === i}
|
60
|
-
|
61
|
-
jdeps.reject!{|dep| dep.done? }
|
62
|
-
@job_deps[job.path] = []
|
63
|
-
jdeps.each do |dep|
|
64
|
-
#next if dep.done?
|
65
|
-
@dep_jobs[dep.path] ||= []
|
66
|
-
@job_deps[job.path] << dep.path
|
67
|
-
@dep_jobs[dep.path] << job.path
|
68
|
-
with_deps << dep unless @job_deps.include? dep.path
|
69
|
-
end
|
70
|
-
end
|
71
|
-
|
72
|
-
def ready
|
73
|
-
@job_deps.select do |jobp,deps|
|
74
|
-
(@missing & deps).empty?
|
75
|
-
end.collect{|jobp,deps| jobp}
|
76
|
-
end
|
77
|
-
|
78
|
-
def used
|
79
|
-
iii @dep_jobs
|
80
|
-
@dep_jobs.select do |dep,jobs|
|
81
|
-
iif [dep, @missing.to_a, jobs]
|
82
|
-
(@missing & jobs).empty?
|
83
|
-
end.keys
|
84
|
-
end
|
85
|
-
|
86
|
-
def next
|
87
|
-
priorities = {}
|
88
|
-
@jobs.each do |job|
|
89
|
-
priorities[job.path] = 1
|
90
|
-
end
|
91
|
-
|
92
|
-
@missing.each do |jobp|
|
93
|
-
end
|
94
|
-
|
95
|
-
ready.first
|
96
|
-
end
|
97
|
-
end
|
98
|
-
end
|
99
|
-
|
100
|
-
#def self._priorities(jobs)
|
101
|
-
# job_level = {}
|
102
|
-
# jobs.each do |job|
|
103
|
-
# job_level[job.path] = 1.0
|
104
|
-
# end
|
105
|
-
|
106
|
-
# with_deps = jobs.dup
|
107
|
-
# dep_jobs = {}
|
108
|
-
# job_deps = {}
|
109
|
-
# while with_deps.any?
|
110
|
-
# job = with_deps.pop
|
111
|
-
# level = job_level[job.path]
|
112
|
-
# job_deps[job.path] = []
|
113
|
-
# jdeps = job.dependencies
|
114
|
-
# jdeps += job.inputs.flatten.select{|i| Step === i}
|
115
|
-
|
116
|
-
# jdeps.reject!{|dep| dep.done? }
|
117
|
-
# jdeps.each do |dep|
|
118
|
-
# next if dep.done?
|
119
|
-
# dep_jobs[dep.path] ||= []
|
120
|
-
# job_level[dep.path] = level / (10 * jdeps.length) if job_level[dep.path].nil? || job_level[dep.path] < level / (10 * jdeps.length)
|
121
|
-
# job_deps[job.path] << dep.path
|
122
|
-
# dep_jobs[dep.path] << job.path
|
123
|
-
# with_deps << dep unless job_deps.include? dep.path
|
124
|
-
# end
|
125
|
-
# end
|
126
|
-
# [job_level, job_deps, dep_jobs]
|
127
|
-
#end
|
128
|
-
|
129
|
-
#def self.produce_jobs(jobs, cpus, step_cpus = {})
|
130
|
-
# require 'fc'
|
131
|
-
|
132
|
-
# step_cpus = IndiferentHash.setup(step_cpus || {})
|
133
|
-
|
134
|
-
# deps = []
|
135
|
-
|
136
|
-
# jobs = [jobs] unless Array === jobs
|
137
|
-
|
138
|
-
# job_level, job_deps, dep_jobs = self._priorities(jobs)
|
139
|
-
|
140
|
-
# jobps = {}
|
141
|
-
# (jobs + jobs.collect{|job| job.rec_dependencies}).flatten.uniq.each do |job|
|
142
|
-
# jobps[job.path] = job
|
143
|
-
# end
|
144
|
-
|
145
|
-
# prio_queue = FastContainers::PriorityQueue.new :max
|
146
|
-
|
147
|
-
# job_deps.each do |jobp,depps|
|
148
|
-
# next if depps.any?
|
149
|
-
# level = job_level[jobp]
|
150
|
-
|
151
|
-
# prio_queue.push(jobp, level)
|
152
|
-
# end
|
153
|
-
|
154
|
-
# queue = RbbtProcessQueue.new cpus
|
155
|
-
|
156
|
-
# missing = job_deps.keys
|
157
|
-
# queue.callback do |jobp|
|
158
|
-
# Log.info "Done: #{jobp}"
|
159
|
-
# missing -= [jobp]
|
160
|
-
|
161
|
-
# job_level, job_deps, dep_jobs = self._priorities(jobs)
|
162
|
-
|
163
|
-
# parentsp = dep_jobs[jobp]
|
164
|
-
|
165
|
-
# parentsp.each do |parentp|
|
166
|
-
# next unless job_deps[parentp].include? jobp
|
167
|
-
# job_deps[parentp] -= [jobp]
|
168
|
-
# if job_deps[parentp].empty?
|
169
|
-
# level = job_level[parentp]
|
170
|
-
# prio_queue.push(parentp, level )
|
171
|
-
# end
|
172
|
-
# end if parentsp
|
173
|
-
# prio_queue_new = FastContainers::PriorityQueue.new :max
|
174
|
-
# while prio_queue.any?
|
175
|
-
# elem = prio_queue.pop
|
176
|
-
# prio_queue_new.push(elem, job_level[elem])
|
177
|
-
# end
|
178
|
-
# prio_queue = prio_queue_new
|
179
|
-
# end
|
180
|
-
#
|
181
|
-
# queue.init do |jobp|
|
182
|
-
# Log.info "Processing: #{jobp}"
|
183
|
-
# job = jobps[jobp]
|
184
|
-
# job_cpus = step_cpus[job.task_name] || 1
|
185
|
-
# sleep 0.5
|
186
|
-
# #job.produce
|
187
|
-
# jobp
|
188
|
-
# end
|
189
|
-
|
190
|
-
# while missing.any?
|
191
|
-
# while prio_queue.empty? && missing.any?
|
192
|
-
# sleep 1
|
193
|
-
# end
|
194
|
-
# break if missing.empty?
|
195
|
-
# jobp = prio_queue.pop
|
196
|
-
# queue.process jobp
|
197
|
-
# end
|
198
|
-
|
199
|
-
# queue.join
|
200
|
-
#end
|
201
|
-
end
|
202
|
-
|
203
|
-
|
204
|
-
if __FILE__ == $0
|
205
|
-
require 'rbbt/workflow'
|
206
|
-
|
207
|
-
module TestWF
|
208
|
-
extend Workflow
|
209
|
-
input :num, :integer
|
210
|
-
task :dep => :integer do |num|
|
211
|
-
num
|
212
|
-
end
|
213
|
-
dep :dep, :num => 1
|
214
|
-
dep :dep, :num => 2
|
215
|
-
dep :dep, :num => 3
|
216
|
-
task :test do
|
217
|
-
dependencies.collect{|d| d.load.to_s} * ","
|
218
|
-
end
|
219
|
-
end
|
220
|
-
Log.severity = 0
|
221
|
-
job = TestWF.job(:test)
|
222
|
-
job.recursive_clean
|
223
|
-
|
224
|
-
Rbbt::Config.load_file Rbbt.etc.config_profile.HTS.find
|
225
|
-
Workflow.require_workflow "Sample"
|
226
|
-
Workflow.require_workflow "HTS"
|
227
|
-
|
228
|
-
jobs = []
|
229
|
-
# jobs << Sample.job(:mutect2, "QUINTANA-15")
|
230
|
-
# jobs << Sample.job(:mutect2, "QUINTANA-25")
|
231
|
-
# jobs << Sample.job(:mutect2, "QUINTANA-22")
|
232
|
-
jobs << Sample.job(:mutect2, "small")
|
233
|
-
|
234
|
-
sched = Step::Scheduler.new(jobs, 3)
|
235
|
-
iii sched.ready
|
236
|
-
iii sched.used
|
237
|
-
iii sched.next
|
238
|
-
end
|
@@ -1,56 +0,0 @@
|
|
1
|
-
require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
|
2
|
-
require 'rbbt/workflow'
|
3
|
-
require 'rbbt/workflow/remote/client'
|
4
|
-
|
5
|
-
class TestWFRest
|
6
|
-
extend Workflow
|
7
|
-
|
8
|
-
input :name, :string, "Name to greet", "World"
|
9
|
-
task :hi => :string do |name|
|
10
|
-
"Hello #{name}"
|
11
|
-
end
|
12
|
-
|
13
|
-
dep :hi
|
14
|
-
task :intro => :string do
|
15
|
-
step(:hi).load + ", nice to meet you"
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
class TestRemote < Test::Unit::TestCase
|
20
|
-
|
21
|
-
def test_ssh
|
22
|
-
Log.severity = 0
|
23
|
-
client = WorkflowRemoteClient.new "ssh://turbo:Translation", "Translation"
|
24
|
-
job = client.job("translate", "SSH-TEST", :genes => ["TP53","KRAS"])
|
25
|
-
assert_equal 2, job.run.select{|l| l =~ /ENSG/}.length
|
26
|
-
end
|
27
|
-
|
28
|
-
def test_rest
|
29
|
-
Log.with_severity 0 do
|
30
|
-
|
31
|
-
workflow_server(TestWFRest) do |client|
|
32
|
-
assert_equal "Hello World", client.job(:hi, nil, {}).run.chomp
|
33
|
-
assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run.chomp
|
34
|
-
assert_equal "Hello Miguel, nice to meet you", client.job(:intro, nil, {:name => :Miguel}).run.chomp
|
35
|
-
end
|
36
|
-
|
37
|
-
workflow_server(TestWFRest, :Port => 1902) do |client|
|
38
|
-
assert_equal "Hello World", client.job(:hi, nil, {}).run.chomp
|
39
|
-
assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run.chomp
|
40
|
-
assert_equal "Hello Miguel, nice to meet you", client.job(:intro, nil, {:name => :Miguel}).run.chomp
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
def test_rest_clean
|
46
|
-
real_job = TestWFRest.job(:hi)
|
47
|
-
real_job.run
|
48
|
-
first_time = File.ctime(real_job.path)
|
49
|
-
workflow_server(TestWFRest) do |client|
|
50
|
-
assert File.ctime(real_job.path) == first_time
|
51
|
-
assert_equal "Hello World", client.job(:hi, nil, {}).clean.run
|
52
|
-
assert File.ctime(real_job.path) > first_time
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
56
|
-
|