rbbt-util 5.42.0 → 5.44.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (143) hide show
  1. checksums.yaml +4 -4
  2. data/lib/rbbt/annotations/util.rb +1 -1
  3. data/lib/rbbt/association/database.rb +2 -1
  4. data/lib/rbbt/association/index.rb +1 -0
  5. data/lib/rbbt/association/util.rb +14 -4
  6. data/lib/rbbt/entity.rb +6 -5
  7. data/lib/rbbt/hpc/batch.rb +19 -17
  8. data/lib/rbbt/hpc/orchestrate/batches.rb +1 -1
  9. data/lib/rbbt/hpc/orchestrate/chains.rb +3 -4
  10. data/lib/rbbt/knowledge_base/traverse.rb +24 -16
  11. data/lib/rbbt/persist/tsv/tokyocabinet.rb +1 -1
  12. data/lib/rbbt/resource/path.rb +7 -2
  13. data/lib/rbbt/resource.rb +1 -1
  14. data/lib/rbbt/tsv/accessor.rb +6 -11
  15. data/lib/rbbt/tsv/dumper.rb +21 -10
  16. data/lib/rbbt/tsv/index.rb +2 -1
  17. data/lib/rbbt/tsv/parallel/traverse.rb +44 -0
  18. data/lib/rbbt/tsv/parser.rb +7 -6
  19. data/lib/rbbt/tsv/serializers.rb +1 -1
  20. data/lib/rbbt/tsv/util.rb +12 -0
  21. data/lib/rbbt/util/log/progress.rb +1 -1
  22. data/lib/rbbt/util/misc/bgzf.rb +5 -4
  23. data/lib/rbbt/util/misc/development.rb +4 -4
  24. data/lib/rbbt/util/misc/math.rb +0 -1
  25. data/lib/rbbt/util/misc.rb +1 -1
  26. data/lib/rbbt/util/open.rb +1 -1
  27. data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +8 -3
  28. data/lib/rbbt/workflow/remote_workflow/remote_step.rb +4 -0
  29. data/lib/rbbt/workflow/step/info.rb +1 -0
  30. data/lib/rbbt/workflow/step.rb +2 -1
  31. data/lib/rbbt/workflow/util/trace.rb +2 -2
  32. data/lib/rbbt/workflow.rb +2 -2
  33. data/python/rbbt/__init__.py +2 -2
  34. data/share/rbbt_commands/workflow/task +1 -1
  35. metadata +4 -220
  36. data/test/rbbt/annotations/test_util.rb +0 -43
  37. data/test/rbbt/association/test_database.rb +0 -89
  38. data/test/rbbt/association/test_index.rb +0 -127
  39. data/test/rbbt/association/test_item.rb +0 -15
  40. data/test/rbbt/association/test_open.rb +0 -63
  41. data/test/rbbt/association/test_util.rb +0 -108
  42. data/test/rbbt/entity/test_identifiers.rb +0 -74
  43. data/test/rbbt/hpc/orchestrate/test_batches.rb +0 -113
  44. data/test/rbbt/hpc/orchestrate/test_chains.rb +0 -139
  45. data/test/rbbt/hpc/orchestrate/test_rules.rb +0 -92
  46. data/test/rbbt/hpc/test_batch.rb +0 -68
  47. data/test/rbbt/hpc/test_orchestrate.rb +0 -144
  48. data/test/rbbt/hpc/test_pbs.rb +0 -43
  49. data/test/rbbt/hpc/test_slurm.rb +0 -32
  50. data/test/rbbt/knowledge_base/test_enrichment.rb +0 -48
  51. data/test/rbbt/knowledge_base/test_entity.rb +0 -42
  52. data/test/rbbt/knowledge_base/test_query.rb +0 -46
  53. data/test/rbbt/knowledge_base/test_registry.rb +0 -81
  54. data/test/rbbt/knowledge_base/test_syndicate.rb +0 -48
  55. data/test/rbbt/knowledge_base/test_traverse.rb +0 -85
  56. data/test/rbbt/persist/test_tsv.rb +0 -88
  57. data/test/rbbt/persist/tsv/test_cdb.rb +0 -18
  58. data/test/rbbt/persist/tsv/test_kyotocabinet.rb +0 -27
  59. data/test/rbbt/persist/tsv/test_leveldb.rb +0 -18
  60. data/test/rbbt/persist/tsv/test_lmdb.rb +0 -18
  61. data/test/rbbt/persist/tsv/test_sharder.rb +0 -164
  62. data/test/rbbt/persist/tsv/test_tokyocabinet.rb +0 -262
  63. data/test/rbbt/resource/test_path.rb +0 -49
  64. data/test/rbbt/test_annotations.rb +0 -167
  65. data/test/rbbt/test_association.rb +0 -103
  66. data/test/rbbt/test_entity.rb +0 -249
  67. data/test/rbbt/test_fix_width_table.rb +0 -135
  68. data/test/rbbt/test_hpc.rb +0 -16
  69. data/test/rbbt/test_knowledge_base.rb +0 -149
  70. data/test/rbbt/test_monitor.rb +0 -11
  71. data/test/rbbt/test_packed_index.rb +0 -68
  72. data/test/rbbt/test_persist.rb +0 -85
  73. data/test/rbbt/test_resource.rb +0 -110
  74. data/test/rbbt/test_tsv.rb +0 -669
  75. data/test/rbbt/test_workflow.rb +0 -604
  76. data/test/rbbt/tsv/parallel/test_through.rb +0 -40
  77. data/test/rbbt/tsv/parallel/test_traverse.rb +0 -456
  78. data/test/rbbt/tsv/test_accessor.rb +0 -315
  79. data/test/rbbt/tsv/test_attach.rb +0 -718
  80. data/test/rbbt/tsv/test_change_id.rb +0 -61
  81. data/test/rbbt/tsv/test_csv.rb +0 -49
  82. data/test/rbbt/tsv/test_excel.rb +0 -171
  83. data/test/rbbt/tsv/test_field_index.rb +0 -19
  84. data/test/rbbt/tsv/test_filter.rb +0 -187
  85. data/test/rbbt/tsv/test_index.rb +0 -302
  86. data/test/rbbt/tsv/test_manipulate.rb +0 -337
  87. data/test/rbbt/tsv/test_marshal.rb +0 -24
  88. data/test/rbbt/tsv/test_matrix.rb +0 -20
  89. data/test/rbbt/tsv/test_parallel.rb +0 -7
  90. data/test/rbbt/tsv/test_parser.rb +0 -101
  91. data/test/rbbt/tsv/test_stream.rb +0 -253
  92. data/test/rbbt/tsv/test_util.rb +0 -52
  93. data/test/rbbt/util/R/test_eval.rb +0 -43
  94. data/test/rbbt/util/R/test_model.rb +0 -130
  95. data/test/rbbt/util/R/test_plot.rb +0 -38
  96. data/test/rbbt/util/concurrency/processes/test_socket.rb +0 -70
  97. data/test/rbbt/util/concurrency/test_processes.rb +0 -192
  98. data/test/rbbt/util/concurrency/test_threads.rb +0 -40
  99. data/test/rbbt/util/log/test_progress.rb +0 -111
  100. data/test/rbbt/util/misc/test_bgzf.rb +0 -48
  101. data/test/rbbt/util/misc/test_communication.rb +0 -11
  102. data/test/rbbt/util/misc/test_development.rb +0 -26
  103. data/test/rbbt/util/misc/test_format.rb +0 -10
  104. data/test/rbbt/util/misc/test_indiferent_hash.rb +0 -14
  105. data/test/rbbt/util/misc/test_lock.rb +0 -77
  106. data/test/rbbt/util/misc/test_multipart_payload.rb +0 -202
  107. data/test/rbbt/util/misc/test_omics.rb +0 -116
  108. data/test/rbbt/util/misc/test_pipes.rb +0 -343
  109. data/test/rbbt/util/misc/test_serialize.rb +0 -24
  110. data/test/rbbt/util/python/test_util.rb +0 -25
  111. data/test/rbbt/util/simpleopt/test_get.rb +0 -12
  112. data/test/rbbt/util/simpleopt/test_parse.rb +0 -10
  113. data/test/rbbt/util/simpleopt/test_setup.rb +0 -76
  114. data/test/rbbt/util/test_R.rb +0 -37
  115. data/test/rbbt/util/test_chain_methods.rb +0 -22
  116. data/test/rbbt/util/test_cmd.rb +0 -87
  117. data/test/rbbt/util/test_colorize.rb +0 -22
  118. data/test/rbbt/util/test_concurrency.rb +0 -6
  119. data/test/rbbt/util/test_config.rb +0 -69
  120. data/test/rbbt/util/test_excel2tsv.rb +0 -10
  121. data/test/rbbt/util/test_filecache.rb +0 -36
  122. data/test/rbbt/util/test_log.rb +0 -52
  123. data/test/rbbt/util/test_migrate.rb +0 -36
  124. data/test/rbbt/util/test_misc.rb +0 -723
  125. data/test/rbbt/util/test_open.rb +0 -200
  126. data/test/rbbt/util/test_procpath.rb +0 -23
  127. data/test/rbbt/util/test_python.rb +0 -140
  128. data/test/rbbt/util/test_semaphore.rb +0 -36
  129. data/test/rbbt/util/test_simpleDSL.rb +0 -55
  130. data/test/rbbt/util/test_simpleopt.rb +0 -11
  131. data/test/rbbt/util/test_ssh.rb +0 -10
  132. data/test/rbbt/util/test_tmpfile.rb +0 -32
  133. data/test/rbbt/workflow/step/test_dependencies.rb +0 -299
  134. data/test/rbbt/workflow/step/test_save_load_inputs.rb +0 -136
  135. data/test/rbbt/workflow/test_doc.rb +0 -30
  136. data/test/rbbt/workflow/test_remote_workflow.rb +0 -98
  137. data/test/rbbt/workflow/test_schedule.rb +0 -0
  138. data/test/rbbt/workflow/test_step.rb +0 -231
  139. data/test/rbbt/workflow/test_task.rb +0 -85
  140. data/test/rbbt/workflow/util/test_archive.rb +0 -31
  141. data/test/rbbt/workflow/util/test_data.rb +0 -67
  142. data/test/rbbt/workflow/util/test_orchestrator.rb +0 -273
  143. data/test/test_helper.rb +0 -90
@@ -1,63 +0,0 @@
1
- require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
2
- require 'rbbt/util/misc'
3
- require 'rbbt/association'
4
- require 'rbbt/association/open'
5
-
6
- class TestAssociationOpen < Test::Unit::TestCase
7
-
8
- EFFECT =<<-END
9
- #: :sep=" "#:type=:double
10
- #SG TG Effect directed?
11
- MDM2 TP53 inhibition false
12
- TP53 NFKB1|GLI1 activation|activation true|true
13
- END
14
-
15
- EFFECT_OPTIONS = {
16
- :source => "SG=~Associated Gene Name",
17
- :target => "TG=~Associated Gene Name=>Ensembl Gene ID",
18
- :persist => false,
19
- :identifiers => datafile_test('identifiers'),
20
- :namespace => "Hsa"
21
- }
22
-
23
- EFFECT_TSV = TSV.open EFFECT, EFFECT_OPTIONS.dup
24
-
25
- def test_open_no_persist
26
- tsv = Association.open(EFFECT_TSV, EFFECT_OPTIONS, :persist => false)
27
- assert_equal "ENSG00000141510", tsv["MDM2"]["Ensembl Gene ID"].first
28
- assert_equal ["false"], tsv["MDM2"]["directed?"]
29
- end
30
-
31
-
32
- def test_open_persist
33
- tsv = Association.open(EFFECT_TSV, EFFECT_OPTIONS, :persist => true, :update => true)
34
- assert_equal "ENSG00000141510", tsv["MDM2"]["Ensembl Gene ID"].first
35
- assert_equal ["false"], tsv["MDM2"]["directed?"]
36
- end
37
-
38
- def test_open_no_persist_string
39
- tsv = Association.open(EFFECT, EFFECT_OPTIONS.merge(:source => "TG", :target => "SG=~Associated Gene Name"), :persist => false)
40
- tsv.include? "TP53"
41
- assert_equal ["MDM2"], tsv["TP53"]["Associated Gene Name"]
42
- end
43
-
44
- def test_index_no_persist_string
45
- tsv = Association.index(EFFECT, EFFECT_OPTIONS.merge(:source => "TG", :target => "SG=~Associated Gene Name"), :persist => false)
46
- tsv.unnamed = false
47
- assert_equal "inhibition", tsv["TP53~MDM2"]["Effect"]
48
- end
49
-
50
- def test_index_persist_string
51
- tsv = Association.index(EFFECT, EFFECT_OPTIONS.merge(:source => "SG", :target => "TG=~Associated Gene Name"), :persist => true)
52
- tsv.unnamed = false
53
- assert_equal "inhibition", tsv["MDM2~TP53"]["Effect"]
54
- assert_equal %w(TP53~GLI1 TP53~NFKB1), tsv.match("TP53").sort
55
- end
56
-
57
- def test_index_persist_reverse
58
- tsv = Association.index(EFFECT, EFFECT_OPTIONS.merge( :source => "TG", :target => "SG=~Associated Gene Name"), :persist => true).reverse
59
- tsv.unnamed = false
60
- assert_equal "inhibition", tsv["MDM2~TP53"]["Effect"]
61
- assert_equal %w(MDM2~TP53), tsv.match("MDM2")
62
- end
63
- end
@@ -1,108 +0,0 @@
1
- require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
2
- require 'rbbt/util/misc'
3
- require 'rbbt/association'
4
- require 'rbbt/association'
5
- require 'rbbt/association/util'
6
-
7
- class TestAssociationUtil < Test::Unit::TestCase
8
-
9
- def test_specs
10
- spec = Association.parse_field_specification "=~Associated Gene Name=>Ensembl Gene ID"
11
- assert_equal [nil, "Associated Gene Name", "Ensembl Gene ID"], spec
12
-
13
- spec = Association.parse_field_specification "TG=~Associated Gene Name=>Ensembl Gene ID"
14
- assert_equal ["TG", "Associated Gene Name", "Ensembl Gene ID"], spec
15
-
16
- spec = Association.parse_field_specification "TG=~Associated Gene Name"
17
- assert_equal ["TG", "Associated Gene Name", nil], spec
18
-
19
- spec = Association.parse_field_specification "Associated Gene Name=>Ensembl Gene ID"
20
- assert_equal ["Associated Gene Name", nil, "Ensembl Gene ID"], spec
21
- end
22
-
23
- def test_normalize_specs
24
- spec = Association.normalize_specs "TG=~Associated Gene Name=>Ensembl Gene ID", %w(SG TG Effect directed?)
25
- assert_equal ["TG", "Associated Gene Name", "Ensembl Gene ID"], spec
26
-
27
- spec = Association.normalize_specs "Ensembl Gene ID", %w(SG TG Effect directed?)
28
- assert_equal ["Ensembl Gene ID", nil, nil], spec
29
-
30
- spec = Association.normalize_specs "Ensembl Gene ID"
31
- assert_equal ["Ensembl Gene ID", nil, nil], spec
32
- end
33
-
34
- def test_process_specs
35
- spec = Association.extract_specs %w(SG TG Effect directed?), :target => "TG=~Associated Gene Name=>Ensembl Gene ID"
36
- assert_equal ["TG", "Associated Gene Name", "Ensembl Gene ID"], spec[:target]
37
- assert_equal ["SG", nil, nil], spec[:source]
38
-
39
- spec = Association.extract_specs %w(SG TG Effect directed?)
40
- assert_equal ["SG", nil, nil], spec[:source]
41
- assert_equal ["TG", nil,nil], spec[:target]
42
-
43
- spec = Association.extract_specs %w(SG TG Effect directed?), :source => "TG"
44
- assert_equal ["TG", nil, nil], spec[:source]
45
- assert_equal ["SG", nil,nil], spec[:target]
46
-
47
- spec = Association.extract_specs %w(SG TG Effect directed?), :source => "SG"
48
- assert_equal ["SG", nil, nil], spec[:source]
49
- assert_equal ["TG", nil,nil], spec[:target]
50
-
51
- spec = Association.extract_specs %w(SG TG Effect directed?), :target => "SG"
52
- assert_equal ["TG", nil, nil], spec[:source]
53
- assert_equal ["SG", nil,nil], spec[:target]
54
- end
55
-
56
- def test_headers
57
- spec = Association.headers %w(SG TG Effect directed?), %w(Effect)
58
- assert_equal 0, spec[0]
59
- assert_equal 1, spec[1][0]
60
-
61
- spec = Association.headers %w(SG TG Effect directed?), %w(Effect), :source => "SG"
62
- assert_equal 0, spec[0]
63
- assert_equal 1, spec[1][0]
64
-
65
- spec = Association.headers %w(SG TG Effect directed?), %w(Effect), :target => "TG"
66
- assert_equal 0, spec[0]
67
- assert_equal 1, spec[1][0]
68
-
69
- spec = Association.headers %w(SG TG Effect directed?), %w(Effect), :target => "TG", :source => "SG"
70
- assert_equal 0, spec[0]
71
- assert_equal 1, spec[1][0]
72
-
73
- spec = Association.headers %w(SG TG Effect directed?), %w(Effect), :source => "TG"
74
- assert_equal 1, spec[0]
75
- assert_equal 0, spec[1][0]
76
-
77
- spec = Association.headers %w(SG TG Effect directed?), %w(Effect), :target => "SG"
78
- assert_equal 1, spec[0]
79
- assert_equal 0, spec[1][0]
80
-
81
- spec = Association.headers %w(SG TG Effect directed?), %w(Effect), :target => "SG", :source => "TG"
82
- assert_equal 1, spec[0]
83
- assert_equal 0, spec[1][0]
84
-
85
- spec = Association.headers %w(SG TG Effect directed?), %w(Effect), :target => "SG", :source => "TG=~Associated Gene Name=>Ensembl Gene ID"
86
- assert_equal 1, spec[0]
87
- assert_equal 0, spec[1][0]
88
-
89
- spec = Association.headers %w(SG TG Effect directed?), %w(Effect), :target => "SG=~Associated Gene Name=>Ensembl Gene ID", :source => "TG=~Associated Gene Name=>Ensembl Gene ID"
90
- assert_equal 1, spec[0]
91
- assert_equal 0, spec[1][0]
92
-
93
- spec = Association.headers %w(SG TG Effect directed?), %w(Effect), :target => "SG=~Associated Gene Name", :source => "TG=~Associated Gene Name=>Ensembl Gene ID"
94
- assert_equal 1, spec[0]
95
- assert_equal 0, spec[1][0]
96
-
97
- spec = Association.headers %w(SG TG Effect directed?), %w(Effect), :target => "TG=~Associated Gene Name", :source => "SG=~Associated Gene Name", :target_format => "Ensembl Gene ID"
98
- assert_equal 0, spec[0]
99
- assert_equal "Ensembl Gene ID", spec[5]
100
- assert_equal nil, spec[4]
101
-
102
-
103
- spec = Association.headers %w(SG TG Effect directed?), %w(Effect), :target => "TG=~Associated Gene Name", :source => "SG=~Associated Gene Name", :source_format => "Ensembl Gene ID"
104
- assert_equal 0, spec[0]
105
- assert_equal "Ensembl Gene ID", spec[4]
106
- assert_equal nil, spec[5]
107
- end
108
- end
@@ -1,74 +0,0 @@
1
- require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
2
- require 'rbbt-util'
3
- require 'rbbt/entity'
4
- require 'rbbt/entity/identifiers'
5
-
6
- module Gene
7
- extend Entity
8
- end
9
-
10
- class TestEntityIdentifiers < Test::Unit::TestCase
11
- def setup
12
- require 'rbbt/sources/kegg'
13
- require 'rbbt/sources/organism'
14
- Gene.add_identifiers Organism.identifiers("NAMESPACE"), "Ensembl Gene ID", "Associated Gene Name"
15
- Gene.add_identifiers KEGG.identifiers
16
- Gene.add_identifiers Organism.identifiers("NAMESPACE"), "Ensembl Gene ID", "Associated Gene Name"
17
- Gene.add_identifiers KEGG.identifiers
18
-
19
- end
20
- def test_name
21
- Gene.add_identifiers datafile_test('identifiers'), "Ensembl Gene ID", "Associated Gene Name"
22
- assert_equal "TP53", Gene.setup("ENSG00000141510").name
23
- end
24
-
25
- def test_name_organism
26
- assert_equal "TP53", Gene.setup("ENSG00000141510", "Ensembl Gene ID", "Hsa/feb2014").name
27
- end
28
-
29
- def test_identifier_files
30
- assert Gene.identifier_files.any?
31
- end
32
-
33
- def test_Entity_identifier_files
34
- assert Entity.identifier_files("Ensembl Gene ID").any?
35
- end
36
-
37
- def test_translate_kegg
38
- assert_match "hsa", Gene.setup("ENSG00000141510", "Ensembl Gene ID", "Hsa/feb2014").to("KEGG Gene ID")
39
- assert_match "TP53", Gene.setup("ENSG00000141510", "Ensembl Gene ID", "Hsa/feb2014").to("KEGG Gene ID").to(:name)
40
- end
41
-
42
- def ___test_complex
43
-
44
- file = Path.setup('/home/mvazquezg/git/workflows/PanCancer/share/network/gene_sets/CORUM_protein_complexes.gmt')
45
- name = File.basename(file).sub(/\.gmt$/,'')
46
- organism = "Hsa/feb2014"
47
- key_field = "#{ name } Pathway ID"
48
- description_field = "#{name} Pathway Description"
49
- description_file = file.find + '.identifiers'
50
- tsv = TSV.open(file, :fix => Proc.new{|l| p=l.split"\t"; [p[0], p[1], p[2..-1]*"|"]*"\t"})
51
- tsv.namespace = organism
52
- tsv.unnamed = true
53
- gene_field, count = Organism.guess_id(organism, tsv.values.collect{|l| l.last}.flatten.uniq )
54
- tsv.key_field = key_field
55
- tsv.fields = [description_field, gene_field]
56
- descriptions = tsv.slice(description_field)
57
- Open.write(description_file, descriptions.to_single.to_s) #unless File.exist? description_file
58
- values = tsv.slice(gene_field)
59
- values.identifiers = description_file
60
-
61
- mod = Module.new
62
- mod_name = Misc.camel_case(key_field.gsub(/\s+/,'_').sub(/_ID$/,''))
63
- Object.const_set(mod_name, mod)
64
- mod.instance_eval do
65
- extend Entity
66
- add_identifiers Path.setup(description_file), key_field, description_field
67
-
68
- annotation :format
69
- end
70
- entity = "CORUM:6052"
71
- mod.setup(entity, :format => key_field)
72
- puts entity.name
73
- end
74
- end
@@ -1,113 +0,0 @@
1
- require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
2
- require 'rbbt/workflow'
3
- require 'rbbt/hpc/orchestrate/batches'
4
-
5
- module TestWFA
6
- extend Workflow
7
-
8
- task :a1 => :string do self.task_name.to_s end
9
-
10
- dep :a1
11
- task :a2 => :string do self.task_name.to_s end
12
-
13
- dep :a2
14
- task :a3 => :string do self.task_name.to_s end
15
- end
16
-
17
- module TestWFB
18
- extend Workflow
19
-
20
- dep TestWFA, :a2
21
- task :b1 => :string do self.task_name.to_s end
22
-
23
- dep :b1
24
- task :b2 => :string do self.task_name.to_s end
25
- end
26
-
27
- module TestWFC
28
- extend Workflow
29
-
30
- dep TestWFA, :a1
31
- dep_task :c1, TestWFB, :b2
32
-
33
- task :c2 => :string do self.task_name.to_s end
34
-
35
- dep :c1
36
- dep :c2
37
- task :c3 => :string do self.task_name.to_s end
38
-
39
- dep_task :c4, TestWFC, :c3
40
- end
41
-
42
- module TestWFD
43
- extend Workflow
44
-
45
- dep TestWFC, :c3, :jobname => "First c3"
46
- dep TestWFC, :c3, :jobname => "Second c3"
47
- task :d1 => :string do self.task_name.to_s end
48
- end
49
-
50
- class TestOrchestrate < Test::Unit::TestCase
51
-
52
- RULES = IndiferentHash.setup(YAML.load(<<-EOF))
53
- ---
54
- defaults:
55
- queue: first_queue
56
- time: 1h
57
- log: 2
58
- config_keys: key1 value1 token1
59
- chains:
60
- chain_a_b:
61
- tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1, TestWFA#a2
62
- config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
63
- chain_a:
64
- workflow: TestWFA
65
- tasks: a1, a2, a3
66
- config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
67
- chain_b:
68
- workflow: TestWFB
69
- tasks: b1, b2
70
- chain_b2:
71
- tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1
72
- chain_d:
73
- tasks: TestWFD#d1, TestWFC#c1, TestWFC#c2, TestWFC#c3
74
- TestWFA:
75
- defaults:
76
- log: 4
77
- config_keys: key4 value4 token4
78
- time: 10min
79
- a1:
80
- cpus: 10
81
- config_keys: key5 value5 token5
82
- TestWFC:
83
- defaults:
84
- skip: true
85
- log: 4
86
- time: 10s
87
- EOF
88
-
89
-
90
- def test_job_batches_d
91
- job = TestWFD.job(:d1, nil)
92
- job.recursive_clean
93
-
94
- batches = HPC::Orchestration.job_batches(RULES, job)
95
- assert_equal 3, batches.length
96
- end
97
-
98
- def test_job_batches_c3
99
- job = TestWFC.job(:c3, nil)
100
- job.recursive_clean
101
-
102
- batches = HPC::Orchestration.job_batches(RULES, job)
103
- end
104
-
105
- def test_job_batches_c4
106
- job = TestWFC.job(:c4, nil)
107
- job.recursive_clean
108
-
109
- batches = HPC::Orchestration.job_batches(RULES, job)
110
- assert_equal 3, batches.length
111
- end
112
-
113
- end
@@ -1,139 +0,0 @@
1
- require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
2
- require 'rbbt/workflow'
3
- require 'rbbt/hpc/orchestrate/chains'
4
-
5
- module TestWFA
6
- extend Workflow
7
-
8
- task :a1 => :string do self.task_name.to_s end
9
-
10
- dep :a1
11
- task :a2 => :string do self.task_name.to_s end
12
-
13
- dep :a2
14
- task :a3 => :string do self.task_name.to_s end
15
- end
16
-
17
- module TestWFB
18
- extend Workflow
19
-
20
- dep TestWFA, :a2
21
- task :b1 => :string do self.task_name.to_s end
22
-
23
- dep :b1
24
- task :b2 => :string do self.task_name.to_s end
25
- end
26
-
27
- module TestWFC
28
- extend Workflow
29
-
30
- dep TestWFA, :a1
31
- dep_task :c1, TestWFB, :b2
32
-
33
- task :c2 => :string do self.task_name.to_s end
34
-
35
- dep :c1
36
- dep :c2
37
- task :c3 => :string do self.task_name.to_s end
38
- end
39
-
40
- module TestWFD
41
- extend Workflow
42
-
43
- dep TestWFC, :c3, :jobname => "First c3"
44
- dep TestWFC, :c3, :jobname => "Second c3"
45
- task :d1 => :string do self.task_name.to_s end
46
- end
47
-
48
- class TestOrchestrate < Test::Unit::TestCase
49
-
50
- RULES = IndiferentHash.setup(YAML.load(<<-EOF))
51
- ---
52
- defaults:
53
- queue: first_queue
54
- time: 1h
55
- log: 2
56
- config_keys: key1 value1 token1
57
- chains:
58
- chain_a_b:
59
- tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1, TestWFA#a2
60
- comment: Should not include TestWFA#a1
61
- chain_a:
62
- workflow: TestWFA
63
- tasks: a1, a2, a3
64
- config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
65
- chain_b:
66
- workflow: TestWFB
67
- tasks: b1, b2
68
- chain_b2:
69
- tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1
70
- chain_d:
71
- tasks: TestWFD#d1, TestWFC#c1, TestWFC#c2, TestWFC#c3
72
- TestWFA:
73
- defaults:
74
- log: 4
75
- config_keys: key4 value4 token4
76
- a1:
77
- cpus: 10
78
- config_keys: key5 value5 token5
79
- TestWFC:
80
- defaults:
81
- skip: true
82
- log: 4
83
- EOF
84
-
85
- def test_parse_chains
86
- chains = HPC::Orchestration.parse_chains RULES
87
- assert chains[:chain_a][:tasks]["TestWFA"].include?("a1")
88
- end
89
-
90
- def test_job_chain_a
91
- job = TestWFA.job(:a3, nil)
92
- job.recursive_clean
93
-
94
- job_chains = HPC::Orchestration.job_chains(RULES, job)
95
- job_chains = NamedArray.setup(job_chains, job_chains.collect{|n,i| n })
96
-
97
- assert_equal %w(chain_a_b chain_a), job_chains.fields
98
- assert_equal 3, job_chains["chain_a"].last[:jobs].length
99
- assert_equal job, job_chains["chain_a"].last[:top_level]
100
- end
101
-
102
- def test_job_chain_b
103
- job = TestWFB.job(:b2, nil)
104
- job.recursive_clean
105
-
106
- job_chains = HPC::Orchestration.job_chains(RULES, job)
107
- job_chains = NamedArray.setup(job_chains, job_chains.collect{|n,i| n })
108
-
109
- assert_equal %w(chain_a chain_b chain_b2 chain_a_b).sort, job_chains.fields.sort
110
-
111
- assert_equal 2, job_chains["chain_a"].last[:jobs].length
112
- assert_equal job.step("a2"), job_chains["chain_a"].last[:top_level]
113
-
114
- assert_equal 2, job_chains["chain_b"].last[:jobs].length
115
- assert_equal job, job_chains["chain_b"].last[:top_level]
116
-
117
- assert_equal 2, job_chains["chain_b2"].last[:jobs].length
118
- assert_equal job, job_chains["chain_b2"].last[:top_level]
119
-
120
- assert_equal 4, job_chains["chain_a_b"].last[:jobs].length
121
- assert_equal job, job_chains["chain_a_b"].last[:top_level]
122
- end
123
-
124
- def test_job_chains_double
125
- job = TestWFD.job(:d1, nil)
126
- job.recursive_clean
127
-
128
- job_chains = HPC::Orchestration.job_chains(RULES, job)
129
- job_chains = NamedArray.setup(job_chains, job_chains.collect{|n,i| n })
130
-
131
- assert_equal 2, job_chains.select{|n,i| n == 'chain_a' }.length
132
- assert_equal ["First c3", "Second c3"].sort, job_chains.
133
- select{|n,i| n == 'chain_a' }.
134
- collect{|n,i| i[:top_level].name }.sort
135
-
136
- assert_equal 1, job_chains.select{|n,i| n == 'chain_d' }.length
137
- end
138
- end
139
-
@@ -1,92 +0,0 @@
1
- require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
2
- require 'rbbt/workflow'
3
- require 'rbbt/hpc/orchestrate/rules'
4
-
5
- module TestWFA
6
- extend Workflow
7
-
8
- task :a1 => :string do self.task_name.to_s end
9
-
10
- dep :a1
11
- task :a2 => :string do self.task_name.to_s end
12
-
13
- dep :a2
14
- task :a3 => :string do self.task_name.to_s end
15
- end
16
-
17
- module TestWFB
18
- extend Workflow
19
-
20
- dep TestWFA, :a2
21
- task :b1 => :string do self.task_name.to_s end
22
-
23
- dep :b1
24
- task :b2 => :string do self.task_name.to_s end
25
- end
26
-
27
- module TestWFC
28
- extend Workflow
29
-
30
- dep TestWFA, :a1
31
- dep_task :c1, TestWFB, :b2
32
-
33
- task :c2 => :string do self.task_name.to_s end
34
-
35
- dep :c1
36
- dep :c2
37
- task :c3 => :string do self.task_name.to_s end
38
- end
39
-
40
- class TestOrchestrate < Test::Unit::TestCase
41
-
42
- RULES = IndiferentHash.setup(YAML.load(<<-EOF))
43
- ---
44
- defaults:
45
- queue: first_queue
46
- time: 1h
47
- log: 2
48
- config_keys: key1 value1 token1
49
- chains:
50
- chain_a:
51
- workflow: TestWFA
52
- tasks: a1, a2, a3
53
- config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
54
- chain_b:
55
- workflow: TestWFB
56
- tasks: b1, b2
57
- chain_b2:
58
- comment: This chain is not valid, it is missing a2
59
- tasks: TestWFA#a1, TestWFA#a3, TestWFB#b1, TestWFB#b2
60
- TestWFA:
61
- defaults:
62
- log: 4
63
- config_keys: key4 value4 token4
64
- a1:
65
- cpus: 10
66
- config_keys: key5 value5 token5
67
- TestWFC:
68
- defaults:
69
- skip: true
70
- log: 4
71
- EOF
72
-
73
- def test_defaults
74
- rules = HPC::Orchestration.task_specific_rules RULES, "TestWFA", :a1
75
- assert_equal "first_queue" , rules[:queue]
76
- end
77
-
78
-
79
- def test_task_options
80
- rules = HPC::Orchestration.task_specific_rules RULES, "TestWFA", :a1
81
- assert_equal 10, rules[:cpus]
82
- assert_equal 4, rules[:log]
83
- end
84
-
85
- def test_skip
86
- rules = HPC::Orchestration.task_specific_rules RULES, "TestWFC", :c1
87
- assert rules[:skip]
88
- end
89
-
90
-
91
- end
92
-
@@ -1,68 +0,0 @@
1
- require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
2
- require 'rbbt/hpc/batch'
3
- require 'rbbt/workflow'
4
-
5
- class TestSLURM < Test::Unit::TestCase
6
-
7
- def setup
8
- Workflow.require_workflow "Sample"
9
- Workflow.require_workflow "HTS"
10
- end
11
-
12
- def test_batch_options
13
- job = Sample.job(:mutect2, "small", :reference => "hg38")
14
-
15
- TmpFile.with_file do |batch_dir|
16
-
17
- options = HPC::BATCH.batch_options(job, :batch_dir => batch_dir, :batch_modules => 'java')
18
-
19
- iii options
20
- end
21
- end
22
-
23
- def test_template
24
- job = Sample.job(:mutect2, "small", :reference => "hg38")
25
-
26
- TmpFile.with_file do |batch_dir|
27
-
28
- template = HPC::BATCH.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java')
29
- ppp template
30
-
31
- end
32
- end
33
-
34
- def test_template_singularity
35
- job = Sample.job(:mutect2, "small", :reference => "hg38")
36
-
37
- TmpFile.with_file do |batch_dir|
38
-
39
- template = HPC::BATCH.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java', :singularity => true)
40
- ppp template
41
-
42
- end
43
- end
44
-
45
- def test_template_contain
46
- job = Sample.job(:mutect2, "small", :reference => "hg38")
47
-
48
- TmpFile.with_file do |batch_dir|
49
-
50
- template = HPC::BATCH.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java', :contain_and_sync => true, :wipe_container => 'force')
51
- ppp template
52
-
53
- end
54
- end
55
-
56
- def test_template_singularity_contain
57
- job = Sample.job(:mutect2, "small", :reference => "hg38")
58
-
59
- TmpFile.with_file do |batch_dir|
60
-
61
- template = HPC::BATCH.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java', :contain_and_sync => true, :wipe_container => 'force', :singularity => true)
62
- ppp template
63
-
64
- end
65
- end
66
-
67
- end
68
-