miga-base 0.7.4.0 → 0.7.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/miga/cli.rb +10 -8
- data/lib/miga/cli/action.rb +2 -3
- data/lib/miga/cli/action/about.rb +5 -6
- data/lib/miga/cli/action/add.rb +18 -12
- data/lib/miga/cli/action/add_result.rb +2 -3
- data/lib/miga/cli/action/archive.rb +1 -2
- data/lib/miga/cli/action/classify_wf.rb +8 -6
- data/lib/miga/cli/action/console.rb +0 -1
- data/lib/miga/cli/action/daemon.rb +7 -7
- data/lib/miga/cli/action/date.rb +0 -1
- data/lib/miga/cli/action/derep_wf.rb +5 -4
- data/lib/miga/cli/action/doctor.rb +28 -20
- data/lib/miga/cli/action/doctor/base.rb +29 -6
- data/lib/miga/cli/action/edit.rb +1 -2
- data/lib/miga/cli/action/files.rb +8 -8
- data/lib/miga/cli/action/find.rb +5 -6
- data/lib/miga/cli/action/generic.rb +7 -7
- data/lib/miga/cli/action/get.rb +20 -17
- data/lib/miga/cli/action/get_db.rb +8 -2
- data/lib/miga/cli/action/index_wf.rb +1 -1
- data/lib/miga/cli/action/init.rb +34 -29
- data/lib/miga/cli/action/init/daemon_helper.rb +65 -43
- data/lib/miga/cli/action/lair.rb +7 -7
- data/lib/miga/cli/action/ln.rb +6 -6
- data/lib/miga/cli/action/ls.rb +1 -2
- data/lib/miga/cli/action/ncbi_get.rb +11 -3
- data/lib/miga/cli/action/new.rb +4 -4
- data/lib/miga/cli/action/next_step.rb +0 -1
- data/lib/miga/cli/action/preproc_wf.rb +3 -3
- data/lib/miga/cli/action/quality_wf.rb +1 -1
- data/lib/miga/cli/action/rm.rb +2 -3
- data/lib/miga/cli/action/run.rb +8 -8
- data/lib/miga/cli/action/stats.rb +3 -3
- data/lib/miga/cli/action/summary.rb +7 -6
- data/lib/miga/cli/action/tax_dist.rb +8 -4
- data/lib/miga/cli/action/tax_index.rb +3 -4
- data/lib/miga/cli/action/tax_set.rb +7 -6
- data/lib/miga/cli/action/tax_test.rb +6 -5
- data/lib/miga/cli/action/wf.rb +21 -19
- data/lib/miga/cli/base.rb +34 -32
- data/lib/miga/cli/objects_helper.rb +24 -17
- data/lib/miga/cli/opt_helper.rb +3 -2
- data/lib/miga/common.rb +2 -5
- data/lib/miga/common/base.rb +15 -16
- data/lib/miga/common/format.rb +8 -5
- data/lib/miga/common/hooks.rb +1 -4
- data/lib/miga/common/path.rb +4 -9
- data/lib/miga/common/with_daemon.rb +5 -2
- data/lib/miga/common/with_daemon_class.rb +1 -1
- data/lib/miga/common/with_result.rb +2 -1
- data/lib/miga/daemon.rb +51 -35
- data/lib/miga/daemon/base.rb +0 -2
- data/lib/miga/dataset.rb +47 -37
- data/lib/miga/dataset/base.rb +52 -37
- data/lib/miga/dataset/hooks.rb +3 -4
- data/lib/miga/dataset/result.rb +17 -1
- data/lib/miga/json.rb +5 -7
- data/lib/miga/lair.rb +4 -0
- data/lib/miga/metadata.rb +4 -3
- data/lib/miga/project.rb +29 -20
- data/lib/miga/project/base.rb +52 -37
- data/lib/miga/project/dataset.rb +27 -13
- data/lib/miga/project/hooks.rb +0 -3
- data/lib/miga/project/result.rb +14 -5
- data/lib/miga/remote_dataset.rb +85 -72
- data/lib/miga/remote_dataset/base.rb +11 -13
- data/lib/miga/remote_dataset/download.rb +33 -12
- data/lib/miga/result.rb +34 -25
- data/lib/miga/result/base.rb +0 -2
- data/lib/miga/result/dates.rb +1 -3
- data/lib/miga/result/source.rb +15 -16
- data/lib/miga/result/stats.rb +36 -25
- data/lib/miga/tax_dist.rb +6 -3
- data/lib/miga/tax_index.rb +17 -17
- data/lib/miga/taxonomy.rb +6 -1
- data/lib/miga/taxonomy/base.rb +19 -15
- data/lib/miga/version.rb +19 -16
- data/test/common_test.rb +3 -11
- data/test/daemon_helper.rb +38 -0
- data/test/daemon_test.rb +73 -101
- data/test/dataset_test.rb +58 -59
- data/test/format_test.rb +3 -11
- data/test/hook_test.rb +50 -55
- data/test/json_test.rb +7 -8
- data/test/lair_test.rb +22 -28
- data/test/metadata_test.rb +6 -14
- data/test/project_test.rb +33 -39
- data/test/remote_dataset_test.rb +20 -28
- data/test/result_stats_test.rb +17 -27
- data/test/result_test.rb +41 -34
- data/test/tax_dist_test.rb +0 -2
- data/test/tax_index_test.rb +4 -10
- data/test/taxonomy_test.rb +7 -9
- data/test/test_helper.rb +42 -1
- data/test/with_daemon_test.rb +14 -22
- data/utils/cleanup-databases.rb +6 -5
- data/utils/distance/base.rb +0 -1
- data/utils/distance/commands.rb +19 -12
- data/utils/distance/database.rb +24 -21
- data/utils/distance/pipeline.rb +12 -9
- data/utils/distance/runner.rb +14 -13
- data/utils/distance/temporal.rb +1 -3
- data/utils/distances.rb +1 -1
- data/utils/domain-ess-genes.rb +7 -7
- data/utils/index_metadata.rb +4 -2
- data/utils/mytaxa_scan.rb +18 -16
- data/utils/representatives.rb +5 -4
- data/utils/requirements.txt +1 -1
- data/utils/subclade/base.rb +0 -1
- data/utils/subclade/pipeline.rb +7 -6
- data/utils/subclade/runner.rb +9 -9
- data/utils/subclade/temporal.rb +0 -2
- data/utils/subclades-compile.rb +39 -37
- data/utils/subclades.rb +1 -1
- metadata +3 -2
data/lib/miga/taxonomy/base.rb
CHANGED
@@ -2,49 +2,54 @@
|
|
2
2
|
# @license Artistic-2.0
|
3
3
|
|
4
4
|
class MiGA::Taxonomy < MiGA::MiGA
|
5
|
-
|
6
5
|
class << self
|
7
|
-
|
8
6
|
##
|
9
|
-
# Returns cannonical rank (Symbol) for the +rank+ String
|
7
|
+
# Returns cannonical rank (Symbol) for the +rank+ String
|
10
8
|
def normalize_rank(rank)
|
11
9
|
return rank.to_sym if @@_KNOWN_RANKS_H[rank.to_sym]
|
10
|
+
|
12
11
|
rank = rank.to_s.downcase
|
13
12
|
return nil if rank == 'no rank'
|
13
|
+
|
14
14
|
rank = @@RANK_SYNONYMS[rank] unless @@RANK_SYNONYMS[rank].nil?
|
15
15
|
rank = rank.to_sym
|
16
16
|
return nil unless @@_KNOWN_RANKS_H[rank]
|
17
|
+
|
17
18
|
rank
|
18
19
|
end
|
19
20
|
|
20
21
|
##
|
21
|
-
# Initialize from JSON-derived Hash +o
|
22
|
+
# Initialize from JSON-derived Hash +o+
|
22
23
|
def json_create(o)
|
23
24
|
new(o['str'], nil, o['alt'])
|
24
25
|
end
|
25
26
|
|
26
|
-
def KNOWN_RANKS()
|
27
|
-
|
27
|
+
def KNOWN_RANKS()
|
28
|
+
@@KNOWN_RANKS
|
29
|
+
end
|
28
30
|
|
31
|
+
def LONG_RANKS()
|
32
|
+
@@LONG_RANKS
|
33
|
+
end
|
29
34
|
end
|
30
|
-
|
31
35
|
end
|
32
36
|
|
33
37
|
module MiGA::Taxonomy::Base
|
34
|
-
|
35
38
|
##
|
36
|
-
# Cannonical ranks
|
39
|
+
# Cannonical ranks
|
37
40
|
@@KNOWN_RANKS = %w{ns d k p c o f g s ssp str ds}.map { |r| r.to_sym }
|
38
|
-
@@_KNOWN_RANKS_H = Hash[
|
41
|
+
@@_KNOWN_RANKS_H = Hash[@@KNOWN_RANKS.map { |i| [i, true] }]
|
39
42
|
|
40
43
|
##
|
41
|
-
# Long names of the cannonical ranks
|
42
|
-
@@LONG_RANKS = {
|
44
|
+
# Long names of the cannonical ranks
|
45
|
+
@@LONG_RANKS = {
|
46
|
+
root: 'root', ns: 'namespace', d: 'domain', k: 'kingdom',
|
43
47
|
p: 'phylum', c: 'class', o: 'order', f: 'family', g: 'genus', s: 'species',
|
44
|
-
ssp: 'subspecies', str: 'strain', ds: 'dataset'
|
48
|
+
ssp: 'subspecies', str: 'strain', ds: 'dataset'
|
49
|
+
}
|
45
50
|
|
46
51
|
##
|
47
|
-
# Synonms for cannonical ranks
|
52
|
+
# Synonms for cannonical ranks
|
48
53
|
@@RANK_SYNONYMS = {
|
49
54
|
'namespace' => 'ns',
|
50
55
|
'domain' => 'd', 'superkingdom' => 'd',
|
@@ -59,5 +64,4 @@ module MiGA::Taxonomy::Base
|
|
59
64
|
'strain' => 'str', 'isolate' => 'str', 'culture' => 'str',
|
60
65
|
'dataset' => 'ds', 'organism' => 'ds', 'genome' => 'ds', 'specimen' => 'ds'
|
61
66
|
}
|
62
|
-
|
63
67
|
end
|
data/lib/miga/version.rb
CHANGED
@@ -1,16 +1,14 @@
|
|
1
|
-
|
2
1
|
require 'date'
|
3
2
|
|
4
3
|
##
|
5
4
|
# High-level minimal requirements for the MiGA::MiGA class.
|
6
5
|
module MiGA
|
7
|
-
|
8
6
|
##
|
9
7
|
# Current version of MiGA. An Array with three values:
|
10
8
|
# - Float representing the major.minor version.
|
11
9
|
# - Integer representing gem releases of the current version.
|
12
10
|
# - Integer representing minor changes that require new version number.
|
13
|
-
VERSION = [0.7,
|
11
|
+
VERSION = [0.7, 5, 0]
|
14
12
|
|
15
13
|
##
|
16
14
|
# Nickname for the current major.minor version.
|
@@ -18,7 +16,7 @@ module MiGA
|
|
18
16
|
|
19
17
|
##
|
20
18
|
# Date of the current gem release.
|
21
|
-
VERSION_DATE = Date.new(2020,
|
19
|
+
VERSION_DATE = Date.new(2020, 5, 13)
|
22
20
|
|
23
21
|
##
|
24
22
|
# Reference of MiGA.
|
@@ -26,33 +24,38 @@ module MiGA
|
|
26
24
|
'The Microbial Genomes Atlas (MiGA) webserver: taxonomic and gene ' \
|
27
25
|
'diversity analysis of Archaea and Bacteria at the whole genome level. ' \
|
28
26
|
'Nucleic Acids Research 46(W1):W282-W288. doi:10.1093/nar/gky467.'
|
29
|
-
|
30
27
|
end
|
31
28
|
|
32
29
|
class MiGA::MiGA
|
33
|
-
|
34
30
|
include MiGA
|
35
31
|
|
36
32
|
##
|
37
|
-
# Major.minor version as Float
|
38
|
-
def self.VERSION
|
33
|
+
# Major.minor version as Float
|
34
|
+
def self.VERSION
|
35
|
+
VERSION[0]
|
36
|
+
end
|
39
37
|
|
40
38
|
##
|
41
|
-
# Complete version as string
|
42
|
-
def self.FULL_VERSION
|
39
|
+
# Complete version as string
|
40
|
+
def self.FULL_VERSION
|
41
|
+
VERSION.join('.')
|
42
|
+
end
|
43
43
|
|
44
44
|
##
|
45
|
-
# Complete version with nickname and date as string
|
45
|
+
# Complete version with nickname and date as string
|
46
46
|
def self.LONG_VERSION
|
47
47
|
"MiGA #{VERSION.join('.')} - #{VERSION_NAME} - #{VERSION_DATE}"
|
48
48
|
end
|
49
49
|
|
50
50
|
##
|
51
|
-
# Date of the current gem release
|
52
|
-
def self.VERSION_DATE
|
51
|
+
# Date of the current gem release
|
52
|
+
def self.VERSION_DATE
|
53
|
+
VERSION_DATE
|
54
|
+
end
|
53
55
|
|
54
56
|
##
|
55
|
-
# Reference of MiGA
|
56
|
-
def self.CITATION
|
57
|
-
|
57
|
+
# Reference of MiGA
|
58
|
+
def self.CITATION
|
59
|
+
CITATION
|
60
|
+
end
|
58
61
|
end
|
data/test/common_test.rb
CHANGED
@@ -1,10 +1,7 @@
|
|
1
1
|
require 'test_helper'
|
2
2
|
|
3
3
|
class CommonTest < Test::Unit::TestCase
|
4
|
-
|
5
|
-
def setup
|
6
|
-
#$jruby_tests = !ENV["JRUBY_TESTS"].nil?
|
7
|
-
end
|
4
|
+
include TestHelper
|
8
5
|
|
9
6
|
def test_debug
|
10
7
|
assert_respond_to(MiGA::MiGA, :DEBUG)
|
@@ -28,7 +25,6 @@ class CommonTest < Test::Unit::TestCase
|
|
28
25
|
assert_respond_to(MiGA::MiGA, :DEBUG)
|
29
26
|
assert_respond_to(MiGA::MiGA, :DEBUG_ON)
|
30
27
|
assert_respond_to(MiGA::MiGA, :DEBUG_OFF)
|
31
|
-
#omit_if($jruby_tests, "JRuby doesn't like interceptions.")
|
32
28
|
MiGA::MiGA.DEBUG_TRACE_ON
|
33
29
|
err = capture_stderr do
|
34
30
|
MiGA::MiGA.DEBUG 'Dandadi'
|
@@ -45,9 +41,8 @@ class CommonTest < Test::Unit::TestCase
|
|
45
41
|
end
|
46
42
|
|
47
43
|
def test_generic_transfer
|
48
|
-
|
49
|
-
|
50
|
-
world = File.expand_path('World', $tmp)
|
44
|
+
hello = File.expand_path('Hello', tmpdir)
|
45
|
+
world = File.expand_path('World', tmpdir)
|
51
46
|
assert_respond_to(File, :generic_transfer)
|
52
47
|
FileUtils.touch(hello)
|
53
48
|
File.generic_transfer(hello, world, :symlink)
|
@@ -67,8 +62,6 @@ class CommonTest < Test::Unit::TestCase
|
|
67
62
|
File.generic_transfer(hello, world, :monkey)
|
68
63
|
end
|
69
64
|
assert_path_not_exist(world, 'A monkey shouldn\'t create files.')
|
70
|
-
ensure
|
71
|
-
FileUtils.rm_rf $tmp
|
72
65
|
end
|
73
66
|
|
74
67
|
def test_miga_name
|
@@ -79,5 +72,4 @@ class CommonTest < Test::Unit::TestCase
|
|
79
72
|
assert_not_predicate('C3-PO', :miga_name?)
|
80
73
|
assert_equal("123\n1\n", '1231'.wrap_width(3))
|
81
74
|
end
|
82
|
-
|
83
75
|
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
##
|
2
|
+
# Helper functions to test daemons.
|
3
|
+
# The class MUST also include +TestHelper+
|
4
|
+
module DaemonHelper
|
5
|
+
def daemon(i = 0)
|
6
|
+
@daemon ||= {}
|
7
|
+
@daemon[project(i).name] ||= MiGA::Daemon.new(project(i))
|
8
|
+
end
|
9
|
+
|
10
|
+
def helper_datasets_with_results(n = 1, project_i = 0)
|
11
|
+
p1 = project(project_i)
|
12
|
+
Array.new(n) do |i|
|
13
|
+
d = "d#{i}"
|
14
|
+
FileUtils.touch(
|
15
|
+
File.join(p1.path, 'data', '02.trimmed_reads', "#{d}.1.clipped.fastq")
|
16
|
+
)
|
17
|
+
FileUtils.touch(
|
18
|
+
File.join(p1.path, 'data', '02.trimmed_reads', "#{d}.done")
|
19
|
+
)
|
20
|
+
p1.add_dataset(MiGA::Dataset.new(p1, d, true).name).tap do |ds|
|
21
|
+
ds.first_preprocessing(true)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def helper_daemon_launch_job(project_i = 0)
|
27
|
+
declare_forks
|
28
|
+
d1 = daemon(project_i)
|
29
|
+
helper_datasets_with_results(1, project_i).first.inactivate!
|
30
|
+
assert_equal(0, d1.jobs_to_run.size, 'The queue should be empty')
|
31
|
+
capture_stderr { d1.check_project }
|
32
|
+
assert_equal(1, d1.jobs_to_run.size, 'The queue should have one job')
|
33
|
+
capture_stderr { d1.flush! }
|
34
|
+
sleep(1)
|
35
|
+
assert_equal(0, d1.jobs_to_run.size, 'There should be nothing running')
|
36
|
+
assert_equal(1, d1.jobs_running.size, 'There should be one job running')
|
37
|
+
end
|
38
|
+
end
|
data/test/daemon_test.rb
CHANGED
@@ -1,69 +1,46 @@
|
|
1
1
|
require 'test_helper'
|
2
|
+
require 'daemon_helper'
|
2
3
|
require 'miga/daemon'
|
3
4
|
|
4
5
|
class DaemonTest < Test::Unit::TestCase
|
6
|
+
include TestHelper
|
7
|
+
include DaemonHelper
|
5
8
|
|
6
9
|
def setup
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
"var":"{{key}}={{value}}","cmd":"echo {{task_name}} >/dev/null",
|
15
|
-
"alive":"echo 1 # {{pid}}","type":"bash","format_version":1}'
|
16
|
-
end
|
17
|
-
$p1 = MiGA::Project.new(File.expand_path('project1', $tmp))
|
18
|
-
$d1 = MiGA::Daemon.new($p1)
|
19
|
-
end
|
20
|
-
|
21
|
-
def teardown
|
22
|
-
FileUtils.rm_rf $tmp
|
23
|
-
ENV['MIGA_HOME'] = nil
|
24
|
-
end
|
25
|
-
|
26
|
-
def helper_datasets_with_results(n = 1)
|
27
|
-
p1 = $p1
|
28
|
-
Array.new(n) do |i|
|
29
|
-
d = "d#{i}"
|
30
|
-
FileUtils.touch(File.expand_path(
|
31
|
-
"data/02.trimmed_reads/#{d}.1.clipped.fastq", p1.path
|
32
|
-
))
|
33
|
-
FileUtils.touch(File.expand_path(
|
34
|
-
"data/02.trimmed_reads/#{d}.done", p1.path
|
35
|
-
))
|
36
|
-
p1.add_dataset(MiGA::Dataset.new(p1, d, true).name).tap do |ds|
|
37
|
-
ds.first_preprocessing(true)
|
38
|
-
end
|
39
|
-
end
|
10
|
+
initialize_miga_home(
|
11
|
+
<<~DAEMON
|
12
|
+
{ "maxjobs": 1, "ppn": 1, "latency": 2, "varsep": " ",
|
13
|
+
"var": "{{key}}={{value}}", "cmd": "echo {{task_name}} >/dev/null",
|
14
|
+
"alive": "echo 1 # {{pid}}", "type": "bash", "format_version": 1 }
|
15
|
+
DAEMON
|
16
|
+
)
|
40
17
|
end
|
41
18
|
|
42
19
|
def test_check_project
|
43
|
-
d1 =
|
44
|
-
helper_datasets_with_results.first.inactivate!
|
45
|
-
out = capture_stderr { d1.check_project }
|
46
|
-
assert_match(/Queueing miga-project:p/, out
|
20
|
+
d1 = daemon(1)
|
21
|
+
helper_datasets_with_results(1, 1).first.inactivate!
|
22
|
+
out = capture_stderr { d1.check_project }.string
|
23
|
+
assert_match(/Queueing miga-project:p/, out)
|
47
24
|
assert_equal(1, d1.jobs_to_run.size)
|
48
25
|
assert_equal(:p, d1.jobs_to_run.first[:job])
|
49
26
|
assert_equal('project1:p:miga-project', d1.get_job(:p)[:task_name])
|
50
27
|
end
|
51
28
|
|
52
29
|
def test_check_datasets
|
53
|
-
p =
|
54
|
-
d =
|
30
|
+
p = project
|
31
|
+
d = daemon
|
55
32
|
d.runopts(:maxjobs, 0, true)
|
56
33
|
assert_empty(d.jobs_to_run)
|
57
34
|
ds = p.add_dataset('ds1')
|
58
35
|
d.check_datasets
|
59
36
|
assert_empty(d.jobs_to_run)
|
60
37
|
FileUtils.cp(
|
61
|
-
File.
|
62
|
-
File.
|
38
|
+
File.join(p.path, 'daemon/daemon.json'),
|
39
|
+
File.join(p.path, 'data/01.raw_reads/ds1.1.fastq')
|
63
40
|
)
|
64
41
|
FileUtils.cp(
|
65
|
-
File.
|
66
|
-
File.
|
42
|
+
File.join(p.path, 'daemon/daemon.json'),
|
43
|
+
File.join(p.path, 'data/01.raw_reads/ds1.done')
|
67
44
|
)
|
68
45
|
ds.first_preprocessing(true)
|
69
46
|
out = capture_stderr do
|
@@ -71,13 +48,13 @@ class DaemonTest < Test::Unit::TestCase
|
|
71
48
|
end
|
72
49
|
assert_match(/Queueing #{ds.name}:d/, out.string)
|
73
50
|
assert_equal(1, d.jobs_to_run.size)
|
74
|
-
assert_equal('echo
|
51
|
+
assert_equal('echo project0:d:ds1 >/dev/null', d.jobs_to_run.first[:cmd])
|
75
52
|
assert_equal(d.jobs_to_run.first, d.get_job(:d, ds))
|
76
53
|
end
|
77
54
|
|
78
55
|
def test_in_loop
|
79
|
-
p =
|
80
|
-
d =
|
56
|
+
p = project
|
57
|
+
d = daemon
|
81
58
|
d.runopts(:latency, 0, true)
|
82
59
|
assert_nil(d.loop_i)
|
83
60
|
assert_nil(d.last_alive)
|
@@ -92,16 +69,17 @@ class DaemonTest < Test::Unit::TestCase
|
|
92
69
|
end
|
93
70
|
|
94
71
|
def test_start
|
95
|
-
p =
|
96
|
-
d =
|
72
|
+
p = project
|
73
|
+
d = daemon
|
97
74
|
d.runopts(:latency, 0, true)
|
98
75
|
assert_equal(0, d.latency)
|
99
|
-
|
100
|
-
|
76
|
+
|
77
|
+
declare_forks
|
78
|
+
child = d.daemon(:start, ['--shush'])
|
101
79
|
assert_not_nil(child)
|
102
80
|
assert_gt(child, 1)
|
103
81
|
assert_equal(0, `ps -p "#{child}" -o ppid=`.strip.to_i,
|
104
|
-
|
82
|
+
'The daemon process should be detached')
|
105
83
|
sleep(3)
|
106
84
|
assert_path_exist(d.pid_file)
|
107
85
|
out = capture_stderr { d.stop }.string
|
@@ -119,23 +97,24 @@ class DaemonTest < Test::Unit::TestCase
|
|
119
97
|
}.each { |k, v| assert_match(v, l[k], "unexpected line: #{k}") }
|
120
98
|
ensure
|
121
99
|
begin
|
122
|
-
Process.kill('KILL',
|
100
|
+
Process.kill('KILL', child) unless child.nil?
|
123
101
|
rescue Errno::ESRCH
|
124
102
|
false
|
125
103
|
end
|
126
104
|
end
|
127
105
|
|
128
106
|
def test_last_alive
|
129
|
-
p = MiGA::Project.new(
|
107
|
+
p = MiGA::Project.new(tmpfile('last_alive'))
|
130
108
|
d = MiGA::Daemon.new(p)
|
131
109
|
assert_nil(d.last_alive)
|
132
|
-
|
110
|
+
|
111
|
+
declare_forks
|
133
112
|
d.declare_alive
|
134
113
|
assert_lt(d.last_alive, Time.now)
|
135
114
|
end
|
136
115
|
|
137
116
|
def test_options
|
138
|
-
d1 =
|
117
|
+
d1 = daemon
|
139
118
|
assert_respond_to(d1, :default_options)
|
140
119
|
assert_equal(:normal, d1.default_options[:dir_mode])
|
141
120
|
assert_equal(2, d1.runopts(:latency))
|
@@ -151,17 +130,17 @@ class DaemonTest < Test::Unit::TestCase
|
|
151
130
|
end
|
152
131
|
|
153
132
|
def test_say
|
154
|
-
out = capture_stderr {
|
133
|
+
out = capture_stderr { daemon.say 'Olm' }.string
|
155
134
|
assert_match(/^\[.*\] Olm/, out)
|
156
135
|
end
|
157
136
|
|
158
137
|
def test_terminate
|
159
|
-
d =
|
138
|
+
d = daemon
|
160
139
|
assert_not_predicate(d, :active?)
|
161
140
|
assert_path_not_exist(d.alive_file)
|
162
141
|
assert_path_not_exist(d.terminated_file)
|
163
142
|
|
164
|
-
|
143
|
+
declare_forks
|
165
144
|
d.declare_alive
|
166
145
|
assert_predicate(d, :active?)
|
167
146
|
assert_not_nil(d.last_alive)
|
@@ -176,7 +155,7 @@ class DaemonTest < Test::Unit::TestCase
|
|
176
155
|
end
|
177
156
|
|
178
157
|
def test_maxjobs_json
|
179
|
-
d1 =
|
158
|
+
d1 = daemon
|
180
159
|
helper_datasets_with_results(3)
|
181
160
|
assert_equal(0, d1.jobs_running.size)
|
182
161
|
assert_equal(0, d1.jobs_to_run.size)
|
@@ -186,7 +165,7 @@ class DaemonTest < Test::Unit::TestCase
|
|
186
165
|
end
|
187
166
|
|
188
167
|
def test_maxjobs_runopts
|
189
|
-
d1 =
|
168
|
+
d1 = daemon
|
190
169
|
helper_datasets_with_results(3)
|
191
170
|
d1.runopts(:maxjobs, 2)
|
192
171
|
assert_equal(0, d1.jobs_running.size)
|
@@ -197,8 +176,8 @@ class DaemonTest < Test::Unit::TestCase
|
|
197
176
|
end
|
198
177
|
|
199
178
|
def test_load_status
|
200
|
-
d1 =
|
201
|
-
p1 =
|
179
|
+
d1 = daemon
|
180
|
+
p1 = project
|
202
181
|
assert_equal(0, d1.jobs_running.size)
|
203
182
|
assert_nil(d1.load_status)
|
204
183
|
assert_equal(0, d1.jobs_running.size)
|
@@ -215,8 +194,8 @@ class DaemonTest < Test::Unit::TestCase
|
|
215
194
|
end
|
216
195
|
|
217
196
|
def test_flush
|
218
|
-
d1 =
|
219
|
-
p1 =
|
197
|
+
d1 = daemon
|
198
|
+
p1 = project
|
220
199
|
helper_datasets_with_results
|
221
200
|
p1.add_dataset(MiGA::Dataset.new(p1, 'd1').name)
|
222
201
|
MiGA::Project.RESULT_DIRS.keys.each { |i| p1.metadata["run_#{i}"] = false }
|
@@ -235,8 +214,8 @@ class DaemonTest < Test::Unit::TestCase
|
|
235
214
|
end
|
236
215
|
|
237
216
|
def test_next_host
|
238
|
-
d1 =
|
239
|
-
f =
|
217
|
+
d1 = daemon
|
218
|
+
f = tmpfile('nodes.txt')
|
240
219
|
File.open(f, 'w') { |h| h.puts 'localhost' }
|
241
220
|
assert_equal(true, d1.next_host)
|
242
221
|
out = capture_stderr { d1.runopts(:nodelist, f) }.string
|
@@ -244,7 +223,7 @@ class DaemonTest < Test::Unit::TestCase
|
|
244
223
|
assert_equal(true, d1.next_host)
|
245
224
|
d1.runopts(:type, 'ssh')
|
246
225
|
assert_equal(0, d1.next_host)
|
247
|
-
f = File.join(
|
226
|
+
f = File.join(project.path, 'daemon', 'status.json')
|
248
227
|
File.open(f, 'w') do |h|
|
249
228
|
h.puts '{"jobs_running":[{"job":"p","hostk":0}], "jobs_to_run":[]}'
|
250
229
|
end
|
@@ -253,60 +232,66 @@ class DaemonTest < Test::Unit::TestCase
|
|
253
232
|
end
|
254
233
|
|
255
234
|
def test_shutdown_when_done
|
256
|
-
|
257
|
-
out = capture_stderr { assert {
|
235
|
+
daemon.runopts(:shutdown_when_done, true)
|
236
|
+
out = capture_stderr { assert { !daemon.in_loop } }.string
|
258
237
|
assert_match(/Nothing else to do/, out)
|
259
238
|
end
|
260
239
|
|
261
240
|
def test_update_format_0
|
262
|
-
f =
|
241
|
+
f = tmpfile('daemon.json')
|
263
242
|
File.open(f, 'w') do |fh|
|
264
|
-
fh.puts
|
265
|
-
|
243
|
+
fh.puts(
|
244
|
+
<<~DAEMON
|
245
|
+
{ "maxjobs": 1, "ppn": 1, "latency": 2, "varsep": " ",
|
246
|
+
"var": "%1$s=%1$s", "cmd": "echo %1$s", "alive": "echo %1$d",
|
247
|
+
"type": "bash" }
|
248
|
+
DAEMON
|
249
|
+
)
|
266
250
|
end
|
267
|
-
d2 = MiGA::Daemon.new(
|
251
|
+
d2 = MiGA::Daemon.new(project, f)
|
268
252
|
assert_equal('echo {{script}}', d2.runopts(:cmd))
|
269
253
|
assert_equal('echo {{pid}}', d2.runopts(:alive))
|
270
254
|
end
|
271
255
|
|
272
256
|
def test_launch_job_bash
|
273
|
-
t =
|
274
|
-
|
275
|
-
|
257
|
+
t = tmpfile('launch_job_bash')
|
258
|
+
daemon.runopts(:type, 'bash')
|
259
|
+
daemon.runopts(:cmd, "echo {{task_name}} > '#{t}'")
|
276
260
|
helper_daemon_launch_job
|
277
|
-
assert_equal("
|
261
|
+
assert_equal("project0:p:miga-project\n", File.read(t))
|
278
262
|
end
|
279
263
|
|
280
264
|
def test_launch_job_ssh
|
281
|
-
d1 =
|
282
|
-
t =
|
265
|
+
d1 = daemon(1)
|
266
|
+
t = tmpfile('launch_job_ssh')
|
283
267
|
d1.runopts(:type, 'ssh')
|
284
268
|
d1.runopts(:cmd, "echo {{task_name}} > '#{t}'")
|
285
|
-
f =
|
269
|
+
f = tmpfile('nodes.txt')
|
286
270
|
File.open(f, 'w') { |h| h.puts 'localhost' }
|
287
271
|
assert_raise('Unset environment variable: $MIGA_TEST_NODELIST') do
|
288
272
|
d1.runopts(:nodelist, '$MIGA_TEST_NODELIST')
|
289
273
|
end
|
290
274
|
ENV['MIGA_TEST_NODELIST'] = f
|
291
275
|
capture_stderr { d1.runopts(:nodelist, '$MIGA_TEST_NODELIST') }
|
292
|
-
helper_daemon_launch_job
|
276
|
+
helper_daemon_launch_job(1)
|
293
277
|
assert_equal("project1:p:miga-project\n", File.read(t))
|
294
278
|
end
|
295
279
|
|
296
280
|
def test_launch_job_qsub
|
297
|
-
|
298
|
-
|
281
|
+
daemon.runopts(:type, 'qsub')
|
282
|
+
daemon.runopts(:cmd, 'echo {{task_name}}')
|
299
283
|
helper_daemon_launch_job
|
300
|
-
assert_equal('
|
284
|
+
assert_equal('project0:p:miga-project', daemon.jobs_running.first[:pid])
|
301
285
|
end
|
302
286
|
|
303
287
|
def test_launch_job_failure
|
304
|
-
d1 =
|
288
|
+
d1 = daemon(1)
|
305
289
|
d1.runopts(:type, 'qsub')
|
306
290
|
d1.runopts(:cmd, 'echo ""')
|
307
|
-
helper_datasets_with_results.first.inactivate!
|
291
|
+
helper_datasets_with_results(1, 1).first.inactivate!
|
308
292
|
capture_stderr { d1.check_project }
|
309
|
-
|
293
|
+
|
294
|
+
declare_forks
|
310
295
|
out = capture_stderr { d1.launch_job(d1.jobs_to_run.shift) }.string
|
311
296
|
assert_match(/Unsuccessful project1:p:miga-project, rescheduling/, out)
|
312
297
|
assert_equal(0, d1.jobs_running.size)
|
@@ -314,7 +299,7 @@ class DaemonTest < Test::Unit::TestCase
|
|
314
299
|
end
|
315
300
|
|
316
301
|
def test_verbosity
|
317
|
-
d1 =
|
302
|
+
d1 = daemon
|
318
303
|
d1.runopts(:verbosity, 0)
|
319
304
|
out = capture_stderr { d1.in_loop }.string
|
320
305
|
assert_empty(out)
|
@@ -332,17 +317,4 @@ class DaemonTest < Test::Unit::TestCase
|
|
332
317
|
out = capture_stderr { d1.in_loop }.string
|
333
318
|
assert_match(/Daemon loop start/, out)
|
334
319
|
end
|
335
|
-
|
336
|
-
def helper_daemon_launch_job
|
337
|
-
omit_if($jruby_tests, 'JRuby doesn\'t implement fork.')
|
338
|
-
d1 = $d1
|
339
|
-
helper_datasets_with_results.first.inactivate!
|
340
|
-
assert_equal(0, d1.jobs_to_run.size, 'The queue should be empty')
|
341
|
-
capture_stderr { d1.check_project }
|
342
|
-
assert_equal(1, d1.jobs_to_run.size, 'The queue should have one job')
|
343
|
-
capture_stderr { d1.flush! }
|
344
|
-
sleep(1)
|
345
|
-
assert_equal(0, d1.jobs_to_run.size, 'There should be nothing running')
|
346
|
-
assert_equal(1, d1.jobs_running.size, 'There should be one job running')
|
347
|
-
end
|
348
320
|
end
|