miga-base 0.6.3.0 → 0.6.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/miga/cli/action/init/daemon_helper.rb +1 -1
- data/lib/miga/cli/action/wf.rb +1 -0
- data/lib/miga/common/with_result.rb +70 -0
- data/lib/miga/daemon.rb +3 -1
- data/lib/miga/dataset.rb +1 -13
- data/lib/miga/dataset/hooks.rb +2 -11
- data/lib/miga/dataset/result.rb +44 -82
- data/lib/miga/project/hooks.rb +2 -11
- data/lib/miga/project/result.rb +102 -121
- data/lib/miga/remote_dataset.rb +2 -1
- data/lib/miga/result.rb +20 -11
- data/lib/miga/result/source.rb +7 -3
- data/lib/miga/result/stats.rb +11 -0
- data/lib/miga/tax_dist.rb +5 -2
- data/lib/miga/version.rb +2 -2
- data/scripts/cds.bash +1 -1
- data/test/daemon_test.rb +1 -1
- data/test/project_test.rb +58 -65
- data/test/remote_dataset_test.rb +49 -4
- data/test/tax_dist_test.rb +0 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9b94d26cedfe4ef7f4fd537dd668777c9ee3820eeb670a106e744250cdc72566
|
4
|
+
data.tar.gz: 37cfd446014fe396ef6ce15d5289e11118095349d400aeefe16f84985f77c27d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 44e9dbe946ab889bb8bef0423544cec4d44894f9617ae28bf9ca33094ceaae16a662816110af42fc8a8941834bdf47f623eb6ec42afb3f0da405bb1267503d30
|
7
|
+
data.tar.gz: 30f712d1f2d9c540d571f66cd5b8b5d21e70540f85ec2b126f11949ea2029d935d5344594817c543d81ce26917c6e5d837798b185000d079c8c700a2ac6f13d8
|
@@ -118,7 +118,7 @@ module MiGA::Cli::Action::Init::DaemonHelper
|
|
118
118
|
v
|
119
119
|
end
|
120
120
|
|
121
|
-
def configure_qsub_msub_daemon
|
121
|
+
def configure_qsub_msub_daemon(v)
|
122
122
|
queue = cli.ask_user('What queue should I use?', nil, nil, true)
|
123
123
|
v[:latency] = cli.ask_user('How long should I sleep? (in secs)', '150').to_i
|
124
124
|
v[:maxjobs] = cli.ask_user('How many jobs can I launch at once?', '300').to_i
|
data/lib/miga/cli/action/wf.rb
CHANGED
@@ -0,0 +1,70 @@
|
|
1
|
+
|
2
|
+
##
|
3
|
+
# Helper module including specific functions to handle objects that
|
4
|
+
# have results.
|
5
|
+
module MiGA::Common::WithResult
|
6
|
+
##
|
7
|
+
# Result directories as a Hash
|
8
|
+
def result_dirs
|
9
|
+
self.class.RESULT_DIRS
|
10
|
+
end
|
11
|
+
|
12
|
+
##
|
13
|
+
# Look for the result with symbol key +task+ and register it in the object.
|
14
|
+
# If +save+ is false, it doesn't register the result, but it still returns a
|
15
|
+
# result if it already exists.
|
16
|
+
#
|
17
|
+
# The +opts+ hash controls result creation (if necessary).
|
18
|
+
# Supported values include:
|
19
|
+
# - +is_clean+: A Boolean indicating if the input files are clean
|
20
|
+
# - +force+: A Boolean indicating if the result must be re-indexed,
|
21
|
+
# ignored unless +save = true+
|
22
|
+
#
|
23
|
+
# Returns MiGA::Result or nil
|
24
|
+
def add_result(task, save = true, opts = {})
|
25
|
+
task = task.to_sym
|
26
|
+
return nil if result_dirs[task].nil?
|
27
|
+
base = File.join(
|
28
|
+
project.path, "data/#{result_dirs[task]}/#{result_base}"
|
29
|
+
)
|
30
|
+
json = "#{base}.json"
|
31
|
+
return MiGA::Result.load(json) unless save
|
32
|
+
|
33
|
+
MiGA::Result.create(json, opts[:force]) do
|
34
|
+
r = send("add_result_#{task}", base, opts) if File.exist?("#{base}.done")
|
35
|
+
unless r.nil?
|
36
|
+
r.save
|
37
|
+
pull_hook(:on_result_ready, r.key)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
##
|
43
|
+
# Get the result MiGA::Result in this object identified by the symbol +task+
|
44
|
+
def result(task)
|
45
|
+
task = task.to_sym
|
46
|
+
return nil if result_dirs[task].nil?
|
47
|
+
MiGA::Result.load(
|
48
|
+
"#{project.path}/data/#{result_dirs[task]}/#{result_base}.json"
|
49
|
+
)
|
50
|
+
end
|
51
|
+
|
52
|
+
##
|
53
|
+
# Get all the results (Array of MiGA::Result) in this object
|
54
|
+
def results
|
55
|
+
result_dirs.keys.map { |k| result k }.compact
|
56
|
+
end
|
57
|
+
|
58
|
+
##
|
59
|
+
# For each result execute the 2-ary block: key symbol and MiGA::Result
|
60
|
+
def each_result
|
61
|
+
results.each { |res| yield(res.key, res) }
|
62
|
+
end
|
63
|
+
|
64
|
+
##
|
65
|
+
# Get a result as MiGA::Result for the object with key +task+.
|
66
|
+
# This is equivalent to +add_result(task, false)+.
|
67
|
+
def get_result(task)
|
68
|
+
add_result(task, false)
|
69
|
+
end
|
70
|
+
end
|
data/lib/miga/daemon.rb
CHANGED
@@ -198,7 +198,7 @@ class MiGA::Daemon < MiGA::MiGA
|
|
198
198
|
@jobs_running.select! do |job|
|
199
199
|
ongoing = case job[:job].to_s
|
200
200
|
when 'd'
|
201
|
-
!job[:ds].next_preprocessing(false).nil?
|
201
|
+
!job[:ds].nil? && !job[:ds].next_preprocessing(false).nil?
|
202
202
|
when 'p'
|
203
203
|
!project.next_task(nil, false).nil?
|
204
204
|
else
|
@@ -244,6 +244,8 @@ class MiGA::Daemon < MiGA::MiGA
|
|
244
244
|
say 'MiGA:%s launched' % project.name
|
245
245
|
say '-----------------------------------'
|
246
246
|
load_status
|
247
|
+
say 'Configuration options:'
|
248
|
+
say @runopts.to_s
|
247
249
|
@loop_i = 0
|
248
250
|
end
|
249
251
|
@loop_i += 1
|
data/lib/miga/dataset.rb
CHANGED
@@ -143,18 +143,6 @@ class MiGA::Dataset < MiGA::MiGA
|
|
143
143
|
metadata[:inactive].nil? or !metadata[:inactive]
|
144
144
|
end
|
145
145
|
|
146
|
-
##
|
147
|
-
# Should I ignore +task+ for this dataset?
|
148
|
-
def ignore_task?(task)
|
149
|
-
return true unless is_active?
|
150
|
-
return !metadata["run_#{task}"] unless metadata["run_#{task}"].nil?
|
151
|
-
return true if task == :taxonomy and project.metadata[:ref_project].nil?
|
152
|
-
pattern = [true, false]
|
153
|
-
( [@@_EXCLUDE_NOREF_TASKS_H[task], is_ref? ] == pattern or
|
154
|
-
[@@_ONLY_MULTI_TASKS_H[task], is_multi? ] == pattern or
|
155
|
-
[@@_ONLY_NONMULTI_TASKS_H[task], is_nonmulti?] == pattern )
|
156
|
-
end
|
157
|
-
|
158
146
|
##
|
159
147
|
# Returns an Array of +how_many+ duples (Arrays) sorted by AAI:
|
160
148
|
# - +0+: A String with the name(s) of the reference dataset.
|
@@ -172,5 +160,5 @@ class MiGA::Dataset < MiGA::MiGA
|
|
172
160
|
'GROUP BY seq2 ORDER BY aai DESC LIMIT ?', [name, how_many])
|
173
161
|
end
|
174
162
|
|
175
|
-
end
|
163
|
+
end
|
176
164
|
|
data/lib/miga/dataset/hooks.rb
CHANGED
@@ -17,7 +17,6 @@ require 'miga/common/hooks'
|
|
17
17
|
# - clear_run_counts()
|
18
18
|
# - run_cmd(cmd)
|
19
19
|
# Internal hooks:
|
20
|
-
# - _pull_preprocessing_ready_hooks()
|
21
20
|
# - _pull_result_hooks()
|
22
21
|
module MiGA::Dataset::Hooks
|
23
22
|
|
@@ -26,10 +25,7 @@ module MiGA::Dataset::Hooks
|
|
26
25
|
def default_hooks
|
27
26
|
{
|
28
27
|
on_preprocessing_ready: [[:clear_run_counts]],
|
29
|
-
on_result_ready: [
|
30
|
-
[:_pull_result_hooks],
|
31
|
-
[:_pull_preprocessing_ready_hooks]
|
32
|
-
]
|
28
|
+
on_result_ready: [[:_pull_result_hooks]]
|
33
29
|
}
|
34
30
|
end
|
35
31
|
|
@@ -57,17 +53,12 @@ module MiGA::Dataset::Hooks
|
|
57
53
|
)
|
58
54
|
end
|
59
55
|
|
60
|
-
##
|
61
|
-
# Pull :dataset_ready hook if preprocessing is complete
|
62
|
-
def hook__pull_preprocessing_ready_hooks(_hook_args, _event_args)
|
63
|
-
pull_hook(:on_preprocessing_ready) if done_preprocessing?
|
64
|
-
end
|
65
|
-
|
66
56
|
##
|
67
57
|
# Dataset Action :pull_result_hooks([], [res])
|
68
58
|
# Pull the hook specific to the type of result
|
69
59
|
def hook__pull_result_hooks(_hook_args, event_args)
|
70
60
|
pull_hook(:"on_result_ready_#{event_args.first}", *event_args)
|
61
|
+
pull_hook(:on_preprocessing_ready) if done_preprocessing?
|
71
62
|
end
|
72
63
|
|
73
64
|
end
|
data/lib/miga/dataset/result.rb
CHANGED
@@ -1,68 +1,53 @@
|
|
1
|
-
|
2
1
|
require 'sqlite3'
|
3
2
|
require 'miga/result'
|
4
3
|
require 'miga/dataset/base'
|
4
|
+
require 'miga/common/with_result'
|
5
5
|
|
6
6
|
##
|
7
7
|
# Helper module including specific functions to add dataset results
|
8
8
|
module MiGA::Dataset::Result
|
9
9
|
include MiGA::Dataset::Base
|
10
|
+
include MiGA::Common::WithResult
|
10
11
|
|
11
12
|
##
|
12
|
-
#
|
13
|
-
def
|
14
|
-
|
15
|
-
MiGA::Result.load(
|
16
|
-
"#{project.path}/data/#{@@RESULT_DIRS[k.to_sym]}/#{name}.json"
|
17
|
-
)
|
13
|
+
# Return the basename for results
|
14
|
+
def result_base
|
15
|
+
name
|
18
16
|
end
|
19
17
|
|
20
18
|
##
|
21
|
-
#
|
22
|
-
def
|
23
|
-
|
19
|
+
# Should I ignore +task+ for this dataset?
|
20
|
+
def ignore_task?(task)
|
21
|
+
why_ignore(task) != :execute
|
24
22
|
end
|
25
23
|
|
26
24
|
##
|
27
|
-
#
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
#
|
36
|
-
#
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
25
|
+
# Return a code explaining why a task is ignored.
|
26
|
+
# The values are symbols:
|
27
|
+
# - empty: the dataset has no data
|
28
|
+
# - inactive: the dataset is inactive
|
29
|
+
# - force: forced to ignore by metadata
|
30
|
+
# - project: incompatible project
|
31
|
+
# - noref: incompatible dataset, only for reference
|
32
|
+
# - multi: incompatible dataset, only for multi
|
33
|
+
# - nonmulti: incompatible dataset, only for nonmulti
|
34
|
+
# - execute: do not ignore, execute the task
|
35
|
+
def why_ignore(task)
|
36
|
+
if !is_active?
|
37
|
+
:inactive
|
38
|
+
elsif !metadata["run_#{task}"].nil?
|
39
|
+
metadata["run_#{task}"] ? :execute : :force
|
40
|
+
elsif task == :taxonomy && project.metadata[:ref_project].nil?
|
41
|
+
:project
|
42
|
+
elsif @@_EXCLUDE_NOREF_TASKS_H[task] && !is_ref?
|
43
|
+
:noref
|
44
|
+
elsif @@_ONLY_MULTI_TASKS_H[task] && !is_multi?
|
45
|
+
:multi
|
46
|
+
elsif @@_ONLY_NONMULTI_TASKS_H[task] && !is_nonmulti?
|
47
|
+
:nonmulti
|
49
48
|
else
|
50
|
-
|
51
|
-
return r_pre if (r_pre.nil? && !save) || !r_pre.nil?
|
49
|
+
:execute
|
52
50
|
end
|
53
|
-
fun = "add_result_#{result_type}"
|
54
|
-
r = send(fun, base, opts) if File.exist?("#{base}.done")
|
55
|
-
return if r.nil?
|
56
|
-
r.save
|
57
|
-
pull_hook(:on_result_ready, result_type)
|
58
|
-
r
|
59
|
-
end
|
60
|
-
|
61
|
-
##
|
62
|
-
# Gets a result as MiGA::Result for the datasets with +result_type+. This is
|
63
|
-
# equivalent to +add_result(result_type, false)+.
|
64
|
-
def get_result(result_type)
|
65
|
-
add_result(result_type, false)
|
66
51
|
end
|
67
52
|
|
68
53
|
##
|
@@ -76,24 +61,22 @@ module MiGA::Dataset::Result
|
|
76
61
|
end
|
77
62
|
|
78
63
|
##
|
79
|
-
# Returns the key symbol of the next task that needs to be executed.
|
80
|
-
# +save+ to #add_result.
|
64
|
+
# Returns the key symbol of the next task that needs to be executed or nil.
|
65
|
+
# Passes +save+ to #add_result.
|
81
66
|
def next_preprocessing(save = false)
|
82
|
-
|
83
|
-
first
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
if after_first && add_result(t, save).nil?
|
67
|
+
first = first_preprocessing(save) or return nil
|
68
|
+
@@PREPROCESSING_TASKS[@@PREPROCESSING_TASKS.index(first) .. -1].find do |t|
|
69
|
+
if ignore_task? t
|
70
|
+
false
|
71
|
+
elsif add_result(t, save).nil?
|
88
72
|
if (metadata["_try_#{t}"] || 0) > (project.metadata[:max_try] || 10)
|
89
73
|
inactivate!
|
90
|
-
|
74
|
+
false
|
75
|
+
else
|
76
|
+
true
|
91
77
|
end
|
92
|
-
return t
|
93
78
|
end
|
94
|
-
after_first = (after_first || (t == first))
|
95
79
|
end
|
96
|
-
nil
|
97
80
|
end
|
98
81
|
|
99
82
|
##
|
@@ -133,14 +116,7 @@ module MiGA::Dataset::Result
|
|
133
116
|
##
|
134
117
|
# Returns the status of +task+. The status values are symbols:
|
135
118
|
# - -: the task is upstream from the initial input
|
136
|
-
# -
|
137
|
-
# - ignore_inactive: the dataset is inactive
|
138
|
-
# - ignore_force: forced to ignore by metadata
|
139
|
-
# - ignore_project: incompatible project
|
140
|
-
# - ignore_noref: incompatible dataset, only for reference
|
141
|
-
# - ignore_multi: incompatible dataset, only for multi
|
142
|
-
# - ignore_nonmulti: incompatible dataset, only for nonmulti
|
143
|
-
# - ignore: incompatible dataset, unknown reason
|
119
|
+
# - ignore_*: the task is to be ignored, see codes in #why_ignore
|
144
120
|
# - complete: a task with registered results
|
145
121
|
# - pending: a task queued to be performed
|
146
122
|
def result_status(task)
|
@@ -152,21 +128,7 @@ module MiGA::Dataset::Result
|
|
152
128
|
@@PREPROCESSING_TASKS.index(first_preprocessing)
|
153
129
|
:-
|
154
130
|
elsif ignore_task?(task)
|
155
|
-
|
156
|
-
:ignore_inactive
|
157
|
-
elsif metadata["run_#{task}"]
|
158
|
-
:ignore_force
|
159
|
-
elsif task == :taxonomy && project.metadata[:ref_project].nil?
|
160
|
-
:ignore_project
|
161
|
-
elsif @@_EXCLUDE_NOREF_TASKS_H[task] && !is_ref?
|
162
|
-
:ignore_noref
|
163
|
-
elsif @@_ONLY_MULTI_TASKS_H[task] && !is_multi?
|
164
|
-
:ignore_multi
|
165
|
-
elsif @@_ONLY_NONMULTI_TASKS_H[task] && !is_nonmulti?
|
166
|
-
:ignore_nonmulti
|
167
|
-
else
|
168
|
-
:ignore
|
169
|
-
end
|
131
|
+
:"ignore_#{why_ignore task}"
|
170
132
|
else
|
171
133
|
:pending
|
172
134
|
end
|
data/lib/miga/project/hooks.rb
CHANGED
@@ -16,7 +16,6 @@ require 'miga/common/hooks'
|
|
16
16
|
# - run_lambda(lambda, args...)
|
17
17
|
# - run_cmd(cmd)
|
18
18
|
# Internal hooks:
|
19
|
-
# - _pull_processing_ready_hooks()
|
20
19
|
# - _pull_result_hooks()
|
21
20
|
module MiGA::Project::Hooks
|
22
21
|
|
@@ -24,10 +23,7 @@ module MiGA::Project::Hooks
|
|
24
23
|
|
25
24
|
def default_hooks
|
26
25
|
{
|
27
|
-
on_result_ready: [
|
28
|
-
[:_pull_result_hooks],
|
29
|
-
[:_pull_processing_ready_hooks]
|
30
|
-
]
|
26
|
+
on_result_ready: [[:_pull_result_hooks]]
|
31
27
|
}
|
32
28
|
end
|
33
29
|
|
@@ -44,17 +40,12 @@ module MiGA::Project::Hooks
|
|
44
40
|
)
|
45
41
|
end
|
46
42
|
|
47
|
-
##
|
48
|
-
# Pull :dataset_ready hook if preprocessing is complete
|
49
|
-
def hook__pull_processing_ready_hooks(_hook_args, _event_args)
|
50
|
-
pull_hook(:on_processing_ready) if next_task(nil, false).nil?
|
51
|
-
end
|
52
|
-
|
53
43
|
##
|
54
44
|
# Dataset Action :pull_result_hooks([], [res])
|
55
45
|
# Pull the hook specific to the type of result
|
56
46
|
def hook__pull_result_hooks(_hook_args, event_args)
|
57
47
|
pull_hook(:"on_result_ready_#{event_args.first}", *event_args)
|
48
|
+
pull_hook(:on_processing_ready) if next_task(nil, false).nil?
|
58
49
|
end
|
59
50
|
|
60
51
|
end
|
data/lib/miga/project/result.rb
CHANGED
@@ -3,69 +3,47 @@
|
|
3
3
|
|
4
4
|
require 'miga/result'
|
5
5
|
require 'miga/project/base'
|
6
|
+
require 'miga/common/with_result'
|
6
7
|
|
7
8
|
##
|
8
9
|
# Helper module including specific functions to add project results.
|
9
10
|
module MiGA::Project::Result
|
10
|
-
|
11
11
|
include MiGA::Project::Base
|
12
|
+
include MiGA::Common::WithResult
|
12
13
|
|
13
14
|
##
|
14
|
-
#
|
15
|
-
def
|
16
|
-
|
17
|
-
return nil if dir.nil?
|
18
|
-
MiGA::Result.load("#{path}/data/#{dir}/miga-project.json")
|
15
|
+
# Return the basename for results
|
16
|
+
def result_base
|
17
|
+
'miga-project'
|
19
18
|
end
|
20
19
|
|
21
20
|
##
|
22
|
-
#
|
23
|
-
def
|
24
|
-
|
25
|
-
end
|
26
|
-
|
27
|
-
##
|
28
|
-
# For each result executes the 2-ary +blk+ block: key symbol and MiGA::Result.
|
29
|
-
def each_result(&blk)
|
30
|
-
@@RESULT_DIRS.keys.each do |k|
|
31
|
-
blk.call(k, result(k)) unless result(k).nil?
|
32
|
-
end
|
21
|
+
# Return itself, to simplify modules
|
22
|
+
def project
|
23
|
+
self
|
33
24
|
end
|
34
25
|
|
35
26
|
##
|
36
|
-
#
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
# it implies save=true.
|
42
|
-
def add_result(name, save = true, opts = {})
|
43
|
-
return nil if @@RESULT_DIRS[name].nil?
|
44
|
-
base = "#{path}/data/#{@@RESULT_DIRS[name]}/miga-project"
|
45
|
-
if opts[:force]
|
46
|
-
FileUtils.rm("#{base}.json") if File.exist?("#{base}.json")
|
47
|
-
else
|
48
|
-
r_pre = MiGA::Result.load("#{base}.json")
|
49
|
-
return r_pre if (r_pre.nil? && !save) || !r_pre.nil?
|
50
|
-
end
|
51
|
-
r = result_files_exist?(base, ".done") ?
|
52
|
-
send("add_result_#{name}", base) : nil
|
53
|
-
unless r.nil?
|
54
|
-
r.save
|
55
|
-
pull_hook(:on_result_ready, name)
|
56
|
-
end
|
57
|
-
r
|
27
|
+
# Is this +task+ to be bypassed?
|
28
|
+
def ignore_task?(task)
|
29
|
+
metadata["run_#{task}"] == false ||
|
30
|
+
(!is_clade? && @@INCLADE_TASKS.include?(task) &&
|
31
|
+
metadata["run_#{task}"] != true)
|
58
32
|
end
|
59
33
|
|
60
34
|
##
|
61
35
|
# Get the next distances task, saving intermediate results if +save+. Returns
|
62
36
|
# a Symbol.
|
63
|
-
def next_distances(save = true)
|
37
|
+
def next_distances(save = true)
|
38
|
+
next_task(@@DISTANCE_TASKS, save)
|
39
|
+
end
|
64
40
|
|
65
41
|
##
|
66
42
|
# Get the next inclade task, saving intermediate results if +save+. Returns a
|
67
43
|
# Symbol.
|
68
|
-
def next_inclade(save = true)
|
44
|
+
def next_inclade(save = true)
|
45
|
+
next_task(@@INCLADE_TASKS, save)
|
46
|
+
end
|
69
47
|
|
70
48
|
##
|
71
49
|
# Get the next task from +tasks+, saving intermediate results if +save+.
|
@@ -74,101 +52,104 @@ module MiGA::Project::Result
|
|
74
52
|
def next_task(tasks = nil, save = true)
|
75
53
|
tasks ||= @@DISTANCE_TASKS + @@INCLADE_TASKS
|
76
54
|
tasks.find do |t|
|
77
|
-
|
78
|
-
(!is_clade? and @@INCLADE_TASKS.include?(t) and
|
79
|
-
metadata["run_#{t}"] != true)
|
80
|
-
false
|
81
|
-
else
|
82
|
-
add_result(t, save).nil?
|
83
|
-
end
|
55
|
+
ignore_task?(t) ? false : add_result(t, save).nil?
|
84
56
|
end
|
85
57
|
end
|
86
58
|
|
87
|
-
|
88
59
|
private
|
89
60
|
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
61
|
+
##
|
62
|
+
# Add result of any type +:*_distances+ at +base+ (no +_opts+ supported).
|
63
|
+
def add_result_distances(base, _opts)
|
64
|
+
return nil unless result_files_exist?(base, %w[.Rdata .log .txt])
|
65
|
+
r = MiGA::Result.new("#{base}.json")
|
66
|
+
r.add_file(:rdata, 'miga-project.Rdata')
|
67
|
+
r.add_file(:matrix, 'miga-project.txt')
|
68
|
+
r.add_file(:log, 'miga-project.log')
|
69
|
+
r.add_file(:hist, 'miga-project.hist')
|
70
|
+
r
|
71
|
+
end
|
101
72
|
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
return
|
109
|
-
return nil unless is_clade? or result_files_exist?(base,
|
110
|
-
%w[.pdf .classif .medoids .class.tsv .class.nwk])
|
111
|
-
r = add_result_iter_clades(base)
|
112
|
-
r.add_file(:aai_tree, 'miga-project.aai.nwk')
|
113
|
-
r.add_file(:proposal, 'miga-project.proposed-clades')
|
114
|
-
r.add_file(:clades_aai90, 'miga-project.aai90-clades')
|
115
|
-
r.add_file(:clades_ani95, 'miga-project.ani95-clades')
|
116
|
-
r.add_file(:clades_gsp, 'miga-project.gsp-clades')
|
117
|
-
r.add_file(:medoids_gsp, 'miga-project.gsp-medoids')
|
118
|
-
r
|
73
|
+
##
|
74
|
+
# Add result type +:clade_finding+ at +base+ (no +_opts+ supported).
|
75
|
+
def add_result_clade_finding(base, _opts)
|
76
|
+
if result_files_exist?(base, %w[.empty])
|
77
|
+
r = MiGA::Result.new("#{base}.json")
|
78
|
+
r.add_file(:empty, 'miga-project.empty')
|
79
|
+
return r
|
119
80
|
end
|
120
|
-
|
121
|
-
|
122
|
-
if result_files_exist?(base, %w[.empty])
|
123
|
-
r = MiGA::Result.new("#{base}.json")
|
124
|
-
r.add_file(:empty, 'miga-project.empty')
|
125
|
-
return r
|
126
|
-
end
|
127
|
-
return nil unless result_files_exist?(base,
|
81
|
+
return nil unless result_files_exist?(base, %w[.proposed-clades])
|
82
|
+
return nil unless is_clade? or result_files_exist?(base,
|
128
83
|
%w[.pdf .classif .medoids .class.tsv .class.nwk])
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
84
|
+
r = add_result_iter_clades(base)
|
85
|
+
r.add_file(:aai_tree, 'miga-project.aai.nwk')
|
86
|
+
r.add_file(:proposal, 'miga-project.proposed-clades')
|
87
|
+
r.add_file(:clades_aai90, 'miga-project.aai90-clades')
|
88
|
+
r.add_file(:clades_ani95, 'miga-project.ani95-clades')
|
89
|
+
r.add_file(:clades_gsp, 'miga-project.gsp-clades')
|
90
|
+
r.add_file(:medoids_gsp, 'miga-project.gsp-medoids')
|
91
|
+
r
|
92
|
+
end
|
133
93
|
|
134
|
-
|
94
|
+
##
|
95
|
+
# Add result type +:subclades+ at +base+ (no +_opts+ supported).
|
96
|
+
def add_result_subclades(base, _opts)
|
97
|
+
if result_files_exist?(base, %w[.empty])
|
135
98
|
r = MiGA::Result.new("#{base}.json")
|
136
|
-
r.add_file(:
|
137
|
-
r
|
138
|
-
r.add_file(:class_tree, 'miga-project.class.nwk')
|
139
|
-
r.add_file(:classif, 'miga-project.classif')
|
140
|
-
r.add_file(:medoids, 'miga-project.medoids')
|
141
|
-
r
|
99
|
+
r.add_file(:empty, 'miga-project.empty')
|
100
|
+
return r
|
142
101
|
end
|
102
|
+
return nil unless result_files_exist?(base,
|
103
|
+
%w[.pdf .classif .medoids .class.tsv .class.nwk])
|
104
|
+
r = add_result_iter_clades(base)
|
105
|
+
r.add_file(:ani_tree, 'miga-project.ani.nwk')
|
106
|
+
r
|
107
|
+
end
|
143
108
|
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
r.add_file(:core_pan, 'miga-project.core-pan.tsv')
|
156
|
-
r.add_file(:core_pan_plot, 'miga-project.core-pan.pdf')
|
157
|
-
r
|
158
|
-
end
|
109
|
+
##
|
110
|
+
# Helper function for clade iterations.
|
111
|
+
def add_result_iter_clades(base)
|
112
|
+
r = MiGA::Result.new("#{base}.json")
|
113
|
+
r.add_file(:report, 'miga-project.pdf')
|
114
|
+
r.add_file(:class_table, 'miga-project.class.tsv')
|
115
|
+
r.add_file(:class_tree, 'miga-project.class.nwk')
|
116
|
+
r.add_file(:classif, 'miga-project.classif')
|
117
|
+
r.add_file(:medoids, 'miga-project.medoids')
|
118
|
+
r
|
119
|
+
end
|
159
120
|
|
160
|
-
|
161
|
-
|
162
|
-
|
121
|
+
##
|
122
|
+
# Add result type +:ogs+ at +base+ (no +_opts+ supported).
|
123
|
+
def add_result_ogs(base, _opts)
|
124
|
+
if result_files_exist?(base, %w[.empty])
|
163
125
|
r = MiGA::Result.new("#{base}.json")
|
164
|
-
r.add_file(:
|
165
|
-
r
|
166
|
-
r
|
126
|
+
r.add_file(:empty, 'miga-project.empty')
|
127
|
+
return r
|
167
128
|
end
|
129
|
+
return nil unless result_files_exist?(base, %w[.ogs .stats])
|
130
|
+
r = MiGA::Result.new("#{base}.json")
|
131
|
+
r.add_file(:ogs, 'miga-project.ogs')
|
132
|
+
r.add_file(:abc, 'miga-project.abc')
|
133
|
+
r.add_file(:stats, 'miga-project.stats')
|
134
|
+
r.add_file(:core_pan, 'miga-project.core-pan.tsv')
|
135
|
+
r.add_file(:core_pan_plot, 'miga-project.core-pan.pdf')
|
136
|
+
r
|
137
|
+
end
|
138
|
+
|
139
|
+
##
|
140
|
+
# Add result type +:project_stats+ at +base+ (no +_opts+ supported).
|
141
|
+
def add_result_project_stats(base, _opts)
|
142
|
+
return nil unless
|
143
|
+
result_files_exist?(base, %w[.taxonomy.json .metadata.db])
|
144
|
+
r = MiGA::Result.new("#{base}.json")
|
145
|
+
r.add_file(:taxonomy_index, 'miga-project.taxonomy.json')
|
146
|
+
r.add_file(:metadata_index, 'miga-project.metadata.db')
|
147
|
+
r
|
148
|
+
end
|
168
149
|
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
150
|
+
alias add_result_haai_distances add_result_distances
|
151
|
+
alias add_result_aai_distances add_result_distances
|
152
|
+
alias add_result_ani_distances add_result_distances
|
153
|
+
alias add_result_ssu_distances add_result_distances
|
173
154
|
|
174
155
|
end
|
data/lib/miga/remote_dataset.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# @package MiGA
|
2
2
|
# @license Artistic-2.0
|
3
3
|
|
4
|
+
require 'miga/taxonomy'
|
4
5
|
require 'miga/remote_dataset/download'
|
5
6
|
|
6
7
|
##
|
@@ -203,7 +204,7 @@ class MiGA::RemoteDataset < MiGA::MiGA
|
|
203
204
|
when ''
|
204
205
|
metadata[:is_type] = false
|
205
206
|
metadata[:is_ref_type] = false
|
206
|
-
when 'assembly from reference material'
|
207
|
+
when 'assembly from reference material', 'assembly designated as reftype'
|
207
208
|
metadata[:is_type] = false
|
208
209
|
metadata[:is_ref_type] = true
|
209
210
|
metadata[:type_rel] = from_type
|
data/lib/miga/result.rb
CHANGED
@@ -14,19 +14,28 @@ class MiGA::Result < MiGA::MiGA
|
|
14
14
|
include MiGA::Result::Stats
|
15
15
|
|
16
16
|
# Class-level
|
17
|
+
class << self
|
18
|
+
##
|
19
|
+
# Check if the result described by the JSON in +path+ already exists
|
20
|
+
def exist?(path)
|
21
|
+
File.exist? path
|
22
|
+
end
|
17
23
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
24
|
+
##
|
25
|
+
# Load the result described by the JSON in +path+.
|
26
|
+
# Returns MiGA::Result if it already exists, nil otherwise.
|
27
|
+
def load(path)
|
28
|
+
return nil unless MiGA::Result.exist? path
|
29
|
+
MiGA::Result.new(path)
|
30
|
+
end
|
23
31
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
32
|
+
def create(path, force = false, &blk)
|
33
|
+
FileUtils.rm(path) if force && File.exist?(path)
|
34
|
+
r_pre = self.load(path)
|
35
|
+
return r_pre unless r_pre.nil?
|
36
|
+
yield
|
37
|
+
self.load(path)
|
38
|
+
end
|
30
39
|
end
|
31
40
|
|
32
41
|
# Instance-level
|
data/lib/miga/result/source.rb
CHANGED
@@ -18,9 +18,13 @@ module MiGA::Result::Source
|
|
18
18
|
##
|
19
19
|
# Detect the result key assigned to this result
|
20
20
|
def key
|
21
|
-
@key ||=
|
22
|
-
|
23
|
-
|
21
|
+
@key ||= if relative_dir == 'data/90.stats' && file_path(:metadata_index)
|
22
|
+
:project_stats
|
23
|
+
else
|
24
|
+
MiGA::Result.RESULT_DIRS.find do |k, v|
|
25
|
+
"data/#{v}" == relative_dir
|
26
|
+
end.first
|
27
|
+
end
|
24
28
|
end
|
25
29
|
|
26
30
|
##
|
data/lib/miga/result/stats.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
|
2
|
+
require 'zlib'
|
2
3
|
require 'miga/result/base'
|
3
4
|
|
4
5
|
##
|
@@ -75,6 +76,16 @@ module MiGA::Result::Stats
|
|
75
76
|
stats[:coding_density] =
|
76
77
|
[300.0 * s[:tot] / asm[:stats][:total_length][0], '%']
|
77
78
|
end
|
79
|
+
if file_path(:gff3) && file_path(:gff3) =~ /\.gz/
|
80
|
+
Zlib::GzipReader.open(file_path(:gff3)) do |fh|
|
81
|
+
fh.each do |ln|
|
82
|
+
if ln =~ /^# Model Data:.*;transl_table=(\d+);/
|
83
|
+
stats[:codon_table] = $1
|
84
|
+
break
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
78
89
|
stats
|
79
90
|
end
|
80
91
|
|
data/lib/miga/tax_dist.rb
CHANGED
@@ -31,6 +31,7 @@ module MiGA::TaxDist
|
|
31
31
|
# correspond to the p-values of +test+ (one of +:intax+ or +:novel+)
|
32
32
|
# with options +opts+. See +aai_path+ for supported options.
|
33
33
|
def aai_pvalues(aai, test, opts = {})
|
34
|
+
y = {}
|
34
35
|
Zlib::GzipReader.open(aai_path(test, opts)) do |fh|
|
35
36
|
keys = nil
|
36
37
|
fh.each_line do |ln|
|
@@ -46,11 +47,13 @@ module MiGA::TaxDist
|
|
46
47
|
rank = i.zero? ? :root : MiGA::Taxonomy.KNOWN_RANKS[i]
|
47
48
|
vals[rank] = v.to_f
|
48
49
|
end
|
49
|
-
|
50
|
+
y = vals
|
51
|
+
break
|
50
52
|
end
|
51
53
|
end
|
54
|
+
fh.rewind # to avoid warnings caused by the break above
|
52
55
|
end
|
53
|
-
|
56
|
+
y
|
54
57
|
end
|
55
58
|
|
56
59
|
##
|
data/lib/miga/version.rb
CHANGED
@@ -10,7 +10,7 @@ module MiGA
|
|
10
10
|
# - Float representing the major.minor version.
|
11
11
|
# - Integer representing gem releases of the current version.
|
12
12
|
# - Integer representing minor changes that require new version number.
|
13
|
-
VERSION = [0.6, 3,
|
13
|
+
VERSION = [0.6, 3, 1]
|
14
14
|
|
15
15
|
##
|
16
16
|
# Nickname for the current major.minor version.
|
@@ -18,7 +18,7 @@ module MiGA
|
|
18
18
|
|
19
19
|
##
|
20
20
|
# Date of the current gem release.
|
21
|
-
VERSION_DATE = Date.new(2020,
|
21
|
+
VERSION_DATE = Date.new(2020, 4, 11)
|
22
22
|
|
23
23
|
##
|
24
24
|
# Reference of MiGA.
|
data/scripts/cds.bash
CHANGED
@@ -36,7 +36,7 @@ case "$TYPE" in
|
|
36
36
|
| perl -pe 's/[^A-Z]//ig' | wc -c | awk '{print $1}')
|
37
37
|
echo "# codon table $ct total length: $C_LEN aa" \
|
38
38
|
>> "${DATASET}.ct.t"
|
39
|
-
if [[ $C_LEN
|
39
|
+
if [[ $C_LEN -gt $(($P_LEN * 11 / 10)) ]] ; then
|
40
40
|
for x in faa fna gff3 ; do
|
41
41
|
mv "${DATASET}.$x.$ct" "${DATASET}.$x"
|
42
42
|
done
|
data/test/daemon_test.rb
CHANGED
@@ -115,7 +115,7 @@ class DaemonTest < Test::Unit::TestCase
|
|
115
115
|
assert(l[0] =~ /-{20}\n/)
|
116
116
|
assert(l[1] =~ /MiGA:#{p.name} launched/)
|
117
117
|
assert(l[2] =~ /-{20}\n/)
|
118
|
-
assert(l[
|
118
|
+
assert(l[5] =~ /Probing running jobs\n/)
|
119
119
|
end
|
120
120
|
ensure
|
121
121
|
begin
|
data/test/project_test.rb
CHANGED
@@ -2,7 +2,6 @@ require 'test_helper'
|
|
2
2
|
require 'miga/project'
|
3
3
|
|
4
4
|
class ProjectTest < Test::Unit::TestCase
|
5
|
-
|
6
5
|
def setup
|
7
6
|
$tmp = Dir.mktmpdir
|
8
7
|
ENV['MIGA_HOME'] = $tmp
|
@@ -16,6 +15,17 @@ class ProjectTest < Test::Unit::TestCase
|
|
16
15
|
ENV['MIGA_HOME'] = nil
|
17
16
|
end
|
18
17
|
|
18
|
+
def create_result_files(project, res, exts)
|
19
|
+
d = MiGA::Project.RESULT_DIRS[res]
|
20
|
+
(['.done'] + exts).each do |x|
|
21
|
+
assert_nil(
|
22
|
+
project.add_result(res),
|
23
|
+
"Premature registration of result #{res} at extension #{x}"
|
24
|
+
)
|
25
|
+
FileUtils.touch(File.join(project.path, "data/#{d}/miga-project#{x}"))
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
19
29
|
def test_class_load
|
20
30
|
assert_nil(MiGA::Project.load($tmp + '/O_o'))
|
21
31
|
assert_equal(MiGA::Project, MiGA::Project.load($p1.path).class)
|
@@ -47,7 +57,8 @@ class ProjectTest < Test::Unit::TestCase
|
|
47
57
|
assert_equal(MiGA::Dataset, d.class)
|
48
58
|
assert_equal([d], p.datasets)
|
49
59
|
assert_equal(['d1'], p.dataset_names)
|
50
|
-
p.each_dataset{ |ds| assert_equal(d, ds) }
|
60
|
+
p.each_dataset { |ds| assert_equal(d, ds) }
|
61
|
+
p.each_dataset { |n, ds| assert_equal(n, ds.name) }
|
51
62
|
dr = p.unlink_dataset('d1')
|
52
63
|
assert_equal(d, dr)
|
53
64
|
assert_equal([], p.datasets)
|
@@ -71,12 +82,10 @@ class ProjectTest < Test::Unit::TestCase
|
|
71
82
|
assert_equal(1, p2.datasets.size)
|
72
83
|
assert_equal(MiGA::Dataset, p2.dataset('d1').class)
|
73
84
|
assert_equal(1, p2.dataset('d1').results.size)
|
74
|
-
assert(
|
75
|
-
File.
|
76
|
-
)
|
77
|
-
assert(File.exist?(
|
78
|
-
File.expand_path("metadata/#{d1.name}.json", p2.path)
|
79
|
-
))
|
85
|
+
assert(
|
86
|
+
File.exist?(File.join(p2.path, "data/01.raw_reads/#{d1.name}.1.fastq"))
|
87
|
+
)
|
88
|
+
assert(File.exist?(File.join(p2.path, "metadata/#{d1.name}.json")))
|
80
89
|
end
|
81
90
|
|
82
91
|
def test_add_result
|
@@ -85,7 +94,7 @@ class ProjectTest < Test::Unit::TestCase
|
|
85
94
|
%w[.Rdata .log .txt .done].each do |x|
|
86
95
|
assert_nil(p1.add_result(:haai_distances))
|
87
96
|
FileUtils.touch(
|
88
|
-
File.
|
97
|
+
File.join(p1.path, "data/09.distances/01.haai/miga-project#{x}")
|
89
98
|
)
|
90
99
|
end
|
91
100
|
assert_equal(MiGA::Result, p1.add_result(:haai_distances).class)
|
@@ -93,12 +102,12 @@ class ProjectTest < Test::Unit::TestCase
|
|
93
102
|
|
94
103
|
def test_result
|
95
104
|
p1 = $p1
|
96
|
-
assert_nil(p1.result
|
97
|
-
assert_nil(p1.result
|
105
|
+
assert_nil(p1.result(:n00b))
|
106
|
+
assert_nil(p1.result(:project_stats))
|
98
107
|
File.open(
|
99
108
|
File.expand_path('data/90.stats/miga-project.json', p1.path), 'w'
|
100
109
|
) { |fh| fh.puts '{}' }
|
101
|
-
assert_not_nil(p1.result
|
110
|
+
assert_not_nil(p1.result(:project_stats))
|
102
111
|
assert_equal(1, p1.results.size)
|
103
112
|
end
|
104
113
|
|
@@ -109,64 +118,48 @@ class ProjectTest < Test::Unit::TestCase
|
|
109
118
|
assert(!p1.done_preprocessing?)
|
110
119
|
FileUtils.touch(File.expand_path("data/90.stats/#{d1.name}.done", p1.path))
|
111
120
|
assert(p1.done_preprocessing?)
|
121
|
+
assert_nil(p1.next_inclade)
|
122
|
+
p1.metadata[:type] = :clade
|
123
|
+
assert_equal(:subclades, p1.next_inclade)
|
112
124
|
|
113
|
-
# Project
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
assert_equal(r, p1.next_distances)
|
129
|
-
assert_equal(Symbol, p1.next_distances.class)
|
130
|
-
d = MiGA::Project.RESULT_DIRS[r]
|
131
|
-
%w[.done .Rdata .log .txt].each do |x|
|
132
|
-
assert_nil(
|
133
|
-
p1.add_result(r),
|
134
|
-
"Premature registration of result #{r} at extension #{x}."
|
135
|
-
)
|
136
|
-
FileUtils.touch(File.expand_path("data/#{d}/miga-project#{x}", p1.path))
|
137
|
-
end
|
138
|
-
assert_equal(MiGA::Result, p1.add_result(r).class,
|
139
|
-
"Imposible to add #{r} result.")
|
125
|
+
# Project tasks
|
126
|
+
expected_files = {
|
127
|
+
project_stats: %w[.taxonomy.json .metadata.db],
|
128
|
+
haai_distances: %w[.Rdata .log .txt],
|
129
|
+
aai_distances: %w[.Rdata .log .txt],
|
130
|
+
ani_distances: %w[.Rdata .log .txt],
|
131
|
+
clade_finding: %w[.pdf .classif .medoids
|
132
|
+
.class.tsv .class.nwk .proposed-clades],
|
133
|
+
subclades: %w[.pdf .classif .medoids .class.tsv .class.nwk],
|
134
|
+
ogs: %w[.ogs .stats]
|
135
|
+
}
|
136
|
+
expected_files.each do |r, exts|
|
137
|
+
assert_equal(r, p1.next_task)
|
138
|
+
create_result_files(p1, r, exts)
|
139
|
+
assert_not_nil(p1.add_result(r), "Imposible to add #{r} result")
|
140
140
|
end
|
141
|
-
|
141
|
+
assert_nil(p1.next_task)
|
142
|
+
p1.each_result { |k, r| assert_equal(k, r.key) }
|
143
|
+
end
|
142
144
|
|
143
|
-
|
144
|
-
|
145
|
+
def test_empty_results
|
146
|
+
p1 = $p1
|
145
147
|
p1.metadata[:type] = :clade
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
[:ogs, %w[.ogs .stats]]
|
151
|
-
]
|
152
|
-
res.each do |rr|
|
153
|
-
(r, xs) = rr
|
154
|
-
d = MiGA::Project.RESULT_DIRS[r]
|
155
|
-
assert_equal(Symbol, p1.next_inclade.class)
|
156
|
-
([".done"] + xs).each do |x|
|
157
|
-
assert_nil(
|
158
|
-
p1.add_result(r),
|
159
|
-
"Premature registration of result #{r} at extension #{x}."
|
160
|
-
)
|
161
|
-
FileUtils.touch(File.expand_path("data/#{d}/miga-project#{x}", p1.path))
|
162
|
-
end
|
163
|
-
assert_equal(
|
164
|
-
MiGA::Result,
|
165
|
-
p1.add_result(r).class,
|
166
|
-
"Impossible to add #{r} result."
|
167
|
-
)
|
148
|
+
%i[clade_finding subclades ogs].each do |r|
|
149
|
+
assert_nil(p1.add_result(r), "Unexpected result exists: #{r}")
|
150
|
+
create_result_files(p1, r, %w[.empty])
|
151
|
+
assert_not_nil(p1.add_result(r), "Cannot register emtpy task: #{r}")
|
168
152
|
end
|
169
|
-
assert_nil(p1.next_inclade)
|
170
153
|
end
|
171
154
|
|
155
|
+
def test_force_result
|
156
|
+
p1 = $p1
|
157
|
+
create_result_files(p1, :ogs, %w[.empty])
|
158
|
+
date1 = p1.add_result(:ogs)[:created]
|
159
|
+
sleep(1)
|
160
|
+
date2 = p1.add_result(:ogs, true, force: false)[:created]
|
161
|
+
assert_equal(date1, date2)
|
162
|
+
date3 = p1.add_result(:ogs, true, force: true)[:created]
|
163
|
+
assert_not_equal(date1, date3)
|
164
|
+
end
|
172
165
|
end
|
data/test/remote_dataset_test.rb
CHANGED
@@ -30,10 +30,10 @@ class RemoteDatasetTest < Test::Unit::TestCase
|
|
30
30
|
|
31
31
|
def test_rest
|
32
32
|
hiv2 = 'M30502.1'
|
33
|
-
{embl: :ebi, nuccore: :ncbi}.each do |db, universe|
|
33
|
+
{ embl: :ebi, nuccore: :ncbi }.each do |db, universe|
|
34
34
|
rd = MiGA::RemoteDataset.new(hiv2, db, universe)
|
35
35
|
assert_equal([hiv2], rd.ids)
|
36
|
-
omit_if(!$remote_tests, 'Remote access is error-prone
|
36
|
+
omit_if(!$remote_tests, 'Remote access is error-prone')
|
37
37
|
tx = rd.get_ncbi_taxonomy
|
38
38
|
msg = "Failed on #{universe}:#{db}"
|
39
39
|
assert_equal(MiGA::Taxonomy, tx.class, msg)
|
@@ -57,7 +57,7 @@ class RemoteDatasetTest < Test::Unit::TestCase
|
|
57
57
|
n = 'Cjac_L14'
|
58
58
|
rd = MiGA::RemoteDataset.new(cjac, :assembly_gz, :web)
|
59
59
|
assert_equal([cjac], rd.ids)
|
60
|
-
omit_if(!$remote_tests, 'Remote access is error-prone
|
60
|
+
omit_if(!$remote_tests, 'Remote access is error-prone')
|
61
61
|
p = $p1
|
62
62
|
assert_nil(p.dataset(n))
|
63
63
|
rd.save_to(p, n)
|
@@ -66,9 +66,54 @@ class RemoteDatasetTest < Test::Unit::TestCase
|
|
66
66
|
assert_equal(MiGA::Result, p.dataset(n).result(:assembly).class)
|
67
67
|
end
|
68
68
|
|
69
|
+
def test_asm_acc2id
|
70
|
+
omit_if(!$remote_tests, 'Remote access is error-prone')
|
71
|
+
assert_nil(MiGA::RemoteDataset.ncbi_asm_acc2id('NotAnAccession'))
|
72
|
+
id = MiGA::RemoteDataset.ncbi_asm_acc2id('GCA_004684205.1')
|
73
|
+
assert_equal('2514661', id)
|
74
|
+
assert_equal(id, MiGA::RemoteDataset.ncbi_asm_acc2id(id))
|
75
|
+
end
|
76
|
+
|
77
|
+
def test_update_metadata
|
78
|
+
omit_if(!$remote_tests, 'Remote access is error-prone')
|
79
|
+
hiv1 = 'GCF_000856385.1'
|
80
|
+
d1 = MiGA::Dataset.new($p1, 'd1')
|
81
|
+
assert_nil(d1.metadata[:ncbi_assembly])
|
82
|
+
rd = MiGA::RemoteDataset.new(hiv1, :assembly, :ncbi)
|
83
|
+
rd.update_metadata(d1, passthrough: 123, metadata_only: true)
|
84
|
+
assert_equal(123, d1.metadata[:passthrough])
|
85
|
+
assert_equal(hiv1, d1.metadata[:ncbi_assembly])
|
86
|
+
assert_equal('Lentivirus', d1.metadata[:tax][:g])
|
87
|
+
end
|
88
|
+
|
89
|
+
def test_type_status_asm
|
90
|
+
omit_if(!$remote_tests, 'Remote access is error-prone')
|
91
|
+
rd = MiGA::RemoteDataset.new('GCF_000018105.1', :assembly, :ncbi)
|
92
|
+
assert(rd.get_metadata[:is_type])
|
93
|
+
end
|
94
|
+
|
95
|
+
def test_nontype_status_asm
|
96
|
+
omit_if(!$remote_tests, 'Remote access is error-prone')
|
97
|
+
rd = MiGA::RemoteDataset.new('GCA_004684205.1', :assembly, :ncbi)
|
98
|
+
assert(!rd.get_metadata[:is_type])
|
99
|
+
end
|
100
|
+
|
101
|
+
def test_type_status_nuccore
|
102
|
+
omit_if(!$remote_tests, 'Remote access is error-prone')
|
103
|
+
rd = MiGA::RemoteDataset.new('NC_019748.1', :nuccore, :ncbi)
|
104
|
+
assert(rd.get_metadata[:is_type])
|
105
|
+
end
|
106
|
+
|
107
|
+
def test_ref_type_status
|
108
|
+
omit_if(!$remote_tests, 'Remote access is error-prone')
|
109
|
+
rd = MiGA::RemoteDataset.new('GCA_002849345', :assembly, :ncbi)
|
110
|
+
assert(!rd.get_metadata[:is_type])
|
111
|
+
assert(rd.get_metadata[:is_ref_type])
|
112
|
+
end
|
113
|
+
|
69
114
|
# This test is too expensive (too much time to run it!)
|
70
115
|
#def test_net_timeout
|
71
|
-
# omit_if(!$remote_tests, "Remote access is error-prone
|
116
|
+
# omit_if(!$remote_tests, "Remote access is error-prone")
|
72
117
|
# bad = "ftp://example.com/miga"
|
73
118
|
# rd = MiGA::RemoteDataset.new(bad, :assembly, :web)
|
74
119
|
# assert_raise(Net::ReadTimeout) { rd.save_to($p1, "bad") }
|
data/test/tax_dist_test.rb
CHANGED
@@ -26,7 +26,6 @@ class TaxDistTest < Test::Unit::TestCase
|
|
26
26
|
assert_lt(close_intax[:s], 0.05)
|
27
27
|
|
28
28
|
distant_novel = MiGA::TaxDist.aai_pvalues(35.0, :novel, engine: :diamond)
|
29
|
-
$stderr.puts distant_novel
|
30
29
|
assert_gt(distant_novel[:root], 0.05)
|
31
30
|
assert_lt(distant_novel[:g], 0.05)
|
32
31
|
assert_nil(distant_novel[:ns])
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: miga-base
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.3.
|
4
|
+
version: 0.6.3.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Luis M. Rodriguez-R
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-04-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: daemons
|
@@ -157,6 +157,7 @@ files:
|
|
157
157
|
- lib/miga/common/format.rb
|
158
158
|
- lib/miga/common/hooks.rb
|
159
159
|
- lib/miga/common/path.rb
|
160
|
+
- lib/miga/common/with_result.rb
|
160
161
|
- lib/miga/daemon.rb
|
161
162
|
- lib/miga/daemon/base.rb
|
162
163
|
- lib/miga/dataset.rb
|