miga-base 0.4.1.0 → 0.4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/miga +2 -244
- data/lib/miga/cli/action/about.rb +44 -0
- data/lib/miga/cli/action/add.rb +139 -0
- data/lib/miga/cli/action/add_result.rb +26 -0
- data/lib/miga/cli/action/console.rb +19 -0
- data/lib/miga/cli/action/daemon.rb +74 -0
- data/lib/miga/cli/action/date.rb +18 -0
- data/lib/miga/cli/action/doctor.rb +210 -0
- data/lib/miga/cli/action/edit.rb +24 -0
- data/lib/miga/cli/action/files.rb +31 -0
- data/lib/miga/cli/action/find.rb +48 -0
- data/lib/miga/cli/action/generic.rb +44 -0
- data/lib/miga/cli/action/get.rb +132 -0
- data/lib/miga/cli/action/init.rb +343 -0
- data/lib/miga/cli/action/ln.rb +42 -0
- data/lib/miga/cli/action/ls.rb +55 -0
- data/lib/miga/cli/action/ncbi_get.rb +218 -0
- data/lib/miga/cli/action/new.rb +45 -0
- data/lib/miga/cli/action/next_step.rb +27 -0
- data/lib/miga/cli/action/plugins.rb +28 -0
- data/lib/miga/cli/action/rm.rb +25 -0
- data/lib/miga/cli/action/run.rb +39 -0
- data/lib/miga/cli/action/stats.rb +140 -0
- data/lib/miga/cli/action/summary.rb +49 -0
- data/lib/miga/cli/action/tax_dist.rb +102 -0
- data/lib/miga/cli/action/tax_index.rb +47 -0
- data/lib/miga/cli/action/tax_set.rb +59 -0
- data/lib/miga/cli/action/tax_test.rb +77 -0
- data/lib/miga/cli/action.rb +66 -0
- data/lib/miga/cli/base.rb +90 -0
- data/lib/miga/cli.rb +426 -0
- data/lib/miga/project/result.rb +14 -6
- data/lib/miga/remote_dataset.rb +1 -1
- data/lib/miga/tax_index.rb +5 -4
- data/lib/miga/taxonomy/base.rb +63 -0
- data/lib/miga/taxonomy.rb +87 -92
- data/lib/miga/version.rb +6 -6
- data/test/taxonomy_test.rb +49 -9
- data/utils/distance/commands.rb +11 -11
- data/utils/distance/pipeline.rb +5 -5
- metadata +43 -49
- data/actions/about.rb +0 -43
- data/actions/add.rb +0 -129
- data/actions/add_result.rb +0 -30
- data/actions/daemon.rb +0 -55
- data/actions/date.rb +0 -14
- data/actions/doctor.rb +0 -201
- data/actions/edit.rb +0 -33
- data/actions/files.rb +0 -43
- data/actions/find.rb +0 -41
- data/actions/get.rb +0 -105
- data/actions/init.rb +0 -301
- data/actions/ln.rb +0 -47
- data/actions/ls.rb +0 -61
- data/actions/ncbi_get.rb +0 -192
- data/actions/new.rb +0 -44
- data/actions/next_step.rb +0 -33
- data/actions/plugins.rb +0 -25
- data/actions/rm.rb +0 -29
- data/actions/run.rb +0 -45
- data/actions/stats.rb +0 -149
- data/actions/summary.rb +0 -57
- data/actions/tax_dist.rb +0 -106
- data/actions/tax_index.rb +0 -46
- data/actions/tax_set.rb +0 -63
- data/actions/tax_test.rb +0 -80
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: miga-base
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Luis M. Rodriguez-R
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-08
|
11
|
+
date: 2019-09-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: daemons
|
@@ -16,68 +16,56 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '1.
|
20
|
-
- - ">="
|
21
|
-
- !ruby/object:Gem::Version
|
22
|
-
version: 1.2.4
|
19
|
+
version: '1.3'
|
23
20
|
type: :runtime
|
24
21
|
prerelease: false
|
25
22
|
version_requirements: !ruby/object:Gem::Requirement
|
26
23
|
requirements:
|
27
24
|
- - "~>"
|
28
25
|
- !ruby/object:Gem::Version
|
29
|
-
version: '1.
|
30
|
-
- - ">="
|
31
|
-
- !ruby/object:Gem::Version
|
32
|
-
version: 1.2.4
|
26
|
+
version: '1.3'
|
33
27
|
- !ruby/object:Gem::Dependency
|
34
28
|
name: json
|
35
29
|
requirement: !ruby/object:Gem::Requirement
|
36
30
|
requirements:
|
37
|
-
- - "
|
38
|
-
- !ruby/object:Gem::Version
|
39
|
-
version: '1.8'
|
40
|
-
- - "<"
|
31
|
+
- - "~>"
|
41
32
|
- !ruby/object:Gem::Version
|
42
|
-
version: '
|
33
|
+
version: '2'
|
43
34
|
type: :runtime
|
44
35
|
prerelease: false
|
45
36
|
version_requirements: !ruby/object:Gem::Requirement
|
46
37
|
requirements:
|
47
|
-
- - "
|
48
|
-
- !ruby/object:Gem::Version
|
49
|
-
version: '1.8'
|
50
|
-
- - "<"
|
38
|
+
- - "~>"
|
51
39
|
- !ruby/object:Gem::Version
|
52
|
-
version: '
|
40
|
+
version: '2'
|
53
41
|
- !ruby/object:Gem::Dependency
|
54
42
|
name: sqlite3
|
55
43
|
requirement: !ruby/object:Gem::Requirement
|
56
44
|
requirements:
|
57
45
|
- - "~>"
|
58
46
|
- !ruby/object:Gem::Version
|
59
|
-
version: '1.
|
47
|
+
version: '1.4'
|
60
48
|
type: :runtime
|
61
49
|
prerelease: false
|
62
50
|
version_requirements: !ruby/object:Gem::Requirement
|
63
51
|
requirements:
|
64
52
|
- - "~>"
|
65
53
|
- !ruby/object:Gem::Version
|
66
|
-
version: '1.
|
54
|
+
version: '1.4'
|
67
55
|
- !ruby/object:Gem::Dependency
|
68
56
|
name: rake
|
69
57
|
requirement: !ruby/object:Gem::Requirement
|
70
58
|
requirements:
|
71
59
|
- - "~>"
|
72
60
|
- !ruby/object:Gem::Version
|
73
|
-
version: '
|
61
|
+
version: '12'
|
74
62
|
type: :development
|
75
63
|
prerelease: false
|
76
64
|
version_requirements: !ruby/object:Gem::Requirement
|
77
65
|
requirements:
|
78
66
|
- - "~>"
|
79
67
|
- !ruby/object:Gem::Version
|
80
|
-
version: '
|
68
|
+
version: '12'
|
81
69
|
- !ruby/object:Gem::Dependency
|
82
70
|
name: test-unit
|
83
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -104,35 +92,40 @@ files:
|
|
104
92
|
- LICENSE
|
105
93
|
- README.md
|
106
94
|
- Rakefile
|
107
|
-
- actions/about.rb
|
108
|
-
- actions/add.rb
|
109
|
-
- actions/add_result.rb
|
110
|
-
- actions/daemon.rb
|
111
|
-
- actions/date.rb
|
112
|
-
- actions/doctor.rb
|
113
|
-
- actions/edit.rb
|
114
|
-
- actions/files.rb
|
115
|
-
- actions/find.rb
|
116
|
-
- actions/get.rb
|
117
|
-
- actions/init.rb
|
118
|
-
- actions/ln.rb
|
119
|
-
- actions/ls.rb
|
120
|
-
- actions/ncbi_get.rb
|
121
|
-
- actions/new.rb
|
122
|
-
- actions/next_step.rb
|
123
|
-
- actions/plugins.rb
|
124
|
-
- actions/rm.rb
|
125
|
-
- actions/run.rb
|
126
|
-
- actions/stats.rb
|
127
|
-
- actions/summary.rb
|
128
|
-
- actions/tax_dist.rb
|
129
|
-
- actions/tax_index.rb
|
130
|
-
- actions/tax_set.rb
|
131
|
-
- actions/tax_test.rb
|
132
95
|
- bin/miga
|
133
96
|
- lib/miga.rb
|
134
97
|
- lib/miga/_data/aai-intax.tsv.gz
|
135
98
|
- lib/miga/_data/aai-novel.tsv.gz
|
99
|
+
- lib/miga/cli.rb
|
100
|
+
- lib/miga/cli/action.rb
|
101
|
+
- lib/miga/cli/action/about.rb
|
102
|
+
- lib/miga/cli/action/add.rb
|
103
|
+
- lib/miga/cli/action/add_result.rb
|
104
|
+
- lib/miga/cli/action/console.rb
|
105
|
+
- lib/miga/cli/action/daemon.rb
|
106
|
+
- lib/miga/cli/action/date.rb
|
107
|
+
- lib/miga/cli/action/doctor.rb
|
108
|
+
- lib/miga/cli/action/edit.rb
|
109
|
+
- lib/miga/cli/action/files.rb
|
110
|
+
- lib/miga/cli/action/find.rb
|
111
|
+
- lib/miga/cli/action/generic.rb
|
112
|
+
- lib/miga/cli/action/get.rb
|
113
|
+
- lib/miga/cli/action/init.rb
|
114
|
+
- lib/miga/cli/action/ln.rb
|
115
|
+
- lib/miga/cli/action/ls.rb
|
116
|
+
- lib/miga/cli/action/ncbi_get.rb
|
117
|
+
- lib/miga/cli/action/new.rb
|
118
|
+
- lib/miga/cli/action/next_step.rb
|
119
|
+
- lib/miga/cli/action/plugins.rb
|
120
|
+
- lib/miga/cli/action/rm.rb
|
121
|
+
- lib/miga/cli/action/run.rb
|
122
|
+
- lib/miga/cli/action/stats.rb
|
123
|
+
- lib/miga/cli/action/summary.rb
|
124
|
+
- lib/miga/cli/action/tax_dist.rb
|
125
|
+
- lib/miga/cli/action/tax_index.rb
|
126
|
+
- lib/miga/cli/action/tax_set.rb
|
127
|
+
- lib/miga/cli/action/tax_test.rb
|
128
|
+
- lib/miga/cli/base.rb
|
136
129
|
- lib/miga/common.rb
|
137
130
|
- lib/miga/common/base.rb
|
138
131
|
- lib/miga/common/format.rb
|
@@ -158,6 +151,7 @@ files:
|
|
158
151
|
- lib/miga/tax_dist.rb
|
159
152
|
- lib/miga/tax_index.rb
|
160
153
|
- lib/miga/taxonomy.rb
|
154
|
+
- lib/miga/taxonomy/base.rb
|
161
155
|
- lib/miga/version.rb
|
162
156
|
- scripts/aai_distances.bash
|
163
157
|
- scripts/ani_distances.bash
|
data/actions/about.rb
DELETED
@@ -1,43 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
# @package MiGA
|
4
|
-
# @license Artistic-2.0
|
5
|
-
|
6
|
-
o = {q:true, info:false, processing:false, tabular:false}
|
7
|
-
OptionParser.new do |opt|
|
8
|
-
opt_banner(opt)
|
9
|
-
opt_object(opt, o, [:project])
|
10
|
-
opt.on("-p", "--processing",
|
11
|
-
"Print information on processing advance."){ |v| o[:processing]=v }
|
12
|
-
opt.on("-m", "--metadata STRING",
|
13
|
-
"Print name and metadata field only."
|
14
|
-
){ |v| o[:datum]=v }
|
15
|
-
opt.on("--tab",
|
16
|
-
"Returns a tab-delimited table."){ |v| o[:tabular] = v }
|
17
|
-
opt_common(opt, o)
|
18
|
-
end.parse!
|
19
|
-
|
20
|
-
|
21
|
-
##=> Main <=
|
22
|
-
opt_require(o, project:"-P")
|
23
|
-
|
24
|
-
$stderr.puts "Loading project." unless o[:q]
|
25
|
-
p = MiGA::Project.load(o[:project])
|
26
|
-
raise "Impossible to load project: #{o[:project]}" if p.nil?
|
27
|
-
|
28
|
-
if not o[:datum].nil?
|
29
|
-
v = p.metadata[ o[:datum] ]
|
30
|
-
puts v.nil? ? '?' : v
|
31
|
-
elsif o[:processing]
|
32
|
-
keys = MiGA::Project.DISTANCE_TASKS + MiGA::Project.INCLADE_TASKS
|
33
|
-
puts MiGA::MiGA.tabulate([:task, :status], keys.map do |k|
|
34
|
-
[k, p.add_result(k, false).nil? ? "queued" : "done"]
|
35
|
-
end, o[:tabular])
|
36
|
-
else
|
37
|
-
puts MiGA::MiGA.tabulate([:key, :value], p.metadata.data.keys.map do |k|
|
38
|
-
v = p.metadata[k]
|
39
|
-
[k, k==:datasets ? v.size : v]
|
40
|
-
end, o[:tabular])
|
41
|
-
end
|
42
|
-
|
43
|
-
$stderr.puts "Done." unless o[:q]
|
data/actions/add.rb
DELETED
@@ -1,129 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
# @package MiGA
|
4
|
-
# @license Artistic-2.0
|
5
|
-
|
6
|
-
input_types = {
|
7
|
-
raw_reads_single:
|
8
|
-
['Single raw reads in a single FastQ file',
|
9
|
-
:raw_reads, %w[.1.fastq]],
|
10
|
-
raw_reads_paired:
|
11
|
-
['Paired raw reads in two FastQ files',
|
12
|
-
:raw_reads, %w[.1.fastq .2.fastq]],
|
13
|
-
trimmed_reads_single:
|
14
|
-
['Single trimmed reads in a single FastA file',
|
15
|
-
:trimmed_fasta, %w[.SingleReads.fa]],
|
16
|
-
trimmed_reads_paired:
|
17
|
-
['Paired trimmed reads in two FastA files',
|
18
|
-
:trimmed_fasta, %w[.1.fasta .2.fasta]],
|
19
|
-
trimmed_reads_interleaved:
|
20
|
-
['Paired trimmed reads in a single FastA file',
|
21
|
-
:trimmed_fasta, %w[.CoupledReads.fa]],
|
22
|
-
assembly:
|
23
|
-
['Assembled contigs or scaffolds in FastA format',
|
24
|
-
:assembly, %w[.LargeContigs.fna]]
|
25
|
-
}
|
26
|
-
|
27
|
-
o = {q: true, ref: true, ignore_dups: false,
|
28
|
-
regexp: /^(?:.*\/)?(.+?)(?:\..*(?:[12]|Reads|Contigs))?(?i:\.f[nastq]+)?$/}
|
29
|
-
OptionParser.new do |opt|
|
30
|
-
opt_banner(opt, true)
|
31
|
-
opt_object(opt, o, [:project, :dataset_opt, :dataset_type_req])
|
32
|
-
opt.on('-q', '--query',
|
33
|
-
'If set, the dataset is registered as a query, not a reference dataset.'
|
34
|
-
){ |v| o[:ref] = !v }
|
35
|
-
opt.on('-d', '--description STRING',
|
36
|
-
'Description of the dataset.'){ |v| o[:description] = v }
|
37
|
-
opt.on('-c', '--comments STRING',
|
38
|
-
'Comments on the dataset.'){ |v| o[:comments] = v }
|
39
|
-
opt.on('-m', '--metadata STRING',
|
40
|
-
'Metadata as key-value pairs separated by = and delimited by comma.',
|
41
|
-
'Values are saved as strings except for booleans (true / false) or nil.'
|
42
|
-
){ |v| o[:metadata] = v }
|
43
|
-
opt.on('-r', '--name-regexp REGEXP', Regexp,
|
44
|
-
'Regular expression indicating how to extract the name from the file path.',
|
45
|
-
"By default: '#{o[:regexp]}'"){ |v| o[:regexp] = v }
|
46
|
-
opt.on('-i', '--input-type STRING',
|
47
|
-
'Type of input data, one of the following:',
|
48
|
-
*input_types.map{ |k,v| "~ #{k}: #{v[0]}." }
|
49
|
-
){ |v| o[:input_type] = v.downcase.to_sym }
|
50
|
-
opt.on('--ignore-dups', 'Continue with a warning if a dataset already exists.'
|
51
|
-
){ |v| o[:ignore_dups] = v }
|
52
|
-
opt_common(opt, o)
|
53
|
-
|
54
|
-
opt.separator 'You can create multiple datasets with a single command, ' \
|
55
|
-
'simply pass all the files at the end (FILES...).'
|
56
|
-
opt.separator 'If -D is passed, only one dataset will be added. ' \
|
57
|
-
'Otherwise, dataset names will be determined by the file paths (-r).'
|
58
|
-
opt.separator ''
|
59
|
-
end.parse!
|
60
|
-
|
61
|
-
##=> Main <=
|
62
|
-
opt_require(o, project: '-P')
|
63
|
-
files = ARGV
|
64
|
-
file_type = nil
|
65
|
-
if files.empty?
|
66
|
-
opt_require_type(o, MiGA::Dataset)
|
67
|
-
files = [nil]
|
68
|
-
else
|
69
|
-
raise 'Please specify input type (-i).' if o[:input_type].nil?
|
70
|
-
file_type = input_types[o[:input_type]]
|
71
|
-
raise "Unrecognized input type: #{o[:input_type]}." if file_type.nil?
|
72
|
-
raise 'Some files are duplicated, files must be unique.' if
|
73
|
-
files.size != files.uniq.size
|
74
|
-
if o[:input_type].to_s =~ /_paired$/
|
75
|
-
raise 'Odd number of files incompatible with input type.' if files.size.odd?
|
76
|
-
files = Hash[*files].to_a
|
77
|
-
else
|
78
|
-
files = files.map{ |i| [i] }
|
79
|
-
end
|
80
|
-
raise 'The dataset name (-D) can only be specified with one input file.' if
|
81
|
-
files.size > 1 and not o[:dataset].nil?
|
82
|
-
end
|
83
|
-
|
84
|
-
$stderr.puts 'Loading project.' unless o[:q]
|
85
|
-
p = MiGA::Project.load(o[:project])
|
86
|
-
raise "Impossible to load project: #{o[:project]}" if p.nil?
|
87
|
-
|
88
|
-
$stderr.puts 'Creating datasets:' unless o[:q]
|
89
|
-
files.each do |file|
|
90
|
-
name = o[:dataset]
|
91
|
-
if name.nil?
|
92
|
-
ref_file = file.is_a?(Array) ? file.first : file
|
93
|
-
m = o[:regexp].match(ref_file)
|
94
|
-
raise "Cannot extract name from file: #{ref_file}" if m.nil? or m[1].nil?
|
95
|
-
name = m[1].miga_name
|
96
|
-
end
|
97
|
-
|
98
|
-
if MiGA::Dataset.exist?(p, name)
|
99
|
-
if o[:ignore_dups]
|
100
|
-
warn "Dataset already exists: #{name}."
|
101
|
-
next
|
102
|
-
else
|
103
|
-
raise "Dataset already exists: #{name}."
|
104
|
-
end
|
105
|
-
end
|
106
|
-
|
107
|
-
$stderr.puts "o #{name}" unless o[:q]
|
108
|
-
d = MiGA::Dataset.new(p, name, o[:ref])
|
109
|
-
raise "Unexpected: Couldn't create dataset: #{name}." if d.nil?
|
110
|
-
|
111
|
-
unless file.nil?
|
112
|
-
r_dir = MiGA::Dataset.RESULT_DIRS[ file_type[1] ]
|
113
|
-
r_path = File.expand_path("data/#{r_dir}/#{d.name}", p.path)
|
114
|
-
file_type[2].each_with_index do |ext, i|
|
115
|
-
gz = file[i] =~ /\.gz/ ? '.gz' : ''
|
116
|
-
FileUtils.cp(file[i], "#{r_path}#{ext}#{gz}")
|
117
|
-
$stderr.puts " file: #{file[i]}" unless o[:q]
|
118
|
-
end
|
119
|
-
File.open("#{r_path}.done", 'w') { |f| f.print Time.now.to_s }
|
120
|
-
end
|
121
|
-
|
122
|
-
d = add_metadata(o, d)
|
123
|
-
d.save
|
124
|
-
p.add_dataset(name)
|
125
|
-
res = d.first_preprocessing(true)
|
126
|
-
$stderr.puts " result: #{res}" unless o[:q]
|
127
|
-
end
|
128
|
-
|
129
|
-
$stderr.puts 'Done.' unless o[:q]
|
data/actions/add_result.rb
DELETED
@@ -1,30 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
# @package MiGA
|
4
|
-
# @license Artistic-2.0
|
5
|
-
|
6
|
-
o = {q:true, force:false}
|
7
|
-
opts = OptionParser.new do |opt|
|
8
|
-
opt_banner(opt)
|
9
|
-
opt_object(opt, o, [:project, :dataset_opt, :result])
|
10
|
-
opt.on("-f", "--force",
|
11
|
-
"Forces re-indexing of the result even if it's already registered."
|
12
|
-
){ |v| o[:force]=v }
|
13
|
-
opt_common(opt, o)
|
14
|
-
end.parse!
|
15
|
-
|
16
|
-
##=> Main <=
|
17
|
-
opts.parse!
|
18
|
-
opt_require(o, project:"-P", name:"-r")
|
19
|
-
|
20
|
-
$stderr.puts "Loading project." unless o[:q]
|
21
|
-
p = MiGA::Project.load(o[:project])
|
22
|
-
raise "Impossible to load project: #{o[:project]}" if p.nil?
|
23
|
-
|
24
|
-
$stderr.puts "Registering result." unless o[:q]
|
25
|
-
obj = o[:dataset].nil? ? p : p.dataset(o[:dataset])
|
26
|
-
r = obj.add_result(o[:name].to_sym, true, force: o[:force])
|
27
|
-
|
28
|
-
raise "Cannot add result, incomplete expected files." if r.nil?
|
29
|
-
|
30
|
-
$stderr.puts "Done." unless o[:q]
|
data/actions/daemon.rb
DELETED
@@ -1,55 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
# @package MiGA
|
4
|
-
# @license Artistic-2.0
|
5
|
-
|
6
|
-
require 'miga/daemon'
|
7
|
-
|
8
|
-
task = ARGV.shift unless %w[-h --help].include? ARGV.first
|
9
|
-
ARGV << '-h' if ARGV.empty?
|
10
|
-
o = {q: true, daemon_opts: []}
|
11
|
-
OptionParser.new do |opt|
|
12
|
-
opt_banner(opt)
|
13
|
-
opt.separator 'task:'
|
14
|
-
{ start: 'Start an instance of the application.',
|
15
|
-
stop: 'Start an instance of the application.',
|
16
|
-
restart: 'Stop all instances and restart them afterwards.',
|
17
|
-
reload: 'Send a SIGHUP to all instances of the application.',
|
18
|
-
run: 'Start the application and stay on top.',
|
19
|
-
zap: 'Set the application to a stopped state.',
|
20
|
-
status: 'Show status (PID) of application instances.'
|
21
|
-
}.each{ |k,v| opt.separator sprintf ' %*s%s', -33, k, v }
|
22
|
-
opt.separator ''
|
23
|
-
opt.separator 'MiGA options:'
|
24
|
-
opt_object(opt, o, [:project])
|
25
|
-
opt.on('--shutdown-when-done',
|
26
|
-
'If passed, the daemon will exit when all processing is done.',
|
27
|
-
'Otherwise (default), it will stay idle awaiting for new data.'
|
28
|
-
){ |v| o[:shutdown_when_done] = v }
|
29
|
-
opt.on('--latency INT',
|
30
|
-
'Number of seconds the daemon will be sleeping.'
|
31
|
-
){ |v| o[:latency]=v.to_i }
|
32
|
-
opt.on('--max-jobs INT',
|
33
|
-
'Maximum number of jobs to use simultaneously.'){ |v| o[:maxjobs]=v.to_i }
|
34
|
-
opt.on('--ppn INT',
|
35
|
-
'Maximum number of cores to use in a single job.'){ |v| o[:ppn]=v.to_i }
|
36
|
-
opt_common(opt, o)
|
37
|
-
opt.separator 'Daemon options:'
|
38
|
-
opt.on('-t', '--ontop',
|
39
|
-
'Stay on top (does not daemonize).'){ o[:daemon_opts] << '-t' }
|
40
|
-
opt.on('-f', '--force', 'Force operation.'){ o[:daemon_opts] << '-f' }
|
41
|
-
opt.on('-n', '--no_wait',
|
42
|
-
'Do not wait for processes to stop.'){ o[:daemon_opts] << '-n' }
|
43
|
-
opt.on('--shush', 'Silence the daemon.'){ o[:daemon_opts] << '--shush' }
|
44
|
-
end.parse!
|
45
|
-
|
46
|
-
##=> Main <=
|
47
|
-
opt_require(o, project: '-P')
|
48
|
-
|
49
|
-
p = MiGA::Project.load(o[:project]) or raise 'Project doesn\'t exist, aborting.'
|
50
|
-
d = MiGA::Daemon.new(p)
|
51
|
-
[:latency, :maxjobs, :ppn, :shutdown_when_done].each do |k|
|
52
|
-
d.runopts(k, o[k]) unless o[k].nil?
|
53
|
-
end
|
54
|
-
d.daemon(task, o[:daemon_opts])
|
55
|
-
|
data/actions/date.rb
DELETED
data/actions/doctor.rb
DELETED
@@ -1,201 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
# @package MiGA
|
4
|
-
# @license Artistic-2.0
|
5
|
-
|
6
|
-
require "sqlite3"
|
7
|
-
|
8
|
-
tasks = {
|
9
|
-
db: ['databases', 'Check database files integrity'],
|
10
|
-
dist: ['distances', 'Check distance summary tables.'],
|
11
|
-
files: ['files', 'Check for outdated files.'],
|
12
|
-
ess: ['essential-genes', 'Check for unarchived essential genes'],
|
13
|
-
mts: ['mytaxa-scan', 'Check for unarchived MyTaxa scan'],
|
14
|
-
start: ['start', 'Check for lingering .start files'],
|
15
|
-
tax: ['taxonomy', 'Check for taxonomy consistency (not implemented)']
|
16
|
-
}
|
17
|
-
o = {q: true, ld: false}
|
18
|
-
tasks.keys.each{ |i| o[i] = true }
|
19
|
-
tasks_n = Hash[tasks.map{ |k,v| [v[0], k] }]
|
20
|
-
|
21
|
-
OptionParser.new do |opt|
|
22
|
-
opt_banner(opt)
|
23
|
-
opt_object(opt, o, [:project])
|
24
|
-
opt.on('-l', '--list-datasets',
|
25
|
-
'List all fixed datasets on advance.'){ |v| o[:ld]=v }
|
26
|
-
opt.on('--ignore TASK1,TASK2', Array,
|
27
|
-
'Do not perform the task(s) listed. Available tasks are:',
|
28
|
-
* tasks.values.map{ |v| "#{v[0]}: #{v[1]}" }
|
29
|
-
){ |v| v.map{ |i| o[tasks_n[i]] = false } }
|
30
|
-
opt.on('--only TASK',
|
31
|
-
'Perform only the specified task (see --ignore).'
|
32
|
-
){ |v| tasks.keys.each{ |i| o[i] = false }; o[v] = true }
|
33
|
-
opt_common(opt, o)
|
34
|
-
end.parse!
|
35
|
-
|
36
|
-
##=> Main <=
|
37
|
-
opt_require(o, project: '-P')
|
38
|
-
|
39
|
-
$stderr.puts 'Loading project' unless o[:q]
|
40
|
-
p = MiGA::Project.load(o[:project])
|
41
|
-
raise "Impossible to load project: #{o[:project]}" if p.nil?
|
42
|
-
|
43
|
-
def check_sqlite3_database(db_file, metric)
|
44
|
-
begin
|
45
|
-
SQLite3::Database.new(db_file) do |conn|
|
46
|
-
conn.execute("select count(*) from #{metric}").first
|
47
|
-
end
|
48
|
-
rescue SQLite3::SQLException
|
49
|
-
yield
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
if o[:db]
|
54
|
-
$stderr.puts 'o Checking databases integrity' unless o[:q]
|
55
|
-
p.each_dataset do |d|
|
56
|
-
[:distances, :taxonomy].each do |r_key|
|
57
|
-
r = d.result(r_key) or next
|
58
|
-
{haai_db: :aai, aai_db: :aai, ani_db: :ani}.each do |db_key, metric|
|
59
|
-
db_file = r.file_path(db_key) or next
|
60
|
-
check_sqlite3_database(db_file, metric) do
|
61
|
-
$stderr.puts(
|
62
|
-
" > Removing #{db_key} #{r_key} table for #{d.name}.") if o[:ld]
|
63
|
-
[db_file, r.path(:done), r.path].each do |f|
|
64
|
-
File.unlink f if File.exist? f
|
65
|
-
end # each |f|
|
66
|
-
end # check_sqlite3_database
|
67
|
-
end # each |db_key, metric|
|
68
|
-
end # each |r_key|
|
69
|
-
end # each |d|
|
70
|
-
end
|
71
|
-
|
72
|
-
[:ani, :aai].each do |dist|
|
73
|
-
res = p.result("#{dist}_distances")
|
74
|
-
next if res.nil?
|
75
|
-
$stderr.puts "o Checking #{dist} table for consistent datasets" unless o[:q]
|
76
|
-
notok = {}
|
77
|
-
fix = {}
|
78
|
-
Zlib::GzipReader.open(res.file_path(:matrix)) do |fh|
|
79
|
-
lineno = 0
|
80
|
-
fh.each_line do |ln|
|
81
|
-
next if (lineno+=1)==1
|
82
|
-
r = ln.split("\t")
|
83
|
-
if [1,2].map{ |i| p.dataset(r[i]).nil? }.any?
|
84
|
-
[1,2].each do |i|
|
85
|
-
if p.dataset(r[i]).nil?
|
86
|
-
notok[r[i]] = true
|
87
|
-
else
|
88
|
-
fix[r[i]] = true
|
89
|
-
end
|
90
|
-
end
|
91
|
-
end
|
92
|
-
end
|
93
|
-
end
|
94
|
-
|
95
|
-
$stderr.puts " - Fixing #{fix.size} datasets" unless fix.empty? or o[:q]
|
96
|
-
fix.keys.each do |d_n|
|
97
|
-
$stderr.puts " > Fixing #{d_n}." if o[:ld]
|
98
|
-
p.dataset(d_n).cleanup_distances!
|
99
|
-
end
|
100
|
-
|
101
|
-
unless notok.empty?
|
102
|
-
unless o[:q]
|
103
|
-
$stderr.puts ' - Unregistered datasets detected: '
|
104
|
-
if notok.size < 3
|
105
|
-
$stderr.puts " - #{notok.keys.join(', ')}"
|
106
|
-
else
|
107
|
-
$stderr.puts " - #{notok.size}, including #{notok.keys.first}"
|
108
|
-
end
|
109
|
-
$stderr.puts ' - Removing tables, recompute'
|
110
|
-
end
|
111
|
-
res.remove!
|
112
|
-
end
|
113
|
-
end if o[:dist]
|
114
|
-
|
115
|
-
if o[:files]
|
116
|
-
$stderr.puts 'o Looking for outdated files in results' unless o[:q]
|
117
|
-
p.each_dataset do |d|
|
118
|
-
d.each_result do |r_k, r|
|
119
|
-
ok = true
|
120
|
-
r.each_file do |_f_sym, _f_rel, f_abs|
|
121
|
-
unless File.exist? f_abs
|
122
|
-
ok = false
|
123
|
-
break
|
124
|
-
end
|
125
|
-
end
|
126
|
-
unless ok
|
127
|
-
$stderr.puts " > Registering again #{d.name}:#{r_k}" if o[:ld]
|
128
|
-
d.add_result(r_k, true, force: true)
|
129
|
-
end
|
130
|
-
end
|
131
|
-
end
|
132
|
-
end
|
133
|
-
|
134
|
-
if o[:ess]
|
135
|
-
$stderr.puts 'o Looking for unarchived essential genes.' unless o[:q]
|
136
|
-
p.each_dataset do |d|
|
137
|
-
res = d.result(:essential_genes)
|
138
|
-
next if res.nil?
|
139
|
-
dir = res.file_path(:collection)
|
140
|
-
if dir.nil?
|
141
|
-
$stderr.puts " > Removing #{d.name}:essential_genes" if o[:ld]
|
142
|
-
res.remove!
|
143
|
-
next
|
144
|
-
end
|
145
|
-
unless Dir["#{dir}/*.faa"].empty?
|
146
|
-
$stderr.puts " > Fixing #{d.name}." if o[:ld]
|
147
|
-
cmdo = `cd '#{dir}' && tar -zcf proteins.tar.gz *.faa && rm *.faa`.chomp
|
148
|
-
warn cmdo unless cmdo.empty?
|
149
|
-
end
|
150
|
-
end
|
151
|
-
end
|
152
|
-
|
153
|
-
if o[:mts]
|
154
|
-
$stderr.puts 'o Looking for unarchived MyTaxa Scan runs.' unless o[:q]
|
155
|
-
p.each_dataset do |d|
|
156
|
-
res = d.result(:mytaxa_scan)
|
157
|
-
next if res.nil?
|
158
|
-
dir = res.file_path(:regions)
|
159
|
-
fix = false
|
160
|
-
unless dir.nil?
|
161
|
-
if Dir.exist? dir
|
162
|
-
cmdo = `cd '#{dir}/..' \
|
163
|
-
&& tar -zcf '#{d.name}.reg.tar.gz' '#{d.name}.reg' \
|
164
|
-
&& rm -r '#{d.name}.reg'`.chomp
|
165
|
-
warn cmdo unless cmdo.empty?
|
166
|
-
end
|
167
|
-
fix = true
|
168
|
-
end
|
169
|
-
%w[blast mytaxain wintax gene_ids region_ids].each do |ext|
|
170
|
-
file = res.file_path(ext.to_sym)
|
171
|
-
unless file.nil?
|
172
|
-
FileUtils.rm(file) if File.exist? file
|
173
|
-
fix = true
|
174
|
-
end
|
175
|
-
end
|
176
|
-
if fix
|
177
|
-
$stderr.puts " > Fixing #{d.name}." if o[:ld]
|
178
|
-
d.add_result(:mytaxa_scan, true, force: true)
|
179
|
-
end
|
180
|
-
end
|
181
|
-
end
|
182
|
-
|
183
|
-
if o[:start]
|
184
|
-
$stderr.puts 'o Looking for legacy .start files lingering.' unless o[:q]
|
185
|
-
p.each_dataset do |d|
|
186
|
-
d.each_result do |r_k, r|
|
187
|
-
if File.exist? r.path(:start)
|
188
|
-
$stderr.puts " > Registering again #{d.name}:#{r_k}" if o[:ld]
|
189
|
-
r.save
|
190
|
-
end
|
191
|
-
end
|
192
|
-
end
|
193
|
-
end
|
194
|
-
|
195
|
-
if o[:tax]
|
196
|
-
#$stderr.puts "o Checking for taxonomy/distances consistency" unless o[:q]
|
197
|
-
# TODO: Find 95%ANI clusters with entries from different species
|
198
|
-
end
|
199
|
-
|
200
|
-
$stderr.puts 'Done' unless o[:q]
|
201
|
-
|
data/actions/edit.rb
DELETED
@@ -1,33 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
# @package MiGA
|
4
|
-
# @license Artistic-2.0
|
5
|
-
|
6
|
-
o = {q: true}
|
7
|
-
OptionParser.new do |opt|
|
8
|
-
opt_banner(opt)
|
9
|
-
opt_object(opt, o, [:project, :dataset_opt])
|
10
|
-
opt.on('-m', '--metadata STRING',
|
11
|
-
'Metadata as key-value pairs separated by = and delimited by comma.',
|
12
|
-
'Values are saved as strings except for booleans (true / false) or nil.'
|
13
|
-
){ |v| o[:metadata] = v }
|
14
|
-
opt_common(opt, o)
|
15
|
-
end.parse!
|
16
|
-
|
17
|
-
##=> Main <=
|
18
|
-
opt_require(o, project: '-P')
|
19
|
-
|
20
|
-
$stderr.puts 'Loading project.' unless o[:q]
|
21
|
-
p = MiGA::Project.load(o[:project])
|
22
|
-
raise "Impossible to load project: #{o[:project]}" if p.nil?
|
23
|
-
|
24
|
-
obj = p
|
25
|
-
if o[:dataset]
|
26
|
-
$stderr.puts 'Loading dataset.' unless o[:q]
|
27
|
-
obj = p.dataset(o[:dataset])
|
28
|
-
raise 'Dataset does not exist.' if obj.nil?
|
29
|
-
end
|
30
|
-
obj = add_metadata(o, obj)
|
31
|
-
obj.save
|
32
|
-
|
33
|
-
$stderr.puts 'Done.' unless o[:q]
|