miga-base 0.4.1.0 → 0.4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/miga +2 -244
- data/lib/miga/cli/action/about.rb +44 -0
- data/lib/miga/cli/action/add.rb +139 -0
- data/lib/miga/cli/action/add_result.rb +26 -0
- data/lib/miga/cli/action/console.rb +19 -0
- data/lib/miga/cli/action/daemon.rb +74 -0
- data/lib/miga/cli/action/date.rb +18 -0
- data/lib/miga/cli/action/doctor.rb +210 -0
- data/lib/miga/cli/action/edit.rb +24 -0
- data/lib/miga/cli/action/files.rb +31 -0
- data/lib/miga/cli/action/find.rb +48 -0
- data/lib/miga/cli/action/generic.rb +44 -0
- data/lib/miga/cli/action/get.rb +132 -0
- data/lib/miga/cli/action/init.rb +343 -0
- data/lib/miga/cli/action/ln.rb +42 -0
- data/lib/miga/cli/action/ls.rb +55 -0
- data/lib/miga/cli/action/ncbi_get.rb +218 -0
- data/lib/miga/cli/action/new.rb +45 -0
- data/lib/miga/cli/action/next_step.rb +27 -0
- data/lib/miga/cli/action/plugins.rb +28 -0
- data/lib/miga/cli/action/rm.rb +25 -0
- data/lib/miga/cli/action/run.rb +39 -0
- data/lib/miga/cli/action/stats.rb +140 -0
- data/lib/miga/cli/action/summary.rb +49 -0
- data/lib/miga/cli/action/tax_dist.rb +102 -0
- data/lib/miga/cli/action/tax_index.rb +47 -0
- data/lib/miga/cli/action/tax_set.rb +59 -0
- data/lib/miga/cli/action/tax_test.rb +77 -0
- data/lib/miga/cli/action.rb +66 -0
- data/lib/miga/cli/base.rb +90 -0
- data/lib/miga/cli.rb +426 -0
- data/lib/miga/project/result.rb +14 -6
- data/lib/miga/remote_dataset.rb +1 -1
- data/lib/miga/tax_index.rb +5 -4
- data/lib/miga/taxonomy/base.rb +63 -0
- data/lib/miga/taxonomy.rb +87 -92
- data/lib/miga/version.rb +6 -6
- data/test/taxonomy_test.rb +49 -9
- data/utils/distance/commands.rb +11 -11
- data/utils/distance/pipeline.rb +5 -5
- metadata +43 -49
- data/actions/about.rb +0 -43
- data/actions/add.rb +0 -129
- data/actions/add_result.rb +0 -30
- data/actions/daemon.rb +0 -55
- data/actions/date.rb +0 -14
- data/actions/doctor.rb +0 -201
- data/actions/edit.rb +0 -33
- data/actions/files.rb +0 -43
- data/actions/find.rb +0 -41
- data/actions/get.rb +0 -105
- data/actions/init.rb +0 -301
- data/actions/ln.rb +0 -47
- data/actions/ls.rb +0 -61
- data/actions/ncbi_get.rb +0 -192
- data/actions/new.rb +0 -44
- data/actions/next_step.rb +0 -33
- data/actions/plugins.rb +0 -25
- data/actions/rm.rb +0 -29
- data/actions/run.rb +0 -45
- data/actions/stats.rb +0 -149
- data/actions/summary.rb +0 -57
- data/actions/tax_dist.rb +0 -106
- data/actions/tax_index.rb +0 -46
- data/actions/tax_set.rb +0 -63
- data/actions/tax_test.rb +0 -80
data/actions/files.rb
DELETED
@@ -1,43 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
# @package MiGA
|
4
|
-
# @license Artistic-2.0
|
5
|
-
|
6
|
-
o = {q:true, details:false, json:true}
|
7
|
-
OptionParser.new do |opt|
|
8
|
-
opt_banner(opt)
|
9
|
-
opt_object(opt, o, [:project, :dataset_opt])
|
10
|
-
opt.on("-i", "--info",
|
11
|
-
"If set, it prints additional details for each file."
|
12
|
-
){ |v| o[:details]=v }
|
13
|
-
opt.on("--[no-]json",
|
14
|
-
"If set to no, excludes json files containing results metadata."
|
15
|
-
){ |v| o[:json]=v }
|
16
|
-
opt_common(opt, o)
|
17
|
-
end.parse!
|
18
|
-
|
19
|
-
##=> Main <=
|
20
|
-
opt_require(o, project:"-P")
|
21
|
-
|
22
|
-
$stderr.puts "Loading project." unless o[:q]
|
23
|
-
p = MiGA::Project.load(o[:project])
|
24
|
-
raise "Impossible to load project: #{o[:project]}" if p.nil?
|
25
|
-
|
26
|
-
if o[:dataset].nil?
|
27
|
-
results = p.results
|
28
|
-
else
|
29
|
-
$stderr.puts "Loading dataset." unless o[:q]
|
30
|
-
ds = p.dataset(o[:dataset])
|
31
|
-
raise "Impossible to load dataset: #{o[:dataset]}" if ds.nil?
|
32
|
-
results = ds.results
|
33
|
-
end
|
34
|
-
|
35
|
-
$stderr.puts "Listing files." unless o[:q]
|
36
|
-
results.each do |result|
|
37
|
-
puts "#{ "#{result.path}\t\t" if o[:details] }#{result.path}" if o[:json]
|
38
|
-
result.each_file do |k,f|
|
39
|
-
puts "#{ "#{result.path}\t#{k}\t" if o[:details] }#{result.dir}/#{f}"
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
$stderr.puts "Done." unless o[:q]
|
data/actions/find.rb
DELETED
@@ -1,41 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
# @package MiGA
|
4
|
-
# @license Artistic-2.0
|
5
|
-
|
6
|
-
o = {q:true, add:false, ref:true}
|
7
|
-
OptionParser.new do |opt|
|
8
|
-
opt_banner(opt)
|
9
|
-
opt_object(opt, o, [:project, :dataset_type])
|
10
|
-
opt.on("-a", "--add",
|
11
|
-
"Register the datasets found. By default, only lists them (dry run)."
|
12
|
-
){ |v| o[:add]=v }
|
13
|
-
opt.on("-q", "--query",
|
14
|
-
"If set, all datasets are registered as query datasets."
|
15
|
-
){ |v| o[:ref]=!v }
|
16
|
-
opt.on("-u", "--user STRING", "Owner of the dataset."){ |v| o[:user]=v }
|
17
|
-
opt_common(opt, o)
|
18
|
-
end.parse!
|
19
|
-
|
20
|
-
##=> Main <=
|
21
|
-
opt_require(o, project:"-P")
|
22
|
-
|
23
|
-
$stderr.puts "Loading project." unless o[:q]
|
24
|
-
p = MiGA::Project.load(o[:project])
|
25
|
-
raise "Impossible to load project: #{o[:project]}" if p.nil?
|
26
|
-
|
27
|
-
$stderr.puts "Finding datasets." unless o[:q]
|
28
|
-
ud = p.unregistered_datasets
|
29
|
-
ud.each do |dn|
|
30
|
-
puts dn
|
31
|
-
if o[:add]
|
32
|
-
md = {}
|
33
|
-
[:type, :user].each{ |k| md[k]=o[k] unless o[k].nil? }
|
34
|
-
d = MiGA::Dataset.new(p, dn, o[:ref], md)
|
35
|
-
p.add_dataset(dn)
|
36
|
-
res = d.first_preprocessing(true)
|
37
|
-
puts "- #{res}" unless o[:q]
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
$stderr.puts "Done." unless o[:q]
|
data/actions/get.rb
DELETED
@@ -1,105 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
# @package MiGA
|
4
|
-
# @license Artistic-2.0
|
5
|
-
|
6
|
-
require 'miga/remote_dataset'
|
7
|
-
|
8
|
-
o = {q: true, query: false, universe: :ebi, db: :embl,
|
9
|
-
get_md: false, only_md: false}
|
10
|
-
OptionParser.new do |opt|
|
11
|
-
opt_banner(opt)
|
12
|
-
opt_object(opt, o, [:project, :dataset, :dataset_type])
|
13
|
-
opt.on('-I', '--ids ID1,ID2,...',
|
14
|
-
'(Mandatory unless -F) IDs in the remote database separated by commas.'
|
15
|
-
){ |v| o[:ids]=v }
|
16
|
-
opt.on('-U', '--universe STRING',
|
17
|
-
"Universe where the remote database lives. By default: #{o[:universe]}."
|
18
|
-
){ |v| o[:universe]=v.to_sym }
|
19
|
-
opt.on('--db STRING',
|
20
|
-
"Name of the remote database. By default: #{o[:db]}."
|
21
|
-
){ |v| o[:db]=v.to_sym }
|
22
|
-
opt.on('-F', '--file PATH',
|
23
|
-
'Tab-delimited file (with header) listing the datasets to download.',
|
24
|
-
'The long form of all the options are supported as header (without the --)',
|
25
|
-
'including dataset, ids, universe, and db. For query use true/false values.'
|
26
|
-
){ |v| o[:file] = v }
|
27
|
-
opt.on('-q', '--query',
|
28
|
-
'If set, the dataset is registered as a query, not a reference dataset.'
|
29
|
-
){ |v| o[:query]=v }
|
30
|
-
opt.on('--ignore-dup',
|
31
|
-
'If set, ignores datasets that already exist.'){ |v| o[:ignore_dup]=v }
|
32
|
-
opt.on('-d', '--description STRING',
|
33
|
-
'Description of the dataset.'){ |v| o[:description]=v }
|
34
|
-
opt.on('-c', '--comments STRING',
|
35
|
-
'Comments on the dataset.'){ |v| o[:comments]=v }
|
36
|
-
opt.on('-m', '--metadata STRING',
|
37
|
-
'Metadata as key-value pairs separated by = and delimited by comma.',
|
38
|
-
'Values are saved as strings except for booleans (true / false) or nil.'
|
39
|
-
){ |v| o[:metadata]=v }
|
40
|
-
opt.on('--get-metadata',
|
41
|
-
'Only download and update metadata for existing datasets'
|
42
|
-
){ |v| o[:get_md] = v }
|
43
|
-
opt.on('--only-metadata',
|
44
|
-
'Create datasets without input data but retrieve all metadata.'
|
45
|
-
){ |v| o[:only_md] = v }
|
46
|
-
opt.on('--api-key STRING',
|
47
|
-
'API key for the given universe.'){ |v| o[:api_key] = v }
|
48
|
-
opt_common(opt, o)
|
49
|
-
end.parse!
|
50
|
-
|
51
|
-
##=> Main <=
|
52
|
-
glob = [o]
|
53
|
-
unless o[:file].nil?
|
54
|
-
glob = []
|
55
|
-
fh = File.open(o[:file], 'r')
|
56
|
-
h = nil
|
57
|
-
fh.each do |ln|
|
58
|
-
r = ln.chomp.split(/\t/)
|
59
|
-
if h.nil?
|
60
|
-
h = r
|
61
|
-
else
|
62
|
-
glob << o.dup
|
63
|
-
h.each_index do |i|
|
64
|
-
glob[glob.size-1][h[i].to_sym] = h[i]=='query' ? r[i]=='true' :
|
65
|
-
%w[type universe db].include?(h[i]) ? r[i].to_sym : r[i]
|
66
|
-
end
|
67
|
-
end
|
68
|
-
end
|
69
|
-
fh.close
|
70
|
-
end
|
71
|
-
|
72
|
-
glob.each do |o_i|
|
73
|
-
opt_require(o_i, project: '-P', dataset: '-D', ids: '-I')
|
74
|
-
unless o_i[:api_key].nil?
|
75
|
-
ENV["#{o_i[:universe].to_s.upcase}_API_KEY"] = o_i[:api_key]
|
76
|
-
end
|
77
|
-
|
78
|
-
$stderr.puts "Dataset: #{o_i[:dataset]}" unless o_i[:q]
|
79
|
-
$stderr.puts 'Loading project.' unless o_i[:q]
|
80
|
-
p = MiGA::Project.load(o_i[:project])
|
81
|
-
raise "Impossible to load project: #{o_i[:project]}" if p.nil?
|
82
|
-
|
83
|
-
next if o_i[:ignore_dup] and not p.dataset(o_i[:dataset]).nil?
|
84
|
-
|
85
|
-
$stderr.puts 'Locating remote dataset.' unless o_i[:q]
|
86
|
-
rd = MiGA::RemoteDataset.new(o_i[:ids], o_i[:db], o_i[:universe])
|
87
|
-
|
88
|
-
if o[:get_md]
|
89
|
-
$stderr.puts 'Updating dataset.' unless o_i[:q]
|
90
|
-
d = p.dataset(o_i[:dataset])
|
91
|
-
next if d.nil?
|
92
|
-
md = add_metadata(o_i, d).metadata.data
|
93
|
-
rd.update_metadata(d, md)
|
94
|
-
else
|
95
|
-
$stderr.puts 'Creating dataset.' unless o_i[:q]
|
96
|
-
dummy_d = MiGA::Dataset.new(p, o_i[:dataset])
|
97
|
-
md = add_metadata(o_i, dummy_d).metadata.data
|
98
|
-
md[:metadata_only] = true if o[:only_md]
|
99
|
-
dummy_d.remove!
|
100
|
-
rd.save_to(p, o_i[:dataset], !o_i[:query], md)
|
101
|
-
p.add_dataset(o_i[:dataset])
|
102
|
-
end
|
103
|
-
|
104
|
-
$stderr.puts 'Done.' unless o_i[:q]
|
105
|
-
end
|
data/actions/init.rb
DELETED
@@ -1,301 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
# @package MiGA
|
4
|
-
# @license Artistic-2.0
|
5
|
-
|
6
|
-
require 'shellwords'
|
7
|
-
|
8
|
-
o = {q: true, mytaxa: nil,
|
9
|
-
config: File.expand_path('.miga_modules', ENV['HOME']),
|
10
|
-
ask: false, auto: false, dtype: 'bash'}
|
11
|
-
OptionParser.new do |opt|
|
12
|
-
opt_banner(opt)
|
13
|
-
opt.on('-c', '--config PATH',
|
14
|
-
'Path to the Bash configuration file.',
|
15
|
-
"By default: #{o[:config]}."){ |v| o[:config] = v }
|
16
|
-
opt.on('--[no-]mytaxa',
|
17
|
-
'Should I try setting up MyTaxa its dependencies?',
|
18
|
-
'By default: interactive (true if --auto).'
|
19
|
-
){ |v| o[:mytaxa] = v }
|
20
|
-
opt.on('--daemon-type STRING',
|
21
|
-
'Type of daemon launcher, one of: bash, qsub, msub, slurm.',
|
22
|
-
"By default: interactive (#{o[:dtype]} if --auto)."
|
23
|
-
){ |v| o[:dtype]=v }
|
24
|
-
opt.on('--ask-all', 'If set, asks for the location of all software.',
|
25
|
-
'By default, only the locations missing in PATH are requested.'
|
26
|
-
){ |v| o[:ask] = v }
|
27
|
-
opt.on('--auto', 'If set, accepts all defaults as answers.'
|
28
|
-
){ |v| o[:auto] = v }
|
29
|
-
opt_common(opt, o)
|
30
|
-
end.parse!
|
31
|
-
$auto_answer = o[:auto]
|
32
|
-
|
33
|
-
def ask_user(q, d = nil, ans = nil, force = false)
|
34
|
-
$stderr.print "#{q}#{" (#{ans.join(" / ")})" unless ans.nil?}" +
|
35
|
-
"#{" [#{d}]" unless d.nil?} > "
|
36
|
-
if $auto_answer and not force
|
37
|
-
$stderr.puts ''
|
38
|
-
else
|
39
|
-
o = gets.chomp
|
40
|
-
end
|
41
|
-
o = d if o.nil? or o.empty?
|
42
|
-
unless ans.nil? or ans.include? o
|
43
|
-
$stderr.puts 'Answer not recognized.'
|
44
|
-
return ask_user(q, d, ans)
|
45
|
-
end
|
46
|
-
o
|
47
|
-
end
|
48
|
-
|
49
|
-
##=> Main <=
|
50
|
-
|
51
|
-
miga = MiGA::MiGA.root_path
|
52
|
-
$stderr.puts <<BANNER
|
53
|
-
===[ Welcome to MiGA, the Microbial Genome Atlas ]===
|
54
|
-
|
55
|
-
I'm the initialization script, and I'll sniff around your computer to
|
56
|
-
make sure you have all the requirements for MiGA data processing.
|
57
|
-
|
58
|
-
BANNER
|
59
|
-
|
60
|
-
if ask_user(
|
61
|
-
'Would you like to see all the requirements before starting?',
|
62
|
-
'no', %w(yes no)) == 'yes'
|
63
|
-
File.open(File.expand_path('utils/requirements.txt', miga), 'r') do |fh|
|
64
|
-
fh.each_line{ |ln| $stderr.puts ln }
|
65
|
-
end
|
66
|
-
end
|
67
|
-
|
68
|
-
rc_path = File.expand_path('.miga_rc', ENV['HOME'])
|
69
|
-
if File.exist? rc_path
|
70
|
-
if ask_user(
|
71
|
-
'I found a previous configuration. Do you want to continue?',
|
72
|
-
'yes', %w(yes no)) == 'no'
|
73
|
-
$stderr.puts 'OK, see you soon!'
|
74
|
-
exit 0
|
75
|
-
end
|
76
|
-
end
|
77
|
-
rc_fh = File.open(rc_path, 'w')
|
78
|
-
rc_fh.puts <<BASH
|
79
|
-
#!/bin/bash
|
80
|
-
# `miga init` made this on #{Time.now}
|
81
|
-
|
82
|
-
BASH
|
83
|
-
|
84
|
-
# Check bash configuration file
|
85
|
-
unless File.exist? o[:config]
|
86
|
-
o[:config] = ask_user(
|
87
|
-
'Is there a script I need to load at startup?', o[:config])
|
88
|
-
end
|
89
|
-
if File.exist? o[:config]
|
90
|
-
o[:config] = File.expand_path o[:config]
|
91
|
-
$stderr.puts "Found bash configuration script: #{o[:config]}."
|
92
|
-
rc_fh.puts "MIGA_STARTUP='#{o[:config]}'"
|
93
|
-
rc_fh.puts '. "$MIGA_STARTUP"'
|
94
|
-
end
|
95
|
-
$stderr.puts ''
|
96
|
-
|
97
|
-
# Check for software requirements
|
98
|
-
$stderr.puts 'Looking for requirements:'
|
99
|
-
if o[:mytaxa].nil?
|
100
|
-
o[:mytaxa] = ask_user(
|
101
|
-
'Should I include MyTaxa modules?', 'yes', %w(yes no)) == 'yes'
|
102
|
-
end
|
103
|
-
rc_fh.puts 'export MIGA_MYTAXA="no"' unless o[:mytaxa]
|
104
|
-
paths = {}
|
105
|
-
rc_fh.puts 'MIGA_PATH=""'
|
106
|
-
File.open(File.expand_path('utils/requirements.txt', miga), 'r') do |fh|
|
107
|
-
fh.each_line do |ln|
|
108
|
-
next if $. < 3
|
109
|
-
r = ln.chomp.split(/\t+/)
|
110
|
-
next if r[0] =~ /\(opt\)$/ and not o[:mytaxa]
|
111
|
-
$stderr.print "Testing #{r[0]}#{" (#{r[3]})" if r[3]}... "
|
112
|
-
path = nil
|
113
|
-
loop do
|
114
|
-
if File.exist? o[:config]
|
115
|
-
d_path = File.dirname(`. "#{o[:config]}" && which "#{r[1]}"`)
|
116
|
-
else
|
117
|
-
d_path = File.dirname(`which "#{r[1]}"`)
|
118
|
-
end
|
119
|
-
if o[:ask] or d_path == '.'
|
120
|
-
path = ask_user('Where can I find it?', d_path, nil, true)
|
121
|
-
else
|
122
|
-
path = d_path
|
123
|
-
$stderr.puts path
|
124
|
-
end
|
125
|
-
if File.executable? File.expand_path(r[1], path)
|
126
|
-
if d_path != path
|
127
|
-
rc_fh.puts "MIGA_PATH=\"#{path}:$MIGA_PATH\" # #{r[1]}"
|
128
|
-
end
|
129
|
-
break
|
130
|
-
end
|
131
|
-
$stderr.print "I cannot find #{r[1]}. "
|
132
|
-
end
|
133
|
-
paths[r[1]] = File.expand_path(r[1], path).shellescape
|
134
|
-
end
|
135
|
-
end
|
136
|
-
rc_fh.puts 'export PATH="$MIGA_PATH$PATH"'
|
137
|
-
$stderr.puts ''
|
138
|
-
|
139
|
-
# Check for other files
|
140
|
-
if o[:mytaxa]
|
141
|
-
$stderr.puts 'Looking for MyTaxa databases:'
|
142
|
-
mt = File.dirname paths["MyTaxa"]
|
143
|
-
$stderr.print 'Looking for scores... '
|
144
|
-
unless Dir.exist? File.expand_path('db', mt)
|
145
|
-
$stderr.puts "no.\nExecute 'python2 #{mt}/utils/download_db.py'."
|
146
|
-
exit 1
|
147
|
-
end
|
148
|
-
$stderr.puts 'yes.'
|
149
|
-
$stderr.print 'Looking for diamond db... '
|
150
|
-
unless File.exist? File.expand_path('AllGenomes.faa.dmnd', mt)
|
151
|
-
$stderr.puts "no.\nDownload " +
|
152
|
-
"'http://enve-omics.ce.gatech.edu/data/public_mytaxa/" +
|
153
|
-
"AllGenomes.faa.dmnd' into #{mt}."
|
154
|
-
exit 1
|
155
|
-
end
|
156
|
-
$stderr.puts ''
|
157
|
-
end
|
158
|
-
|
159
|
-
# Check for R packages
|
160
|
-
$stderr.puts 'Looking for R packages:'
|
161
|
-
%w(enveomics.R ape cluster vegan).each do |pkg|
|
162
|
-
$stderr.print "Testing #{pkg}... "
|
163
|
-
`echo "library('#{pkg}')" | #{paths["R"].shellescape} --vanilla -q 2>&1`
|
164
|
-
if $?.success?
|
165
|
-
$stderr.puts 'yes.'
|
166
|
-
else
|
167
|
-
$stderr.puts 'no, installing.'
|
168
|
-
$stderr.print '' +
|
169
|
-
`echo "install.packages('#{pkg}', repos='http://cran.rstudio.com/')" \
|
170
|
-
| #{paths["R"].shellescape} --vanilla -q 2>&1`
|
171
|
-
`echo "library('#{pkg}')" | #{paths["R"].shellescape} --vanilla -q 2>&1`
|
172
|
-
raise "Unable to auto-install R package #{pkg}." unless $?.success?
|
173
|
-
end
|
174
|
-
end
|
175
|
-
$stderr.puts ''
|
176
|
-
|
177
|
-
# Check for Ruby gems
|
178
|
-
$stderr.puts 'Looking for Ruby gems:'
|
179
|
-
%w(sqlite3 daemons json).each do |pkg|
|
180
|
-
$stderr.print "Testing #{pkg}... "
|
181
|
-
`#{paths["ruby"].shellescape} -r "#{pkg}" -e "" 2>/dev/null`
|
182
|
-
if $?.success?
|
183
|
-
$stderr.puts 'yes.'
|
184
|
-
else
|
185
|
-
$stderr.puts 'no, installing.'
|
186
|
-
# This hackey mess is meant to ensure the test and installation are done
|
187
|
-
# on the configuration Ruby, not on the Ruby currently executing the init
|
188
|
-
# action
|
189
|
-
$stderr.print `#{paths['ruby'].shellescape} \
|
190
|
-
-r rubygems -r rubygems/gem_runner \
|
191
|
-
-e "Gem::GemRunner.new.run %w(install --user #{pkg})" 2>&1`
|
192
|
-
raise "Unable to auto-install Ruby gem #{pkg}." unless $?.success?
|
193
|
-
end
|
194
|
-
end
|
195
|
-
$stderr.puts ''
|
196
|
-
|
197
|
-
# Configure daemon
|
198
|
-
$stderr.puts 'Default daemon configuration:'
|
199
|
-
daemon_f = File.expand_path('.miga_daemon.json', ENV['HOME'])
|
200
|
-
unless File.exist?(daemon_f) and ask_user(
|
201
|
-
'A template daemon already exists, do you want to preserve it?',
|
202
|
-
'yes', %w(yes no)) == 'yes'
|
203
|
-
v = {created:Time.now.to_s, updated:Time.now.to_s}
|
204
|
-
v[:type] = ask_user('Please select the type of daemon you want to setup',
|
205
|
-
o[:dtype], %w(bash qsub msub slurm))
|
206
|
-
case v[:type]
|
207
|
-
when 'bash'
|
208
|
-
v[:latency] = ask_user('How long should I sleep? (in seconds)', '30').to_i
|
209
|
-
v[:maxjobs] = ask_user('How many jobs can I launch at once?', '6').to_i
|
210
|
-
v[:ppn] = ask_user('How many CPUs can I use per job?', '2').to_i
|
211
|
-
$stderr.puts 'Setting up internal daemon defaults.'
|
212
|
-
$stderr.puts 'If you don\'t understand this just leave default values:'
|
213
|
-
v[:cmd] = ask_user(
|
214
|
-
"How should I launch tasks?\n %1$s: script path, %2$s: variables, " +
|
215
|
-
"%3$d: CPUs, %4$s: log file, %5$s: task name.\n",
|
216
|
-
"%2$s '%1$s' > '%4$s' 2>&1")
|
217
|
-
v[:var] = ask_user(
|
218
|
-
"How should I pass variables?\n %1$s: keys, %2$s: values.\n",
|
219
|
-
"%1$s=%2$s")
|
220
|
-
v[:varsep] = ask_user('What should I use to separate variables?', ' ')
|
221
|
-
v[:alive] = ask_user(
|
222
|
-
"How can I know that a process is still alive?\n %1$s: PID, " +
|
223
|
-
"output should be 1 for running and 0 for non-running.\n",
|
224
|
-
"ps -p '%1$s'|tail -n+2|wc -l")
|
225
|
-
v[:kill] = ask_user(
|
226
|
-
"How should I terminate tasks?\n %s: process ID.", "kill -9 '%s'")
|
227
|
-
when 'slurm'
|
228
|
-
queue = ask_user('What queue should I use?', nil, nil, true)
|
229
|
-
v[:latency] = ask_user(
|
230
|
-
'How long should I sleep? (in seconds)', '150').to_i
|
231
|
-
v[:maxjobs] = ask_user('How many jobs can I launch at once?', '300').to_i
|
232
|
-
v[:ppn] = ask_user('How many CPUs can I use per job?', '2').to_i
|
233
|
-
$stderr.puts 'Setting up internal daemon defaults.'
|
234
|
-
$stderr.puts 'If you don\'t understand this just leave default values:'
|
235
|
-
v[:cmd] = ask_user(
|
236
|
-
"How should I launch tasks?\n %1$s: script path, %2$s: variables, " +
|
237
|
-
"%3$d: CPUs, %4$d: log file, %5$s: task name.\n",
|
238
|
-
"%2$s sbatch --partition='#{queue}' --export=ALL " +
|
239
|
-
"--nodes=1 --ntasks-per-node=%3$d --output='%4$s' --job-name='%5$s' " +
|
240
|
-
"--mem=9G --time=12:00:00 %1$s | perl -pe 's/.* //'")
|
241
|
-
v[:var] = ask_user(
|
242
|
-
"How should I pass variables?\n %1$s: keys, %2$s: values.\n",
|
243
|
-
"%1$s=%2$s")
|
244
|
-
v[:varsep] = ask_user('What should I use to separate variables?', ' ')
|
245
|
-
v[:alive] = ask_user(
|
246
|
-
"How can I know that a process is still alive?\n %1$s: job id, " +
|
247
|
-
"output should be 1 for running and 0 for non-running.\n",
|
248
|
-
"squeue -h -o %%t -j '%1$s' | grep '^PD\\|R\\|CF\\|CG$' " +
|
249
|
-
"| tail -n 1 | wc -l")
|
250
|
-
v[:kill] = ask_user(
|
251
|
-
"How should I terminate tasks?\n %s: process ID.", "scancel '%s'")
|
252
|
-
else # [qm]sub
|
253
|
-
queue = ask_user('What queue should I use?', nil, nil, true)
|
254
|
-
v[:latency] = ask_user(
|
255
|
-
'How long should I sleep? (in seconds)', '150').to_i
|
256
|
-
v[:maxjobs] = ask_user('How many jobs can I launch at once?', '300').to_i
|
257
|
-
v[:ppn] = ask_user('How many CPUs can I use per job?', '2').to_i
|
258
|
-
$stderr.puts 'Setting up internal daemon defaults.'
|
259
|
-
$stderr.puts 'If you don\'t understand this just leave default values:'
|
260
|
-
v[:cmd] = ask_user(
|
261
|
-
"How should I launch tasks?\n %1$s: script path, %2$s: variables, " +
|
262
|
-
"%3$d: CPUs, %4$d: log file, %5$s: task name.\n",
|
263
|
-
"#{v[:type]} -q '#{queue}' -v '%2$s' -l nodes=1:ppn=%3$d %1$s " +
|
264
|
-
"-j oe -o '%4$s' -N '%5$s' -l mem=9g -l walltime=12:00:00 | grep .")
|
265
|
-
v[:var] = ask_user(
|
266
|
-
"How should I pass variables?\n %1$s: keys, %2$s: values.\n",
|
267
|
-
"%1$s=%2$s")
|
268
|
-
v[:varsep] = ask_user('What should I use to separate variables?', ',')
|
269
|
-
if v[:type] == 'qsub'
|
270
|
-
v[:alive] = ask_user(
|
271
|
-
"How can I know that a process is still alive?\n %1$s: job id, " +
|
272
|
-
"output should be 1 for running and 0 for non-running.\n",
|
273
|
-
"qstat -f '%1$s'|grep ' job_state ='|perl -pe 's/.*= //'|grep '[^C]'"+
|
274
|
-
"|tail -n1|wc -l|awk '{print $1}'")
|
275
|
-
v[:kill] = ask_user(
|
276
|
-
"How should I terminate tasks?\n %s: process ID.", "qdel '%s'")
|
277
|
-
else
|
278
|
-
v[:alive] = ask_user(
|
279
|
-
"How can I know that a process is still alive?\n %1$s: job id, " +
|
280
|
-
"output should be 1 for running and 0 for non-running.\n",
|
281
|
-
"checkjob '%1$s'|grep '^State:'|perl -pe 's/.*: //'" +
|
282
|
-
"|grep 'Deferred\\|Hold\\|Idle\\|Starting\\|Running\\|Blocked'"+
|
283
|
-
"|tail -n1|wc -l|awk '{print $1}'")
|
284
|
-
v[:kill] = ask_user(
|
285
|
-
"How should I terminate tasks?\n %s: process ID.", "canceljob '%s'")
|
286
|
-
end
|
287
|
-
end
|
288
|
-
File.open(daemon_f, 'w'){ |fh| fh.puts JSON.pretty_generate(v) }
|
289
|
-
end
|
290
|
-
$stderr.puts ''
|
291
|
-
|
292
|
-
rc_fh.puts <<FOOT
|
293
|
-
|
294
|
-
MIGA_CONFIG_VERSION='#{MiGA::MiGA.VERSION}'
|
295
|
-
MIGA_CONFIG_LONGVERSION='#{MiGA::MiGA.LONG_VERSION}'
|
296
|
-
MIGA_CONFIG_DATE='#{Time.now}'
|
297
|
-
|
298
|
-
FOOT
|
299
|
-
|
300
|
-
$stderr.puts 'Configuration complete. MiGA is ready to work!'
|
301
|
-
$stderr.puts ''
|
data/actions/ln.rb
DELETED
@@ -1,47 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
# @package MiGA
|
4
|
-
# @license Artistic-2.0
|
5
|
-
|
6
|
-
o = {q:true, info:false, force:false, method: :hardlink }
|
7
|
-
OptionParser.new do |opt|
|
8
|
-
opt_banner(opt)
|
9
|
-
opt_object(opt, o, [:project, :dataset_opt])
|
10
|
-
opt.on("-Q", "--project-target PATH",
|
11
|
-
"(Mandatory) Path to the project where to link the dataset."
|
12
|
-
){ |v| o[:project2]=v }
|
13
|
-
opt.on("-f", "--force",
|
14
|
-
"Forces linking, even if dataset's preprocessing is incomplete."
|
15
|
-
){ |v| o[:force]=v }
|
16
|
-
opt.on("-s", "--symlink",
|
17
|
-
"Creates symlinks instead of the default hard links."
|
18
|
-
){ o[:method] = :symlink }
|
19
|
-
opt.on("-c", "--copy",
|
20
|
-
"Creates copies instead of the default hard links."){ o[:method] = :copy }
|
21
|
-
opt_filter_datasets(opt, o)
|
22
|
-
opt_common(opt, o)
|
23
|
-
end.parse!
|
24
|
-
|
25
|
-
##=> Main <=
|
26
|
-
opt_require(o, project:"-P", project2:"-Q")
|
27
|
-
|
28
|
-
$stderr.puts "Loading project." unless o[:q]
|
29
|
-
p = MiGA::Project.load(o[:project])
|
30
|
-
raise "Impossible to load project: #{o[:project]}" if p.nil?
|
31
|
-
q = MiGA::Project.load(o[:project2])
|
32
|
-
raise "Impossible to load project: #{o[:project2]}" if q.nil?
|
33
|
-
|
34
|
-
$stderr.puts "Listing dataset." unless o[:q]
|
35
|
-
if o[:dataset].nil?
|
36
|
-
ds = p.datasets
|
37
|
-
else
|
38
|
-
ds = [p.dataset(o[:dataset])]
|
39
|
-
end
|
40
|
-
ds = filter_datasets!(ds, o)
|
41
|
-
ds.each do |d|
|
42
|
-
next unless o[:force] or d.done_preprocessing?
|
43
|
-
puts d.name
|
44
|
-
q.import_dataset(d, o[:method])
|
45
|
-
end
|
46
|
-
|
47
|
-
$stderr.puts "Done." unless o[:q]
|
data/actions/ls.rb
DELETED
@@ -1,61 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
# @package MiGA
|
4
|
-
# @license Artistic-2.0
|
5
|
-
|
6
|
-
o = {q:true, info:false, processing:false, silent:false, tabular:false}
|
7
|
-
OptionParser.new do |opt|
|
8
|
-
opt_banner(opt)
|
9
|
-
opt_object(opt, o, [:project, :dataset_opt])
|
10
|
-
opt_filter_datasets(opt, o)
|
11
|
-
opt.on("-i", "--info",
|
12
|
-
"Print additional information on each dataset."){ |v| o[:info]=v }
|
13
|
-
opt.on("-p", "--processing",
|
14
|
-
"Print information on processing advance."){ |v| o[:processing]=v }
|
15
|
-
opt.on("-m", "--metadata STRING",
|
16
|
-
"Print name and metadata field only. If set, ignores -i and assumes --tab."
|
17
|
-
){ |v| o[:datum]=v }
|
18
|
-
opt.on("--tab",
|
19
|
-
"Returns a tab-delimited table."){ |v| o[:tabular] = v }
|
20
|
-
opt.on("-s", "--silent",
|
21
|
-
"No output and exit with non-zero status if the dataset list is empty."
|
22
|
-
){ |v| o[:silent] = v }
|
23
|
-
opt_common(opt, o)
|
24
|
-
end.parse!
|
25
|
-
|
26
|
-
##=> Main <=
|
27
|
-
opt_require(o, project:"-P")
|
28
|
-
o[:q] = true if o[:silent]
|
29
|
-
|
30
|
-
$stderr.puts "Loading project." unless o[:q]
|
31
|
-
p = MiGA::Project.load(o[:project])
|
32
|
-
raise "Impossible to load project: #{o[:project]}" if p.nil?
|
33
|
-
|
34
|
-
$stderr.puts "Listing datasets." unless o[:q]
|
35
|
-
if o[:dataset].nil?
|
36
|
-
ds = p.datasets
|
37
|
-
elsif MiGA::Dataset.exist? p, o[:dataset]
|
38
|
-
ds = [p.dataset(o[:dataset])]
|
39
|
-
else
|
40
|
-
ds = []
|
41
|
-
end
|
42
|
-
ds = filter_datasets!(ds, o)
|
43
|
-
exit(1) if o[:silent] and ds.empty?
|
44
|
-
|
45
|
-
if not o[:datum].nil?
|
46
|
-
ds.each do |d|
|
47
|
-
v = d.metadata[ o[:datum] ]
|
48
|
-
puts "#{d.name}\t#{v.nil? ? '?' : v}"
|
49
|
-
end
|
50
|
-
elsif o[:info]
|
51
|
-
puts MiGA::MiGA.tabulate(
|
52
|
-
MiGA::Dataset.INFO_FIELDS, ds.map{ |d| d.info }, o[:tabular])
|
53
|
-
elsif o[:processing]
|
54
|
-
comp = ["-","done","queued"]
|
55
|
-
puts MiGA::MiGA.tabulate([:name] + MiGA::Dataset.PREPROCESSING_TASKS,
|
56
|
-
ds.map{ |d| [d.name] + d.profile_advance.map{ |i| comp[i] } }, o[:tabular])
|
57
|
-
else
|
58
|
-
ds.each{|d| puts d.name}
|
59
|
-
end
|
60
|
-
|
61
|
-
$stderr.puts "Done." unless o[:q]
|