miga-base 0.4.1.0 → 0.4.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/miga +2 -244
- data/lib/miga/cli/action/about.rb +44 -0
- data/lib/miga/cli/action/add.rb +139 -0
- data/lib/miga/cli/action/add_result.rb +26 -0
- data/lib/miga/cli/action/console.rb +19 -0
- data/lib/miga/cli/action/daemon.rb +74 -0
- data/lib/miga/cli/action/date.rb +18 -0
- data/lib/miga/cli/action/doctor.rb +210 -0
- data/lib/miga/cli/action/edit.rb +24 -0
- data/lib/miga/cli/action/files.rb +31 -0
- data/lib/miga/cli/action/find.rb +48 -0
- data/lib/miga/cli/action/generic.rb +44 -0
- data/lib/miga/cli/action/get.rb +132 -0
- data/lib/miga/cli/action/init.rb +343 -0
- data/lib/miga/cli/action/ln.rb +42 -0
- data/lib/miga/cli/action/ls.rb +55 -0
- data/lib/miga/cli/action/ncbi_get.rb +218 -0
- data/lib/miga/cli/action/new.rb +45 -0
- data/lib/miga/cli/action/next_step.rb +27 -0
- data/lib/miga/cli/action/plugins.rb +28 -0
- data/lib/miga/cli/action/rm.rb +25 -0
- data/lib/miga/cli/action/run.rb +39 -0
- data/lib/miga/cli/action/stats.rb +140 -0
- data/lib/miga/cli/action/summary.rb +49 -0
- data/lib/miga/cli/action/tax_dist.rb +102 -0
- data/lib/miga/cli/action/tax_index.rb +47 -0
- data/lib/miga/cli/action/tax_set.rb +59 -0
- data/lib/miga/cli/action/tax_test.rb +77 -0
- data/lib/miga/cli/action.rb +66 -0
- data/lib/miga/cli/base.rb +90 -0
- data/lib/miga/cli.rb +426 -0
- data/lib/miga/project/result.rb +14 -6
- data/lib/miga/remote_dataset.rb +1 -1
- data/lib/miga/tax_index.rb +5 -4
- data/lib/miga/taxonomy/base.rb +63 -0
- data/lib/miga/taxonomy.rb +87 -92
- data/lib/miga/version.rb +6 -6
- data/test/taxonomy_test.rb +49 -9
- data/utils/distance/commands.rb +11 -11
- data/utils/distance/pipeline.rb +5 -5
- metadata +43 -49
- data/actions/about.rb +0 -43
- data/actions/add.rb +0 -129
- data/actions/add_result.rb +0 -30
- data/actions/daemon.rb +0 -55
- data/actions/date.rb +0 -14
- data/actions/doctor.rb +0 -201
- data/actions/edit.rb +0 -33
- data/actions/files.rb +0 -43
- data/actions/find.rb +0 -41
- data/actions/get.rb +0 -105
- data/actions/init.rb +0 -301
- data/actions/ln.rb +0 -47
- data/actions/ls.rb +0 -61
- data/actions/ncbi_get.rb +0 -192
- data/actions/new.rb +0 -44
- data/actions/next_step.rb +0 -33
- data/actions/plugins.rb +0 -25
- data/actions/rm.rb +0 -29
- data/actions/run.rb +0 -45
- data/actions/stats.rb +0 -149
- data/actions/summary.rb +0 -57
- data/actions/tax_dist.rb +0 -106
- data/actions/tax_index.rb +0 -46
- data/actions/tax_set.rb +0 -63
- data/actions/tax_test.rb +0 -80
data/actions/files.rb
DELETED
@@ -1,43 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
# @package MiGA
|
4
|
-
# @license Artistic-2.0
|
5
|
-
|
6
|
-
o = {q:true, details:false, json:true}
|
7
|
-
OptionParser.new do |opt|
|
8
|
-
opt_banner(opt)
|
9
|
-
opt_object(opt, o, [:project, :dataset_opt])
|
10
|
-
opt.on("-i", "--info",
|
11
|
-
"If set, it prints additional details for each file."
|
12
|
-
){ |v| o[:details]=v }
|
13
|
-
opt.on("--[no-]json",
|
14
|
-
"If set to no, excludes json files containing results metadata."
|
15
|
-
){ |v| o[:json]=v }
|
16
|
-
opt_common(opt, o)
|
17
|
-
end.parse!
|
18
|
-
|
19
|
-
##=> Main <=
|
20
|
-
opt_require(o, project:"-P")
|
21
|
-
|
22
|
-
$stderr.puts "Loading project." unless o[:q]
|
23
|
-
p = MiGA::Project.load(o[:project])
|
24
|
-
raise "Impossible to load project: #{o[:project]}" if p.nil?
|
25
|
-
|
26
|
-
if o[:dataset].nil?
|
27
|
-
results = p.results
|
28
|
-
else
|
29
|
-
$stderr.puts "Loading dataset." unless o[:q]
|
30
|
-
ds = p.dataset(o[:dataset])
|
31
|
-
raise "Impossible to load dataset: #{o[:dataset]}" if ds.nil?
|
32
|
-
results = ds.results
|
33
|
-
end
|
34
|
-
|
35
|
-
$stderr.puts "Listing files." unless o[:q]
|
36
|
-
results.each do |result|
|
37
|
-
puts "#{ "#{result.path}\t\t" if o[:details] }#{result.path}" if o[:json]
|
38
|
-
result.each_file do |k,f|
|
39
|
-
puts "#{ "#{result.path}\t#{k}\t" if o[:details] }#{result.dir}/#{f}"
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
$stderr.puts "Done." unless o[:q]
|
data/actions/find.rb
DELETED
@@ -1,41 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
# @package MiGA
|
4
|
-
# @license Artistic-2.0
|
5
|
-
|
6
|
-
o = {q:true, add:false, ref:true}
|
7
|
-
OptionParser.new do |opt|
|
8
|
-
opt_banner(opt)
|
9
|
-
opt_object(opt, o, [:project, :dataset_type])
|
10
|
-
opt.on("-a", "--add",
|
11
|
-
"Register the datasets found. By default, only lists them (dry run)."
|
12
|
-
){ |v| o[:add]=v }
|
13
|
-
opt.on("-q", "--query",
|
14
|
-
"If set, all datasets are registered as query datasets."
|
15
|
-
){ |v| o[:ref]=!v }
|
16
|
-
opt.on("-u", "--user STRING", "Owner of the dataset."){ |v| o[:user]=v }
|
17
|
-
opt_common(opt, o)
|
18
|
-
end.parse!
|
19
|
-
|
20
|
-
##=> Main <=
|
21
|
-
opt_require(o, project:"-P")
|
22
|
-
|
23
|
-
$stderr.puts "Loading project." unless o[:q]
|
24
|
-
p = MiGA::Project.load(o[:project])
|
25
|
-
raise "Impossible to load project: #{o[:project]}" if p.nil?
|
26
|
-
|
27
|
-
$stderr.puts "Finding datasets." unless o[:q]
|
28
|
-
ud = p.unregistered_datasets
|
29
|
-
ud.each do |dn|
|
30
|
-
puts dn
|
31
|
-
if o[:add]
|
32
|
-
md = {}
|
33
|
-
[:type, :user].each{ |k| md[k]=o[k] unless o[k].nil? }
|
34
|
-
d = MiGA::Dataset.new(p, dn, o[:ref], md)
|
35
|
-
p.add_dataset(dn)
|
36
|
-
res = d.first_preprocessing(true)
|
37
|
-
puts "- #{res}" unless o[:q]
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
$stderr.puts "Done." unless o[:q]
|
data/actions/get.rb
DELETED
@@ -1,105 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
# @package MiGA
|
4
|
-
# @license Artistic-2.0
|
5
|
-
|
6
|
-
require 'miga/remote_dataset'
|
7
|
-
|
8
|
-
o = {q: true, query: false, universe: :ebi, db: :embl,
|
9
|
-
get_md: false, only_md: false}
|
10
|
-
OptionParser.new do |opt|
|
11
|
-
opt_banner(opt)
|
12
|
-
opt_object(opt, o, [:project, :dataset, :dataset_type])
|
13
|
-
opt.on('-I', '--ids ID1,ID2,...',
|
14
|
-
'(Mandatory unless -F) IDs in the remote database separated by commas.'
|
15
|
-
){ |v| o[:ids]=v }
|
16
|
-
opt.on('-U', '--universe STRING',
|
17
|
-
"Universe where the remote database lives. By default: #{o[:universe]}."
|
18
|
-
){ |v| o[:universe]=v.to_sym }
|
19
|
-
opt.on('--db STRING',
|
20
|
-
"Name of the remote database. By default: #{o[:db]}."
|
21
|
-
){ |v| o[:db]=v.to_sym }
|
22
|
-
opt.on('-F', '--file PATH',
|
23
|
-
'Tab-delimited file (with header) listing the datasets to download.',
|
24
|
-
'The long form of all the options are supported as header (without the --)',
|
25
|
-
'including dataset, ids, universe, and db. For query use true/false values.'
|
26
|
-
){ |v| o[:file] = v }
|
27
|
-
opt.on('-q', '--query',
|
28
|
-
'If set, the dataset is registered as a query, not a reference dataset.'
|
29
|
-
){ |v| o[:query]=v }
|
30
|
-
opt.on('--ignore-dup',
|
31
|
-
'If set, ignores datasets that already exist.'){ |v| o[:ignore_dup]=v }
|
32
|
-
opt.on('-d', '--description STRING',
|
33
|
-
'Description of the dataset.'){ |v| o[:description]=v }
|
34
|
-
opt.on('-c', '--comments STRING',
|
35
|
-
'Comments on the dataset.'){ |v| o[:comments]=v }
|
36
|
-
opt.on('-m', '--metadata STRING',
|
37
|
-
'Metadata as key-value pairs separated by = and delimited by comma.',
|
38
|
-
'Values are saved as strings except for booleans (true / false) or nil.'
|
39
|
-
){ |v| o[:metadata]=v }
|
40
|
-
opt.on('--get-metadata',
|
41
|
-
'Only download and update metadata for existing datasets'
|
42
|
-
){ |v| o[:get_md] = v }
|
43
|
-
opt.on('--only-metadata',
|
44
|
-
'Create datasets without input data but retrieve all metadata.'
|
45
|
-
){ |v| o[:only_md] = v }
|
46
|
-
opt.on('--api-key STRING',
|
47
|
-
'API key for the given universe.'){ |v| o[:api_key] = v }
|
48
|
-
opt_common(opt, o)
|
49
|
-
end.parse!
|
50
|
-
|
51
|
-
##=> Main <=
|
52
|
-
glob = [o]
|
53
|
-
unless o[:file].nil?
|
54
|
-
glob = []
|
55
|
-
fh = File.open(o[:file], 'r')
|
56
|
-
h = nil
|
57
|
-
fh.each do |ln|
|
58
|
-
r = ln.chomp.split(/\t/)
|
59
|
-
if h.nil?
|
60
|
-
h = r
|
61
|
-
else
|
62
|
-
glob << o.dup
|
63
|
-
h.each_index do |i|
|
64
|
-
glob[glob.size-1][h[i].to_sym] = h[i]=='query' ? r[i]=='true' :
|
65
|
-
%w[type universe db].include?(h[i]) ? r[i].to_sym : r[i]
|
66
|
-
end
|
67
|
-
end
|
68
|
-
end
|
69
|
-
fh.close
|
70
|
-
end
|
71
|
-
|
72
|
-
glob.each do |o_i|
|
73
|
-
opt_require(o_i, project: '-P', dataset: '-D', ids: '-I')
|
74
|
-
unless o_i[:api_key].nil?
|
75
|
-
ENV["#{o_i[:universe].to_s.upcase}_API_KEY"] = o_i[:api_key]
|
76
|
-
end
|
77
|
-
|
78
|
-
$stderr.puts "Dataset: #{o_i[:dataset]}" unless o_i[:q]
|
79
|
-
$stderr.puts 'Loading project.' unless o_i[:q]
|
80
|
-
p = MiGA::Project.load(o_i[:project])
|
81
|
-
raise "Impossible to load project: #{o_i[:project]}" if p.nil?
|
82
|
-
|
83
|
-
next if o_i[:ignore_dup] and not p.dataset(o_i[:dataset]).nil?
|
84
|
-
|
85
|
-
$stderr.puts 'Locating remote dataset.' unless o_i[:q]
|
86
|
-
rd = MiGA::RemoteDataset.new(o_i[:ids], o_i[:db], o_i[:universe])
|
87
|
-
|
88
|
-
if o[:get_md]
|
89
|
-
$stderr.puts 'Updating dataset.' unless o_i[:q]
|
90
|
-
d = p.dataset(o_i[:dataset])
|
91
|
-
next if d.nil?
|
92
|
-
md = add_metadata(o_i, d).metadata.data
|
93
|
-
rd.update_metadata(d, md)
|
94
|
-
else
|
95
|
-
$stderr.puts 'Creating dataset.' unless o_i[:q]
|
96
|
-
dummy_d = MiGA::Dataset.new(p, o_i[:dataset])
|
97
|
-
md = add_metadata(o_i, dummy_d).metadata.data
|
98
|
-
md[:metadata_only] = true if o[:only_md]
|
99
|
-
dummy_d.remove!
|
100
|
-
rd.save_to(p, o_i[:dataset], !o_i[:query], md)
|
101
|
-
p.add_dataset(o_i[:dataset])
|
102
|
-
end
|
103
|
-
|
104
|
-
$stderr.puts 'Done.' unless o_i[:q]
|
105
|
-
end
|
data/actions/init.rb
DELETED
@@ -1,301 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
# @package MiGA
|
4
|
-
# @license Artistic-2.0
|
5
|
-
|
6
|
-
require 'shellwords'
|
7
|
-
|
8
|
-
o = {q: true, mytaxa: nil,
|
9
|
-
config: File.expand_path('.miga_modules', ENV['HOME']),
|
10
|
-
ask: false, auto: false, dtype: 'bash'}
|
11
|
-
OptionParser.new do |opt|
|
12
|
-
opt_banner(opt)
|
13
|
-
opt.on('-c', '--config PATH',
|
14
|
-
'Path to the Bash configuration file.',
|
15
|
-
"By default: #{o[:config]}."){ |v| o[:config] = v }
|
16
|
-
opt.on('--[no-]mytaxa',
|
17
|
-
'Should I try setting up MyTaxa its dependencies?',
|
18
|
-
'By default: interactive (true if --auto).'
|
19
|
-
){ |v| o[:mytaxa] = v }
|
20
|
-
opt.on('--daemon-type STRING',
|
21
|
-
'Type of daemon launcher, one of: bash, qsub, msub, slurm.',
|
22
|
-
"By default: interactive (#{o[:dtype]} if --auto)."
|
23
|
-
){ |v| o[:dtype]=v }
|
24
|
-
opt.on('--ask-all', 'If set, asks for the location of all software.',
|
25
|
-
'By default, only the locations missing in PATH are requested.'
|
26
|
-
){ |v| o[:ask] = v }
|
27
|
-
opt.on('--auto', 'If set, accepts all defaults as answers.'
|
28
|
-
){ |v| o[:auto] = v }
|
29
|
-
opt_common(opt, o)
|
30
|
-
end.parse!
|
31
|
-
$auto_answer = o[:auto]
|
32
|
-
|
33
|
-
def ask_user(q, d = nil, ans = nil, force = false)
|
34
|
-
$stderr.print "#{q}#{" (#{ans.join(" / ")})" unless ans.nil?}" +
|
35
|
-
"#{" [#{d}]" unless d.nil?} > "
|
36
|
-
if $auto_answer and not force
|
37
|
-
$stderr.puts ''
|
38
|
-
else
|
39
|
-
o = gets.chomp
|
40
|
-
end
|
41
|
-
o = d if o.nil? or o.empty?
|
42
|
-
unless ans.nil? or ans.include? o
|
43
|
-
$stderr.puts 'Answer not recognized.'
|
44
|
-
return ask_user(q, d, ans)
|
45
|
-
end
|
46
|
-
o
|
47
|
-
end
|
48
|
-
|
49
|
-
##=> Main <=
|
50
|
-
|
51
|
-
miga = MiGA::MiGA.root_path
|
52
|
-
$stderr.puts <<BANNER
|
53
|
-
===[ Welcome to MiGA, the Microbial Genome Atlas ]===
|
54
|
-
|
55
|
-
I'm the initialization script, and I'll sniff around your computer to
|
56
|
-
make sure you have all the requirements for MiGA data processing.
|
57
|
-
|
58
|
-
BANNER
|
59
|
-
|
60
|
-
if ask_user(
|
61
|
-
'Would you like to see all the requirements before starting?',
|
62
|
-
'no', %w(yes no)) == 'yes'
|
63
|
-
File.open(File.expand_path('utils/requirements.txt', miga), 'r') do |fh|
|
64
|
-
fh.each_line{ |ln| $stderr.puts ln }
|
65
|
-
end
|
66
|
-
end
|
67
|
-
|
68
|
-
rc_path = File.expand_path('.miga_rc', ENV['HOME'])
|
69
|
-
if File.exist? rc_path
|
70
|
-
if ask_user(
|
71
|
-
'I found a previous configuration. Do you want to continue?',
|
72
|
-
'yes', %w(yes no)) == 'no'
|
73
|
-
$stderr.puts 'OK, see you soon!'
|
74
|
-
exit 0
|
75
|
-
end
|
76
|
-
end
|
77
|
-
rc_fh = File.open(rc_path, 'w')
|
78
|
-
rc_fh.puts <<BASH
|
79
|
-
#!/bin/bash
|
80
|
-
# `miga init` made this on #{Time.now}
|
81
|
-
|
82
|
-
BASH
|
83
|
-
|
84
|
-
# Check bash configuration file
|
85
|
-
unless File.exist? o[:config]
|
86
|
-
o[:config] = ask_user(
|
87
|
-
'Is there a script I need to load at startup?', o[:config])
|
88
|
-
end
|
89
|
-
if File.exist? o[:config]
|
90
|
-
o[:config] = File.expand_path o[:config]
|
91
|
-
$stderr.puts "Found bash configuration script: #{o[:config]}."
|
92
|
-
rc_fh.puts "MIGA_STARTUP='#{o[:config]}'"
|
93
|
-
rc_fh.puts '. "$MIGA_STARTUP"'
|
94
|
-
end
|
95
|
-
$stderr.puts ''
|
96
|
-
|
97
|
-
# Check for software requirements
|
98
|
-
$stderr.puts 'Looking for requirements:'
|
99
|
-
if o[:mytaxa].nil?
|
100
|
-
o[:mytaxa] = ask_user(
|
101
|
-
'Should I include MyTaxa modules?', 'yes', %w(yes no)) == 'yes'
|
102
|
-
end
|
103
|
-
rc_fh.puts 'export MIGA_MYTAXA="no"' unless o[:mytaxa]
|
104
|
-
paths = {}
|
105
|
-
rc_fh.puts 'MIGA_PATH=""'
|
106
|
-
File.open(File.expand_path('utils/requirements.txt', miga), 'r') do |fh|
|
107
|
-
fh.each_line do |ln|
|
108
|
-
next if $. < 3
|
109
|
-
r = ln.chomp.split(/\t+/)
|
110
|
-
next if r[0] =~ /\(opt\)$/ and not o[:mytaxa]
|
111
|
-
$stderr.print "Testing #{r[0]}#{" (#{r[3]})" if r[3]}... "
|
112
|
-
path = nil
|
113
|
-
loop do
|
114
|
-
if File.exist? o[:config]
|
115
|
-
d_path = File.dirname(`. "#{o[:config]}" && which "#{r[1]}"`)
|
116
|
-
else
|
117
|
-
d_path = File.dirname(`which "#{r[1]}"`)
|
118
|
-
end
|
119
|
-
if o[:ask] or d_path == '.'
|
120
|
-
path = ask_user('Where can I find it?', d_path, nil, true)
|
121
|
-
else
|
122
|
-
path = d_path
|
123
|
-
$stderr.puts path
|
124
|
-
end
|
125
|
-
if File.executable? File.expand_path(r[1], path)
|
126
|
-
if d_path != path
|
127
|
-
rc_fh.puts "MIGA_PATH=\"#{path}:$MIGA_PATH\" # #{r[1]}"
|
128
|
-
end
|
129
|
-
break
|
130
|
-
end
|
131
|
-
$stderr.print "I cannot find #{r[1]}. "
|
132
|
-
end
|
133
|
-
paths[r[1]] = File.expand_path(r[1], path).shellescape
|
134
|
-
end
|
135
|
-
end
|
136
|
-
rc_fh.puts 'export PATH="$MIGA_PATH$PATH"'
|
137
|
-
$stderr.puts ''
|
138
|
-
|
139
|
-
# Check for other files
|
140
|
-
if o[:mytaxa]
|
141
|
-
$stderr.puts 'Looking for MyTaxa databases:'
|
142
|
-
mt = File.dirname paths["MyTaxa"]
|
143
|
-
$stderr.print 'Looking for scores... '
|
144
|
-
unless Dir.exist? File.expand_path('db', mt)
|
145
|
-
$stderr.puts "no.\nExecute 'python2 #{mt}/utils/download_db.py'."
|
146
|
-
exit 1
|
147
|
-
end
|
148
|
-
$stderr.puts 'yes.'
|
149
|
-
$stderr.print 'Looking for diamond db... '
|
150
|
-
unless File.exist? File.expand_path('AllGenomes.faa.dmnd', mt)
|
151
|
-
$stderr.puts "no.\nDownload " +
|
152
|
-
"'http://enve-omics.ce.gatech.edu/data/public_mytaxa/" +
|
153
|
-
"AllGenomes.faa.dmnd' into #{mt}."
|
154
|
-
exit 1
|
155
|
-
end
|
156
|
-
$stderr.puts ''
|
157
|
-
end
|
158
|
-
|
159
|
-
# Check for R packages
|
160
|
-
$stderr.puts 'Looking for R packages:'
|
161
|
-
%w(enveomics.R ape cluster vegan).each do |pkg|
|
162
|
-
$stderr.print "Testing #{pkg}... "
|
163
|
-
`echo "library('#{pkg}')" | #{paths["R"].shellescape} --vanilla -q 2>&1`
|
164
|
-
if $?.success?
|
165
|
-
$stderr.puts 'yes.'
|
166
|
-
else
|
167
|
-
$stderr.puts 'no, installing.'
|
168
|
-
$stderr.print '' +
|
169
|
-
`echo "install.packages('#{pkg}', repos='http://cran.rstudio.com/')" \
|
170
|
-
| #{paths["R"].shellescape} --vanilla -q 2>&1`
|
171
|
-
`echo "library('#{pkg}')" | #{paths["R"].shellescape} --vanilla -q 2>&1`
|
172
|
-
raise "Unable to auto-install R package #{pkg}." unless $?.success?
|
173
|
-
end
|
174
|
-
end
|
175
|
-
$stderr.puts ''
|
176
|
-
|
177
|
-
# Check for Ruby gems
|
178
|
-
$stderr.puts 'Looking for Ruby gems:'
|
179
|
-
%w(sqlite3 daemons json).each do |pkg|
|
180
|
-
$stderr.print "Testing #{pkg}... "
|
181
|
-
`#{paths["ruby"].shellescape} -r "#{pkg}" -e "" 2>/dev/null`
|
182
|
-
if $?.success?
|
183
|
-
$stderr.puts 'yes.'
|
184
|
-
else
|
185
|
-
$stderr.puts 'no, installing.'
|
186
|
-
# This hackey mess is meant to ensure the test and installation are done
|
187
|
-
# on the configuration Ruby, not on the Ruby currently executing the init
|
188
|
-
# action
|
189
|
-
$stderr.print `#{paths['ruby'].shellescape} \
|
190
|
-
-r rubygems -r rubygems/gem_runner \
|
191
|
-
-e "Gem::GemRunner.new.run %w(install --user #{pkg})" 2>&1`
|
192
|
-
raise "Unable to auto-install Ruby gem #{pkg}." unless $?.success?
|
193
|
-
end
|
194
|
-
end
|
195
|
-
$stderr.puts ''
|
196
|
-
|
197
|
-
# Configure daemon
|
198
|
-
$stderr.puts 'Default daemon configuration:'
|
199
|
-
daemon_f = File.expand_path('.miga_daemon.json', ENV['HOME'])
|
200
|
-
unless File.exist?(daemon_f) and ask_user(
|
201
|
-
'A template daemon already exists, do you want to preserve it?',
|
202
|
-
'yes', %w(yes no)) == 'yes'
|
203
|
-
v = {created:Time.now.to_s, updated:Time.now.to_s}
|
204
|
-
v[:type] = ask_user('Please select the type of daemon you want to setup',
|
205
|
-
o[:dtype], %w(bash qsub msub slurm))
|
206
|
-
case v[:type]
|
207
|
-
when 'bash'
|
208
|
-
v[:latency] = ask_user('How long should I sleep? (in seconds)', '30').to_i
|
209
|
-
v[:maxjobs] = ask_user('How many jobs can I launch at once?', '6').to_i
|
210
|
-
v[:ppn] = ask_user('How many CPUs can I use per job?', '2').to_i
|
211
|
-
$stderr.puts 'Setting up internal daemon defaults.'
|
212
|
-
$stderr.puts 'If you don\'t understand this just leave default values:'
|
213
|
-
v[:cmd] = ask_user(
|
214
|
-
"How should I launch tasks?\n %1$s: script path, %2$s: variables, " +
|
215
|
-
"%3$d: CPUs, %4$s: log file, %5$s: task name.\n",
|
216
|
-
"%2$s '%1$s' > '%4$s' 2>&1")
|
217
|
-
v[:var] = ask_user(
|
218
|
-
"How should I pass variables?\n %1$s: keys, %2$s: values.\n",
|
219
|
-
"%1$s=%2$s")
|
220
|
-
v[:varsep] = ask_user('What should I use to separate variables?', ' ')
|
221
|
-
v[:alive] = ask_user(
|
222
|
-
"How can I know that a process is still alive?\n %1$s: PID, " +
|
223
|
-
"output should be 1 for running and 0 for non-running.\n",
|
224
|
-
"ps -p '%1$s'|tail -n+2|wc -l")
|
225
|
-
v[:kill] = ask_user(
|
226
|
-
"How should I terminate tasks?\n %s: process ID.", "kill -9 '%s'")
|
227
|
-
when 'slurm'
|
228
|
-
queue = ask_user('What queue should I use?', nil, nil, true)
|
229
|
-
v[:latency] = ask_user(
|
230
|
-
'How long should I sleep? (in seconds)', '150').to_i
|
231
|
-
v[:maxjobs] = ask_user('How many jobs can I launch at once?', '300').to_i
|
232
|
-
v[:ppn] = ask_user('How many CPUs can I use per job?', '2').to_i
|
233
|
-
$stderr.puts 'Setting up internal daemon defaults.'
|
234
|
-
$stderr.puts 'If you don\'t understand this just leave default values:'
|
235
|
-
v[:cmd] = ask_user(
|
236
|
-
"How should I launch tasks?\n %1$s: script path, %2$s: variables, " +
|
237
|
-
"%3$d: CPUs, %4$d: log file, %5$s: task name.\n",
|
238
|
-
"%2$s sbatch --partition='#{queue}' --export=ALL " +
|
239
|
-
"--nodes=1 --ntasks-per-node=%3$d --output='%4$s' --job-name='%5$s' " +
|
240
|
-
"--mem=9G --time=12:00:00 %1$s | perl -pe 's/.* //'")
|
241
|
-
v[:var] = ask_user(
|
242
|
-
"How should I pass variables?\n %1$s: keys, %2$s: values.\n",
|
243
|
-
"%1$s=%2$s")
|
244
|
-
v[:varsep] = ask_user('What should I use to separate variables?', ' ')
|
245
|
-
v[:alive] = ask_user(
|
246
|
-
"How can I know that a process is still alive?\n %1$s: job id, " +
|
247
|
-
"output should be 1 for running and 0 for non-running.\n",
|
248
|
-
"squeue -h -o %%t -j '%1$s' | grep '^PD\\|R\\|CF\\|CG$' " +
|
249
|
-
"| tail -n 1 | wc -l")
|
250
|
-
v[:kill] = ask_user(
|
251
|
-
"How should I terminate tasks?\n %s: process ID.", "scancel '%s'")
|
252
|
-
else # [qm]sub
|
253
|
-
queue = ask_user('What queue should I use?', nil, nil, true)
|
254
|
-
v[:latency] = ask_user(
|
255
|
-
'How long should I sleep? (in seconds)', '150').to_i
|
256
|
-
v[:maxjobs] = ask_user('How many jobs can I launch at once?', '300').to_i
|
257
|
-
v[:ppn] = ask_user('How many CPUs can I use per job?', '2').to_i
|
258
|
-
$stderr.puts 'Setting up internal daemon defaults.'
|
259
|
-
$stderr.puts 'If you don\'t understand this just leave default values:'
|
260
|
-
v[:cmd] = ask_user(
|
261
|
-
"How should I launch tasks?\n %1$s: script path, %2$s: variables, " +
|
262
|
-
"%3$d: CPUs, %4$d: log file, %5$s: task name.\n",
|
263
|
-
"#{v[:type]} -q '#{queue}' -v '%2$s' -l nodes=1:ppn=%3$d %1$s " +
|
264
|
-
"-j oe -o '%4$s' -N '%5$s' -l mem=9g -l walltime=12:00:00 | grep .")
|
265
|
-
v[:var] = ask_user(
|
266
|
-
"How should I pass variables?\n %1$s: keys, %2$s: values.\n",
|
267
|
-
"%1$s=%2$s")
|
268
|
-
v[:varsep] = ask_user('What should I use to separate variables?', ',')
|
269
|
-
if v[:type] == 'qsub'
|
270
|
-
v[:alive] = ask_user(
|
271
|
-
"How can I know that a process is still alive?\n %1$s: job id, " +
|
272
|
-
"output should be 1 for running and 0 for non-running.\n",
|
273
|
-
"qstat -f '%1$s'|grep ' job_state ='|perl -pe 's/.*= //'|grep '[^C]'"+
|
274
|
-
"|tail -n1|wc -l|awk '{print $1}'")
|
275
|
-
v[:kill] = ask_user(
|
276
|
-
"How should I terminate tasks?\n %s: process ID.", "qdel '%s'")
|
277
|
-
else
|
278
|
-
v[:alive] = ask_user(
|
279
|
-
"How can I know that a process is still alive?\n %1$s: job id, " +
|
280
|
-
"output should be 1 for running and 0 for non-running.\n",
|
281
|
-
"checkjob '%1$s'|grep '^State:'|perl -pe 's/.*: //'" +
|
282
|
-
"|grep 'Deferred\\|Hold\\|Idle\\|Starting\\|Running\\|Blocked'"+
|
283
|
-
"|tail -n1|wc -l|awk '{print $1}'")
|
284
|
-
v[:kill] = ask_user(
|
285
|
-
"How should I terminate tasks?\n %s: process ID.", "canceljob '%s'")
|
286
|
-
end
|
287
|
-
end
|
288
|
-
File.open(daemon_f, 'w'){ |fh| fh.puts JSON.pretty_generate(v) }
|
289
|
-
end
|
290
|
-
$stderr.puts ''
|
291
|
-
|
292
|
-
rc_fh.puts <<FOOT
|
293
|
-
|
294
|
-
MIGA_CONFIG_VERSION='#{MiGA::MiGA.VERSION}'
|
295
|
-
MIGA_CONFIG_LONGVERSION='#{MiGA::MiGA.LONG_VERSION}'
|
296
|
-
MIGA_CONFIG_DATE='#{Time.now}'
|
297
|
-
|
298
|
-
FOOT
|
299
|
-
|
300
|
-
$stderr.puts 'Configuration complete. MiGA is ready to work!'
|
301
|
-
$stderr.puts ''
|
data/actions/ln.rb
DELETED
@@ -1,47 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
# @package MiGA
|
4
|
-
# @license Artistic-2.0
|
5
|
-
|
6
|
-
o = {q:true, info:false, force:false, method: :hardlink }
|
7
|
-
OptionParser.new do |opt|
|
8
|
-
opt_banner(opt)
|
9
|
-
opt_object(opt, o, [:project, :dataset_opt])
|
10
|
-
opt.on("-Q", "--project-target PATH",
|
11
|
-
"(Mandatory) Path to the project where to link the dataset."
|
12
|
-
){ |v| o[:project2]=v }
|
13
|
-
opt.on("-f", "--force",
|
14
|
-
"Forces linking, even if dataset's preprocessing is incomplete."
|
15
|
-
){ |v| o[:force]=v }
|
16
|
-
opt.on("-s", "--symlink",
|
17
|
-
"Creates symlinks instead of the default hard links."
|
18
|
-
){ o[:method] = :symlink }
|
19
|
-
opt.on("-c", "--copy",
|
20
|
-
"Creates copies instead of the default hard links."){ o[:method] = :copy }
|
21
|
-
opt_filter_datasets(opt, o)
|
22
|
-
opt_common(opt, o)
|
23
|
-
end.parse!
|
24
|
-
|
25
|
-
##=> Main <=
|
26
|
-
opt_require(o, project:"-P", project2:"-Q")
|
27
|
-
|
28
|
-
$stderr.puts "Loading project." unless o[:q]
|
29
|
-
p = MiGA::Project.load(o[:project])
|
30
|
-
raise "Impossible to load project: #{o[:project]}" if p.nil?
|
31
|
-
q = MiGA::Project.load(o[:project2])
|
32
|
-
raise "Impossible to load project: #{o[:project2]}" if q.nil?
|
33
|
-
|
34
|
-
$stderr.puts "Listing dataset." unless o[:q]
|
35
|
-
if o[:dataset].nil?
|
36
|
-
ds = p.datasets
|
37
|
-
else
|
38
|
-
ds = [p.dataset(o[:dataset])]
|
39
|
-
end
|
40
|
-
ds = filter_datasets!(ds, o)
|
41
|
-
ds.each do |d|
|
42
|
-
next unless o[:force] or d.done_preprocessing?
|
43
|
-
puts d.name
|
44
|
-
q.import_dataset(d, o[:method])
|
45
|
-
end
|
46
|
-
|
47
|
-
$stderr.puts "Done." unless o[:q]
|
data/actions/ls.rb
DELETED
@@ -1,61 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
# @package MiGA
|
4
|
-
# @license Artistic-2.0
|
5
|
-
|
6
|
-
o = {q:true, info:false, processing:false, silent:false, tabular:false}
|
7
|
-
OptionParser.new do |opt|
|
8
|
-
opt_banner(opt)
|
9
|
-
opt_object(opt, o, [:project, :dataset_opt])
|
10
|
-
opt_filter_datasets(opt, o)
|
11
|
-
opt.on("-i", "--info",
|
12
|
-
"Print additional information on each dataset."){ |v| o[:info]=v }
|
13
|
-
opt.on("-p", "--processing",
|
14
|
-
"Print information on processing advance."){ |v| o[:processing]=v }
|
15
|
-
opt.on("-m", "--metadata STRING",
|
16
|
-
"Print name and metadata field only. If set, ignores -i and assumes --tab."
|
17
|
-
){ |v| o[:datum]=v }
|
18
|
-
opt.on("--tab",
|
19
|
-
"Returns a tab-delimited table."){ |v| o[:tabular] = v }
|
20
|
-
opt.on("-s", "--silent",
|
21
|
-
"No output and exit with non-zero status if the dataset list is empty."
|
22
|
-
){ |v| o[:silent] = v }
|
23
|
-
opt_common(opt, o)
|
24
|
-
end.parse!
|
25
|
-
|
26
|
-
##=> Main <=
|
27
|
-
opt_require(o, project:"-P")
|
28
|
-
o[:q] = true if o[:silent]
|
29
|
-
|
30
|
-
$stderr.puts "Loading project." unless o[:q]
|
31
|
-
p = MiGA::Project.load(o[:project])
|
32
|
-
raise "Impossible to load project: #{o[:project]}" if p.nil?
|
33
|
-
|
34
|
-
$stderr.puts "Listing datasets." unless o[:q]
|
35
|
-
if o[:dataset].nil?
|
36
|
-
ds = p.datasets
|
37
|
-
elsif MiGA::Dataset.exist? p, o[:dataset]
|
38
|
-
ds = [p.dataset(o[:dataset])]
|
39
|
-
else
|
40
|
-
ds = []
|
41
|
-
end
|
42
|
-
ds = filter_datasets!(ds, o)
|
43
|
-
exit(1) if o[:silent] and ds.empty?
|
44
|
-
|
45
|
-
if not o[:datum].nil?
|
46
|
-
ds.each do |d|
|
47
|
-
v = d.metadata[ o[:datum] ]
|
48
|
-
puts "#{d.name}\t#{v.nil? ? '?' : v}"
|
49
|
-
end
|
50
|
-
elsif o[:info]
|
51
|
-
puts MiGA::MiGA.tabulate(
|
52
|
-
MiGA::Dataset.INFO_FIELDS, ds.map{ |d| d.info }, o[:tabular])
|
53
|
-
elsif o[:processing]
|
54
|
-
comp = ["-","done","queued"]
|
55
|
-
puts MiGA::MiGA.tabulate([:name] + MiGA::Dataset.PREPROCESSING_TASKS,
|
56
|
-
ds.map{ |d| [d.name] + d.profile_advance.map{ |i| comp[i] } }, o[:tabular])
|
57
|
-
else
|
58
|
-
ds.each{|d| puts d.name}
|
59
|
-
end
|
60
|
-
|
61
|
-
$stderr.puts "Done." unless o[:q]
|