miga-base 0.4.1.0 → 0.4.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/bin/miga +2 -244
  3. data/lib/miga/cli/action/about.rb +44 -0
  4. data/lib/miga/cli/action/add.rb +139 -0
  5. data/lib/miga/cli/action/add_result.rb +26 -0
  6. data/lib/miga/cli/action/console.rb +19 -0
  7. data/lib/miga/cli/action/daemon.rb +74 -0
  8. data/lib/miga/cli/action/date.rb +18 -0
  9. data/lib/miga/cli/action/doctor.rb +210 -0
  10. data/lib/miga/cli/action/edit.rb +24 -0
  11. data/lib/miga/cli/action/files.rb +31 -0
  12. data/lib/miga/cli/action/find.rb +48 -0
  13. data/lib/miga/cli/action/generic.rb +44 -0
  14. data/lib/miga/cli/action/get.rb +132 -0
  15. data/lib/miga/cli/action/init.rb +343 -0
  16. data/lib/miga/cli/action/ln.rb +42 -0
  17. data/lib/miga/cli/action/ls.rb +55 -0
  18. data/lib/miga/cli/action/ncbi_get.rb +218 -0
  19. data/lib/miga/cli/action/new.rb +45 -0
  20. data/lib/miga/cli/action/next_step.rb +27 -0
  21. data/lib/miga/cli/action/plugins.rb +28 -0
  22. data/lib/miga/cli/action/rm.rb +25 -0
  23. data/lib/miga/cli/action/run.rb +39 -0
  24. data/lib/miga/cli/action/stats.rb +140 -0
  25. data/lib/miga/cli/action/summary.rb +49 -0
  26. data/lib/miga/cli/action/tax_dist.rb +102 -0
  27. data/lib/miga/cli/action/tax_index.rb +47 -0
  28. data/lib/miga/cli/action/tax_set.rb +59 -0
  29. data/lib/miga/cli/action/tax_test.rb +77 -0
  30. data/lib/miga/cli/action.rb +66 -0
  31. data/lib/miga/cli/base.rb +90 -0
  32. data/lib/miga/cli.rb +426 -0
  33. data/lib/miga/project/result.rb +14 -6
  34. data/lib/miga/remote_dataset.rb +1 -1
  35. data/lib/miga/tax_index.rb +5 -4
  36. data/lib/miga/taxonomy/base.rb +63 -0
  37. data/lib/miga/taxonomy.rb +87 -92
  38. data/lib/miga/version.rb +6 -6
  39. data/test/taxonomy_test.rb +49 -9
  40. data/utils/distance/commands.rb +11 -11
  41. data/utils/distance/pipeline.rb +5 -5
  42. metadata +43 -49
  43. data/actions/about.rb +0 -43
  44. data/actions/add.rb +0 -129
  45. data/actions/add_result.rb +0 -30
  46. data/actions/daemon.rb +0 -55
  47. data/actions/date.rb +0 -14
  48. data/actions/doctor.rb +0 -201
  49. data/actions/edit.rb +0 -33
  50. data/actions/files.rb +0 -43
  51. data/actions/find.rb +0 -41
  52. data/actions/get.rb +0 -105
  53. data/actions/init.rb +0 -301
  54. data/actions/ln.rb +0 -47
  55. data/actions/ls.rb +0 -61
  56. data/actions/ncbi_get.rb +0 -192
  57. data/actions/new.rb +0 -44
  58. data/actions/next_step.rb +0 -33
  59. data/actions/plugins.rb +0 -25
  60. data/actions/rm.rb +0 -29
  61. data/actions/run.rb +0 -45
  62. data/actions/stats.rb +0 -149
  63. data/actions/summary.rb +0 -57
  64. data/actions/tax_dist.rb +0 -106
  65. data/actions/tax_index.rb +0 -46
  66. data/actions/tax_set.rb +0 -63
  67. data/actions/tax_test.rb +0 -80
data/actions/files.rb DELETED
@@ -1,43 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- # @package MiGA
4
- # @license Artistic-2.0
5
-
6
- o = {q:true, details:false, json:true}
7
- OptionParser.new do |opt|
8
- opt_banner(opt)
9
- opt_object(opt, o, [:project, :dataset_opt])
10
- opt.on("-i", "--info",
11
- "If set, it prints additional details for each file."
12
- ){ |v| o[:details]=v }
13
- opt.on("--[no-]json",
14
- "If set to no, excludes json files containing results metadata."
15
- ){ |v| o[:json]=v }
16
- opt_common(opt, o)
17
- end.parse!
18
-
19
- ##=> Main <=
20
- opt_require(o, project:"-P")
21
-
22
- $stderr.puts "Loading project." unless o[:q]
23
- p = MiGA::Project.load(o[:project])
24
- raise "Impossible to load project: #{o[:project]}" if p.nil?
25
-
26
- if o[:dataset].nil?
27
- results = p.results
28
- else
29
- $stderr.puts "Loading dataset." unless o[:q]
30
- ds = p.dataset(o[:dataset])
31
- raise "Impossible to load dataset: #{o[:dataset]}" if ds.nil?
32
- results = ds.results
33
- end
34
-
35
- $stderr.puts "Listing files." unless o[:q]
36
- results.each do |result|
37
- puts "#{ "#{result.path}\t\t" if o[:details] }#{result.path}" if o[:json]
38
- result.each_file do |k,f|
39
- puts "#{ "#{result.path}\t#{k}\t" if o[:details] }#{result.dir}/#{f}"
40
- end
41
- end
42
-
43
- $stderr.puts "Done." unless o[:q]
data/actions/find.rb DELETED
@@ -1,41 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- # @package MiGA
4
- # @license Artistic-2.0
5
-
6
- o = {q:true, add:false, ref:true}
7
- OptionParser.new do |opt|
8
- opt_banner(opt)
9
- opt_object(opt, o, [:project, :dataset_type])
10
- opt.on("-a", "--add",
11
- "Register the datasets found. By default, only lists them (dry run)."
12
- ){ |v| o[:add]=v }
13
- opt.on("-q", "--query",
14
- "If set, all datasets are registered as query datasets."
15
- ){ |v| o[:ref]=!v }
16
- opt.on("-u", "--user STRING", "Owner of the dataset."){ |v| o[:user]=v }
17
- opt_common(opt, o)
18
- end.parse!
19
-
20
- ##=> Main <=
21
- opt_require(o, project:"-P")
22
-
23
- $stderr.puts "Loading project." unless o[:q]
24
- p = MiGA::Project.load(o[:project])
25
- raise "Impossible to load project: #{o[:project]}" if p.nil?
26
-
27
- $stderr.puts "Finding datasets." unless o[:q]
28
- ud = p.unregistered_datasets
29
- ud.each do |dn|
30
- puts dn
31
- if o[:add]
32
- md = {}
33
- [:type, :user].each{ |k| md[k]=o[k] unless o[k].nil? }
34
- d = MiGA::Dataset.new(p, dn, o[:ref], md)
35
- p.add_dataset(dn)
36
- res = d.first_preprocessing(true)
37
- puts "- #{res}" unless o[:q]
38
- end
39
- end
40
-
41
- $stderr.puts "Done." unless o[:q]
data/actions/get.rb DELETED
@@ -1,105 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- # @package MiGA
4
- # @license Artistic-2.0
5
-
6
- require 'miga/remote_dataset'
7
-
8
- o = {q: true, query: false, universe: :ebi, db: :embl,
9
- get_md: false, only_md: false}
10
- OptionParser.new do |opt|
11
- opt_banner(opt)
12
- opt_object(opt, o, [:project, :dataset, :dataset_type])
13
- opt.on('-I', '--ids ID1,ID2,...',
14
- '(Mandatory unless -F) IDs in the remote database separated by commas.'
15
- ){ |v| o[:ids]=v }
16
- opt.on('-U', '--universe STRING',
17
- "Universe where the remote database lives. By default: #{o[:universe]}."
18
- ){ |v| o[:universe]=v.to_sym }
19
- opt.on('--db STRING',
20
- "Name of the remote database. By default: #{o[:db]}."
21
- ){ |v| o[:db]=v.to_sym }
22
- opt.on('-F', '--file PATH',
23
- 'Tab-delimited file (with header) listing the datasets to download.',
24
- 'The long form of all the options are supported as header (without the --)',
25
- 'including dataset, ids, universe, and db. For query use true/false values.'
26
- ){ |v| o[:file] = v }
27
- opt.on('-q', '--query',
28
- 'If set, the dataset is registered as a query, not a reference dataset.'
29
- ){ |v| o[:query]=v }
30
- opt.on('--ignore-dup',
31
- 'If set, ignores datasets that already exist.'){ |v| o[:ignore_dup]=v }
32
- opt.on('-d', '--description STRING',
33
- 'Description of the dataset.'){ |v| o[:description]=v }
34
- opt.on('-c', '--comments STRING',
35
- 'Comments on the dataset.'){ |v| o[:comments]=v }
36
- opt.on('-m', '--metadata STRING',
37
- 'Metadata as key-value pairs separated by = and delimited by comma.',
38
- 'Values are saved as strings except for booleans (true / false) or nil.'
39
- ){ |v| o[:metadata]=v }
40
- opt.on('--get-metadata',
41
- 'Only download and update metadata for existing datasets'
42
- ){ |v| o[:get_md] = v }
43
- opt.on('--only-metadata',
44
- 'Create datasets without input data but retrieve all metadata.'
45
- ){ |v| o[:only_md] = v }
46
- opt.on('--api-key STRING',
47
- 'API key for the given universe.'){ |v| o[:api_key] = v }
48
- opt_common(opt, o)
49
- end.parse!
50
-
51
- ##=> Main <=
52
- glob = [o]
53
- unless o[:file].nil?
54
- glob = []
55
- fh = File.open(o[:file], 'r')
56
- h = nil
57
- fh.each do |ln|
58
- r = ln.chomp.split(/\t/)
59
- if h.nil?
60
- h = r
61
- else
62
- glob << o.dup
63
- h.each_index do |i|
64
- glob[glob.size-1][h[i].to_sym] = h[i]=='query' ? r[i]=='true' :
65
- %w[type universe db].include?(h[i]) ? r[i].to_sym : r[i]
66
- end
67
- end
68
- end
69
- fh.close
70
- end
71
-
72
- glob.each do |o_i|
73
- opt_require(o_i, project: '-P', dataset: '-D', ids: '-I')
74
- unless o_i[:api_key].nil?
75
- ENV["#{o_i[:universe].to_s.upcase}_API_KEY"] = o_i[:api_key]
76
- end
77
-
78
- $stderr.puts "Dataset: #{o_i[:dataset]}" unless o_i[:q]
79
- $stderr.puts 'Loading project.' unless o_i[:q]
80
- p = MiGA::Project.load(o_i[:project])
81
- raise "Impossible to load project: #{o_i[:project]}" if p.nil?
82
-
83
- next if o_i[:ignore_dup] and not p.dataset(o_i[:dataset]).nil?
84
-
85
- $stderr.puts 'Locating remote dataset.' unless o_i[:q]
86
- rd = MiGA::RemoteDataset.new(o_i[:ids], o_i[:db], o_i[:universe])
87
-
88
- if o[:get_md]
89
- $stderr.puts 'Updating dataset.' unless o_i[:q]
90
- d = p.dataset(o_i[:dataset])
91
- next if d.nil?
92
- md = add_metadata(o_i, d).metadata.data
93
- rd.update_metadata(d, md)
94
- else
95
- $stderr.puts 'Creating dataset.' unless o_i[:q]
96
- dummy_d = MiGA::Dataset.new(p, o_i[:dataset])
97
- md = add_metadata(o_i, dummy_d).metadata.data
98
- md[:metadata_only] = true if o[:only_md]
99
- dummy_d.remove!
100
- rd.save_to(p, o_i[:dataset], !o_i[:query], md)
101
- p.add_dataset(o_i[:dataset])
102
- end
103
-
104
- $stderr.puts 'Done.' unless o_i[:q]
105
- end
data/actions/init.rb DELETED
@@ -1,301 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- # @package MiGA
4
- # @license Artistic-2.0
5
-
6
- require 'shellwords'
7
-
8
- o = {q: true, mytaxa: nil,
9
- config: File.expand_path('.miga_modules', ENV['HOME']),
10
- ask: false, auto: false, dtype: 'bash'}
11
- OptionParser.new do |opt|
12
- opt_banner(opt)
13
- opt.on('-c', '--config PATH',
14
- 'Path to the Bash configuration file.',
15
- "By default: #{o[:config]}."){ |v| o[:config] = v }
16
- opt.on('--[no-]mytaxa',
17
- 'Should I try setting up MyTaxa its dependencies?',
18
- 'By default: interactive (true if --auto).'
19
- ){ |v| o[:mytaxa] = v }
20
- opt.on('--daemon-type STRING',
21
- 'Type of daemon launcher, one of: bash, qsub, msub, slurm.',
22
- "By default: interactive (#{o[:dtype]} if --auto)."
23
- ){ |v| o[:dtype]=v }
24
- opt.on('--ask-all', 'If set, asks for the location of all software.',
25
- 'By default, only the locations missing in PATH are requested.'
26
- ){ |v| o[:ask] = v }
27
- opt.on('--auto', 'If set, accepts all defaults as answers.'
28
- ){ |v| o[:auto] = v }
29
- opt_common(opt, o)
30
- end.parse!
31
- $auto_answer = o[:auto]
32
-
33
- def ask_user(q, d = nil, ans = nil, force = false)
34
- $stderr.print "#{q}#{" (#{ans.join(" / ")})" unless ans.nil?}" +
35
- "#{" [#{d}]" unless d.nil?} > "
36
- if $auto_answer and not force
37
- $stderr.puts ''
38
- else
39
- o = gets.chomp
40
- end
41
- o = d if o.nil? or o.empty?
42
- unless ans.nil? or ans.include? o
43
- $stderr.puts 'Answer not recognized.'
44
- return ask_user(q, d, ans)
45
- end
46
- o
47
- end
48
-
49
- ##=> Main <=
50
-
51
- miga = MiGA::MiGA.root_path
52
- $stderr.puts <<BANNER
53
- ===[ Welcome to MiGA, the Microbial Genome Atlas ]===
54
-
55
- I'm the initialization script, and I'll sniff around your computer to
56
- make sure you have all the requirements for MiGA data processing.
57
-
58
- BANNER
59
-
60
- if ask_user(
61
- 'Would you like to see all the requirements before starting?',
62
- 'no', %w(yes no)) == 'yes'
63
- File.open(File.expand_path('utils/requirements.txt', miga), 'r') do |fh|
64
- fh.each_line{ |ln| $stderr.puts ln }
65
- end
66
- end
67
-
68
- rc_path = File.expand_path('.miga_rc', ENV['HOME'])
69
- if File.exist? rc_path
70
- if ask_user(
71
- 'I found a previous configuration. Do you want to continue?',
72
- 'yes', %w(yes no)) == 'no'
73
- $stderr.puts 'OK, see you soon!'
74
- exit 0
75
- end
76
- end
77
- rc_fh = File.open(rc_path, 'w')
78
- rc_fh.puts <<BASH
79
- #!/bin/bash
80
- # `miga init` made this on #{Time.now}
81
-
82
- BASH
83
-
84
- # Check bash configuration file
85
- unless File.exist? o[:config]
86
- o[:config] = ask_user(
87
- 'Is there a script I need to load at startup?', o[:config])
88
- end
89
- if File.exist? o[:config]
90
- o[:config] = File.expand_path o[:config]
91
- $stderr.puts "Found bash configuration script: #{o[:config]}."
92
- rc_fh.puts "MIGA_STARTUP='#{o[:config]}'"
93
- rc_fh.puts '. "$MIGA_STARTUP"'
94
- end
95
- $stderr.puts ''
96
-
97
- # Check for software requirements
98
- $stderr.puts 'Looking for requirements:'
99
- if o[:mytaxa].nil?
100
- o[:mytaxa] = ask_user(
101
- 'Should I include MyTaxa modules?', 'yes', %w(yes no)) == 'yes'
102
- end
103
- rc_fh.puts 'export MIGA_MYTAXA="no"' unless o[:mytaxa]
104
- paths = {}
105
- rc_fh.puts 'MIGA_PATH=""'
106
- File.open(File.expand_path('utils/requirements.txt', miga), 'r') do |fh|
107
- fh.each_line do |ln|
108
- next if $. < 3
109
- r = ln.chomp.split(/\t+/)
110
- next if r[0] =~ /\(opt\)$/ and not o[:mytaxa]
111
- $stderr.print "Testing #{r[0]}#{" (#{r[3]})" if r[3]}... "
112
- path = nil
113
- loop do
114
- if File.exist? o[:config]
115
- d_path = File.dirname(`. "#{o[:config]}" && which "#{r[1]}"`)
116
- else
117
- d_path = File.dirname(`which "#{r[1]}"`)
118
- end
119
- if o[:ask] or d_path == '.'
120
- path = ask_user('Where can I find it?', d_path, nil, true)
121
- else
122
- path = d_path
123
- $stderr.puts path
124
- end
125
- if File.executable? File.expand_path(r[1], path)
126
- if d_path != path
127
- rc_fh.puts "MIGA_PATH=\"#{path}:$MIGA_PATH\" # #{r[1]}"
128
- end
129
- break
130
- end
131
- $stderr.print "I cannot find #{r[1]}. "
132
- end
133
- paths[r[1]] = File.expand_path(r[1], path).shellescape
134
- end
135
- end
136
- rc_fh.puts 'export PATH="$MIGA_PATH$PATH"'
137
- $stderr.puts ''
138
-
139
- # Check for other files
140
- if o[:mytaxa]
141
- $stderr.puts 'Looking for MyTaxa databases:'
142
- mt = File.dirname paths["MyTaxa"]
143
- $stderr.print 'Looking for scores... '
144
- unless Dir.exist? File.expand_path('db', mt)
145
- $stderr.puts "no.\nExecute 'python2 #{mt}/utils/download_db.py'."
146
- exit 1
147
- end
148
- $stderr.puts 'yes.'
149
- $stderr.print 'Looking for diamond db... '
150
- unless File.exist? File.expand_path('AllGenomes.faa.dmnd', mt)
151
- $stderr.puts "no.\nDownload " +
152
- "'http://enve-omics.ce.gatech.edu/data/public_mytaxa/" +
153
- "AllGenomes.faa.dmnd' into #{mt}."
154
- exit 1
155
- end
156
- $stderr.puts ''
157
- end
158
-
159
- # Check for R packages
160
- $stderr.puts 'Looking for R packages:'
161
- %w(enveomics.R ape cluster vegan).each do |pkg|
162
- $stderr.print "Testing #{pkg}... "
163
- `echo "library('#{pkg}')" | #{paths["R"].shellescape} --vanilla -q 2>&1`
164
- if $?.success?
165
- $stderr.puts 'yes.'
166
- else
167
- $stderr.puts 'no, installing.'
168
- $stderr.print '' +
169
- `echo "install.packages('#{pkg}', repos='http://cran.rstudio.com/')" \
170
- | #{paths["R"].shellescape} --vanilla -q 2>&1`
171
- `echo "library('#{pkg}')" | #{paths["R"].shellescape} --vanilla -q 2>&1`
172
- raise "Unable to auto-install R package #{pkg}." unless $?.success?
173
- end
174
- end
175
- $stderr.puts ''
176
-
177
- # Check for Ruby gems
178
- $stderr.puts 'Looking for Ruby gems:'
179
- %w(sqlite3 daemons json).each do |pkg|
180
- $stderr.print "Testing #{pkg}... "
181
- `#{paths["ruby"].shellescape} -r "#{pkg}" -e "" 2>/dev/null`
182
- if $?.success?
183
- $stderr.puts 'yes.'
184
- else
185
- $stderr.puts 'no, installing.'
186
- # This hackey mess is meant to ensure the test and installation are done
187
- # on the configuration Ruby, not on the Ruby currently executing the init
188
- # action
189
- $stderr.print `#{paths['ruby'].shellescape} \
190
- -r rubygems -r rubygems/gem_runner \
191
- -e "Gem::GemRunner.new.run %w(install --user #{pkg})" 2>&1`
192
- raise "Unable to auto-install Ruby gem #{pkg}." unless $?.success?
193
- end
194
- end
195
- $stderr.puts ''
196
-
197
- # Configure daemon
198
- $stderr.puts 'Default daemon configuration:'
199
- daemon_f = File.expand_path('.miga_daemon.json', ENV['HOME'])
200
- unless File.exist?(daemon_f) and ask_user(
201
- 'A template daemon already exists, do you want to preserve it?',
202
- 'yes', %w(yes no)) == 'yes'
203
- v = {created:Time.now.to_s, updated:Time.now.to_s}
204
- v[:type] = ask_user('Please select the type of daemon you want to setup',
205
- o[:dtype], %w(bash qsub msub slurm))
206
- case v[:type]
207
- when 'bash'
208
- v[:latency] = ask_user('How long should I sleep? (in seconds)', '30').to_i
209
- v[:maxjobs] = ask_user('How many jobs can I launch at once?', '6').to_i
210
- v[:ppn] = ask_user('How many CPUs can I use per job?', '2').to_i
211
- $stderr.puts 'Setting up internal daemon defaults.'
212
- $stderr.puts 'If you don\'t understand this just leave default values:'
213
- v[:cmd] = ask_user(
214
- "How should I launch tasks?\n %1$s: script path, %2$s: variables, " +
215
- "%3$d: CPUs, %4$s: log file, %5$s: task name.\n",
216
- "%2$s '%1$s' > '%4$s' 2>&1")
217
- v[:var] = ask_user(
218
- "How should I pass variables?\n %1$s: keys, %2$s: values.\n",
219
- "%1$s=%2$s")
220
- v[:varsep] = ask_user('What should I use to separate variables?', ' ')
221
- v[:alive] = ask_user(
222
- "How can I know that a process is still alive?\n %1$s: PID, " +
223
- "output should be 1 for running and 0 for non-running.\n",
224
- "ps -p '%1$s'|tail -n+2|wc -l")
225
- v[:kill] = ask_user(
226
- "How should I terminate tasks?\n %s: process ID.", "kill -9 '%s'")
227
- when 'slurm'
228
- queue = ask_user('What queue should I use?', nil, nil, true)
229
- v[:latency] = ask_user(
230
- 'How long should I sleep? (in seconds)', '150').to_i
231
- v[:maxjobs] = ask_user('How many jobs can I launch at once?', '300').to_i
232
- v[:ppn] = ask_user('How many CPUs can I use per job?', '2').to_i
233
- $stderr.puts 'Setting up internal daemon defaults.'
234
- $stderr.puts 'If you don\'t understand this just leave default values:'
235
- v[:cmd] = ask_user(
236
- "How should I launch tasks?\n %1$s: script path, %2$s: variables, " +
237
- "%3$d: CPUs, %4$d: log file, %5$s: task name.\n",
238
- "%2$s sbatch --partition='#{queue}' --export=ALL " +
239
- "--nodes=1 --ntasks-per-node=%3$d --output='%4$s' --job-name='%5$s' " +
240
- "--mem=9G --time=12:00:00 %1$s | perl -pe 's/.* //'")
241
- v[:var] = ask_user(
242
- "How should I pass variables?\n %1$s: keys, %2$s: values.\n",
243
- "%1$s=%2$s")
244
- v[:varsep] = ask_user('What should I use to separate variables?', ' ')
245
- v[:alive] = ask_user(
246
- "How can I know that a process is still alive?\n %1$s: job id, " +
247
- "output should be 1 for running and 0 for non-running.\n",
248
- "squeue -h -o %%t -j '%1$s' | grep '^PD\\|R\\|CF\\|CG$' " +
249
- "| tail -n 1 | wc -l")
250
- v[:kill] = ask_user(
251
- "How should I terminate tasks?\n %s: process ID.", "scancel '%s'")
252
- else # [qm]sub
253
- queue = ask_user('What queue should I use?', nil, nil, true)
254
- v[:latency] = ask_user(
255
- 'How long should I sleep? (in seconds)', '150').to_i
256
- v[:maxjobs] = ask_user('How many jobs can I launch at once?', '300').to_i
257
- v[:ppn] = ask_user('How many CPUs can I use per job?', '2').to_i
258
- $stderr.puts 'Setting up internal daemon defaults.'
259
- $stderr.puts 'If you don\'t understand this just leave default values:'
260
- v[:cmd] = ask_user(
261
- "How should I launch tasks?\n %1$s: script path, %2$s: variables, " +
262
- "%3$d: CPUs, %4$d: log file, %5$s: task name.\n",
263
- "#{v[:type]} -q '#{queue}' -v '%2$s' -l nodes=1:ppn=%3$d %1$s " +
264
- "-j oe -o '%4$s' -N '%5$s' -l mem=9g -l walltime=12:00:00 | grep .")
265
- v[:var] = ask_user(
266
- "How should I pass variables?\n %1$s: keys, %2$s: values.\n",
267
- "%1$s=%2$s")
268
- v[:varsep] = ask_user('What should I use to separate variables?', ',')
269
- if v[:type] == 'qsub'
270
- v[:alive] = ask_user(
271
- "How can I know that a process is still alive?\n %1$s: job id, " +
272
- "output should be 1 for running and 0 for non-running.\n",
273
- "qstat -f '%1$s'|grep ' job_state ='|perl -pe 's/.*= //'|grep '[^C]'"+
274
- "|tail -n1|wc -l|awk '{print $1}'")
275
- v[:kill] = ask_user(
276
- "How should I terminate tasks?\n %s: process ID.", "qdel '%s'")
277
- else
278
- v[:alive] = ask_user(
279
- "How can I know that a process is still alive?\n %1$s: job id, " +
280
- "output should be 1 for running and 0 for non-running.\n",
281
- "checkjob '%1$s'|grep '^State:'|perl -pe 's/.*: //'" +
282
- "|grep 'Deferred\\|Hold\\|Idle\\|Starting\\|Running\\|Blocked'"+
283
- "|tail -n1|wc -l|awk '{print $1}'")
284
- v[:kill] = ask_user(
285
- "How should I terminate tasks?\n %s: process ID.", "canceljob '%s'")
286
- end
287
- end
288
- File.open(daemon_f, 'w'){ |fh| fh.puts JSON.pretty_generate(v) }
289
- end
290
- $stderr.puts ''
291
-
292
- rc_fh.puts <<FOOT
293
-
294
- MIGA_CONFIG_VERSION='#{MiGA::MiGA.VERSION}'
295
- MIGA_CONFIG_LONGVERSION='#{MiGA::MiGA.LONG_VERSION}'
296
- MIGA_CONFIG_DATE='#{Time.now}'
297
-
298
- FOOT
299
-
300
- $stderr.puts 'Configuration complete. MiGA is ready to work!'
301
- $stderr.puts ''
data/actions/ln.rb DELETED
@@ -1,47 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- # @package MiGA
4
- # @license Artistic-2.0
5
-
6
- o = {q:true, info:false, force:false, method: :hardlink }
7
- OptionParser.new do |opt|
8
- opt_banner(opt)
9
- opt_object(opt, o, [:project, :dataset_opt])
10
- opt.on("-Q", "--project-target PATH",
11
- "(Mandatory) Path to the project where to link the dataset."
12
- ){ |v| o[:project2]=v }
13
- opt.on("-f", "--force",
14
- "Forces linking, even if dataset's preprocessing is incomplete."
15
- ){ |v| o[:force]=v }
16
- opt.on("-s", "--symlink",
17
- "Creates symlinks instead of the default hard links."
18
- ){ o[:method] = :symlink }
19
- opt.on("-c", "--copy",
20
- "Creates copies instead of the default hard links."){ o[:method] = :copy }
21
- opt_filter_datasets(opt, o)
22
- opt_common(opt, o)
23
- end.parse!
24
-
25
- ##=> Main <=
26
- opt_require(o, project:"-P", project2:"-Q")
27
-
28
- $stderr.puts "Loading project." unless o[:q]
29
- p = MiGA::Project.load(o[:project])
30
- raise "Impossible to load project: #{o[:project]}" if p.nil?
31
- q = MiGA::Project.load(o[:project2])
32
- raise "Impossible to load project: #{o[:project2]}" if q.nil?
33
-
34
- $stderr.puts "Listing dataset." unless o[:q]
35
- if o[:dataset].nil?
36
- ds = p.datasets
37
- else
38
- ds = [p.dataset(o[:dataset])]
39
- end
40
- ds = filter_datasets!(ds, o)
41
- ds.each do |d|
42
- next unless o[:force] or d.done_preprocessing?
43
- puts d.name
44
- q.import_dataset(d, o[:method])
45
- end
46
-
47
- $stderr.puts "Done." unless o[:q]
data/actions/ls.rb DELETED
@@ -1,61 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- # @package MiGA
4
- # @license Artistic-2.0
5
-
6
- o = {q:true, info:false, processing:false, silent:false, tabular:false}
7
- OptionParser.new do |opt|
8
- opt_banner(opt)
9
- opt_object(opt, o, [:project, :dataset_opt])
10
- opt_filter_datasets(opt, o)
11
- opt.on("-i", "--info",
12
- "Print additional information on each dataset."){ |v| o[:info]=v }
13
- opt.on("-p", "--processing",
14
- "Print information on processing advance."){ |v| o[:processing]=v }
15
- opt.on("-m", "--metadata STRING",
16
- "Print name and metadata field only. If set, ignores -i and assumes --tab."
17
- ){ |v| o[:datum]=v }
18
- opt.on("--tab",
19
- "Returns a tab-delimited table."){ |v| o[:tabular] = v }
20
- opt.on("-s", "--silent",
21
- "No output and exit with non-zero status if the dataset list is empty."
22
- ){ |v| o[:silent] = v }
23
- opt_common(opt, o)
24
- end.parse!
25
-
26
- ##=> Main <=
27
- opt_require(o, project:"-P")
28
- o[:q] = true if o[:silent]
29
-
30
- $stderr.puts "Loading project." unless o[:q]
31
- p = MiGA::Project.load(o[:project])
32
- raise "Impossible to load project: #{o[:project]}" if p.nil?
33
-
34
- $stderr.puts "Listing datasets." unless o[:q]
35
- if o[:dataset].nil?
36
- ds = p.datasets
37
- elsif MiGA::Dataset.exist? p, o[:dataset]
38
- ds = [p.dataset(o[:dataset])]
39
- else
40
- ds = []
41
- end
42
- ds = filter_datasets!(ds, o)
43
- exit(1) if o[:silent] and ds.empty?
44
-
45
- if not o[:datum].nil?
46
- ds.each do |d|
47
- v = d.metadata[ o[:datum] ]
48
- puts "#{d.name}\t#{v.nil? ? '?' : v}"
49
- end
50
- elsif o[:info]
51
- puts MiGA::MiGA.tabulate(
52
- MiGA::Dataset.INFO_FIELDS, ds.map{ |d| d.info }, o[:tabular])
53
- elsif o[:processing]
54
- comp = ["-","done","queued"]
55
- puts MiGA::MiGA.tabulate([:name] + MiGA::Dataset.PREPROCESSING_TASKS,
56
- ds.map{ |d| [d.name] + d.profile_advance.map{ |i| comp[i] } }, o[:tabular])
57
- else
58
- ds.each{|d| puts d.name}
59
- end
60
-
61
- $stderr.puts "Done." unless o[:q]