miga-base 0.4.1.0 → 0.4.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/bin/miga +2 -244
  3. data/lib/miga/cli/action/about.rb +44 -0
  4. data/lib/miga/cli/action/add.rb +139 -0
  5. data/lib/miga/cli/action/add_result.rb +26 -0
  6. data/lib/miga/cli/action/console.rb +19 -0
  7. data/lib/miga/cli/action/daemon.rb +74 -0
  8. data/lib/miga/cli/action/date.rb +18 -0
  9. data/lib/miga/cli/action/doctor.rb +210 -0
  10. data/lib/miga/cli/action/edit.rb +24 -0
  11. data/lib/miga/cli/action/files.rb +31 -0
  12. data/lib/miga/cli/action/find.rb +48 -0
  13. data/lib/miga/cli/action/generic.rb +44 -0
  14. data/lib/miga/cli/action/get.rb +132 -0
  15. data/lib/miga/cli/action/init.rb +343 -0
  16. data/lib/miga/cli/action/ln.rb +42 -0
  17. data/lib/miga/cli/action/ls.rb +55 -0
  18. data/lib/miga/cli/action/ncbi_get.rb +218 -0
  19. data/lib/miga/cli/action/new.rb +45 -0
  20. data/lib/miga/cli/action/next_step.rb +27 -0
  21. data/lib/miga/cli/action/plugins.rb +28 -0
  22. data/lib/miga/cli/action/rm.rb +25 -0
  23. data/lib/miga/cli/action/run.rb +39 -0
  24. data/lib/miga/cli/action/stats.rb +140 -0
  25. data/lib/miga/cli/action/summary.rb +49 -0
  26. data/lib/miga/cli/action/tax_dist.rb +102 -0
  27. data/lib/miga/cli/action/tax_index.rb +47 -0
  28. data/lib/miga/cli/action/tax_set.rb +59 -0
  29. data/lib/miga/cli/action/tax_test.rb +77 -0
  30. data/lib/miga/cli/action.rb +66 -0
  31. data/lib/miga/cli/base.rb +90 -0
  32. data/lib/miga/cli.rb +426 -0
  33. data/lib/miga/project/result.rb +14 -6
  34. data/lib/miga/remote_dataset.rb +1 -1
  35. data/lib/miga/tax_index.rb +5 -4
  36. data/lib/miga/taxonomy/base.rb +63 -0
  37. data/lib/miga/taxonomy.rb +87 -92
  38. data/lib/miga/version.rb +6 -6
  39. data/test/taxonomy_test.rb +49 -9
  40. data/utils/distance/commands.rb +11 -11
  41. data/utils/distance/pipeline.rb +5 -5
  42. metadata +43 -49
  43. data/actions/about.rb +0 -43
  44. data/actions/add.rb +0 -129
  45. data/actions/add_result.rb +0 -30
  46. data/actions/daemon.rb +0 -55
  47. data/actions/date.rb +0 -14
  48. data/actions/doctor.rb +0 -201
  49. data/actions/edit.rb +0 -33
  50. data/actions/files.rb +0 -43
  51. data/actions/find.rb +0 -41
  52. data/actions/get.rb +0 -105
  53. data/actions/init.rb +0 -301
  54. data/actions/ln.rb +0 -47
  55. data/actions/ls.rb +0 -61
  56. data/actions/ncbi_get.rb +0 -192
  57. data/actions/new.rb +0 -44
  58. data/actions/next_step.rb +0 -33
  59. data/actions/plugins.rb +0 -25
  60. data/actions/rm.rb +0 -29
  61. data/actions/run.rb +0 -45
  62. data/actions/stats.rb +0 -149
  63. data/actions/summary.rb +0 -57
  64. data/actions/tax_dist.rb +0 -106
  65. data/actions/tax_index.rb +0 -46
  66. data/actions/tax_set.rb +0 -63
  67. data/actions/tax_test.rb +0 -80
data/actions/files.rb DELETED
@@ -1,43 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- # @package MiGA
4
- # @license Artistic-2.0
5
-
6
- o = {q:true, details:false, json:true}
7
- OptionParser.new do |opt|
8
- opt_banner(opt)
9
- opt_object(opt, o, [:project, :dataset_opt])
10
- opt.on("-i", "--info",
11
- "If set, it prints additional details for each file."
12
- ){ |v| o[:details]=v }
13
- opt.on("--[no-]json",
14
- "If set to no, excludes json files containing results metadata."
15
- ){ |v| o[:json]=v }
16
- opt_common(opt, o)
17
- end.parse!
18
-
19
- ##=> Main <=
20
- opt_require(o, project:"-P")
21
-
22
- $stderr.puts "Loading project." unless o[:q]
23
- p = MiGA::Project.load(o[:project])
24
- raise "Impossible to load project: #{o[:project]}" if p.nil?
25
-
26
- if o[:dataset].nil?
27
- results = p.results
28
- else
29
- $stderr.puts "Loading dataset." unless o[:q]
30
- ds = p.dataset(o[:dataset])
31
- raise "Impossible to load dataset: #{o[:dataset]}" if ds.nil?
32
- results = ds.results
33
- end
34
-
35
- $stderr.puts "Listing files." unless o[:q]
36
- results.each do |result|
37
- puts "#{ "#{result.path}\t\t" if o[:details] }#{result.path}" if o[:json]
38
- result.each_file do |k,f|
39
- puts "#{ "#{result.path}\t#{k}\t" if o[:details] }#{result.dir}/#{f}"
40
- end
41
- end
42
-
43
- $stderr.puts "Done." unless o[:q]
data/actions/find.rb DELETED
@@ -1,41 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- # @package MiGA
4
- # @license Artistic-2.0
5
-
6
- o = {q:true, add:false, ref:true}
7
- OptionParser.new do |opt|
8
- opt_banner(opt)
9
- opt_object(opt, o, [:project, :dataset_type])
10
- opt.on("-a", "--add",
11
- "Register the datasets found. By default, only lists them (dry run)."
12
- ){ |v| o[:add]=v }
13
- opt.on("-q", "--query",
14
- "If set, all datasets are registered as query datasets."
15
- ){ |v| o[:ref]=!v }
16
- opt.on("-u", "--user STRING", "Owner of the dataset."){ |v| o[:user]=v }
17
- opt_common(opt, o)
18
- end.parse!
19
-
20
- ##=> Main <=
21
- opt_require(o, project:"-P")
22
-
23
- $stderr.puts "Loading project." unless o[:q]
24
- p = MiGA::Project.load(o[:project])
25
- raise "Impossible to load project: #{o[:project]}" if p.nil?
26
-
27
- $stderr.puts "Finding datasets." unless o[:q]
28
- ud = p.unregistered_datasets
29
- ud.each do |dn|
30
- puts dn
31
- if o[:add]
32
- md = {}
33
- [:type, :user].each{ |k| md[k]=o[k] unless o[k].nil? }
34
- d = MiGA::Dataset.new(p, dn, o[:ref], md)
35
- p.add_dataset(dn)
36
- res = d.first_preprocessing(true)
37
- puts "- #{res}" unless o[:q]
38
- end
39
- end
40
-
41
- $stderr.puts "Done." unless o[:q]
data/actions/get.rb DELETED
@@ -1,105 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- # @package MiGA
4
- # @license Artistic-2.0
5
-
6
- require 'miga/remote_dataset'
7
-
8
- o = {q: true, query: false, universe: :ebi, db: :embl,
9
- get_md: false, only_md: false}
10
- OptionParser.new do |opt|
11
- opt_banner(opt)
12
- opt_object(opt, o, [:project, :dataset, :dataset_type])
13
- opt.on('-I', '--ids ID1,ID2,...',
14
- '(Mandatory unless -F) IDs in the remote database separated by commas.'
15
- ){ |v| o[:ids]=v }
16
- opt.on('-U', '--universe STRING',
17
- "Universe where the remote database lives. By default: #{o[:universe]}."
18
- ){ |v| o[:universe]=v.to_sym }
19
- opt.on('--db STRING',
20
- "Name of the remote database. By default: #{o[:db]}."
21
- ){ |v| o[:db]=v.to_sym }
22
- opt.on('-F', '--file PATH',
23
- 'Tab-delimited file (with header) listing the datasets to download.',
24
- 'The long form of all the options are supported as header (without the --)',
25
- 'including dataset, ids, universe, and db. For query use true/false values.'
26
- ){ |v| o[:file] = v }
27
- opt.on('-q', '--query',
28
- 'If set, the dataset is registered as a query, not a reference dataset.'
29
- ){ |v| o[:query]=v }
30
- opt.on('--ignore-dup',
31
- 'If set, ignores datasets that already exist.'){ |v| o[:ignore_dup]=v }
32
- opt.on('-d', '--description STRING',
33
- 'Description of the dataset.'){ |v| o[:description]=v }
34
- opt.on('-c', '--comments STRING',
35
- 'Comments on the dataset.'){ |v| o[:comments]=v }
36
- opt.on('-m', '--metadata STRING',
37
- 'Metadata as key-value pairs separated by = and delimited by comma.',
38
- 'Values are saved as strings except for booleans (true / false) or nil.'
39
- ){ |v| o[:metadata]=v }
40
- opt.on('--get-metadata',
41
- 'Only download and update metadata for existing datasets'
42
- ){ |v| o[:get_md] = v }
43
- opt.on('--only-metadata',
44
- 'Create datasets without input data but retrieve all metadata.'
45
- ){ |v| o[:only_md] = v }
46
- opt.on('--api-key STRING',
47
- 'API key for the given universe.'){ |v| o[:api_key] = v }
48
- opt_common(opt, o)
49
- end.parse!
50
-
51
- ##=> Main <=
52
- glob = [o]
53
- unless o[:file].nil?
54
- glob = []
55
- fh = File.open(o[:file], 'r')
56
- h = nil
57
- fh.each do |ln|
58
- r = ln.chomp.split(/\t/)
59
- if h.nil?
60
- h = r
61
- else
62
- glob << o.dup
63
- h.each_index do |i|
64
- glob[glob.size-1][h[i].to_sym] = h[i]=='query' ? r[i]=='true' :
65
- %w[type universe db].include?(h[i]) ? r[i].to_sym : r[i]
66
- end
67
- end
68
- end
69
- fh.close
70
- end
71
-
72
- glob.each do |o_i|
73
- opt_require(o_i, project: '-P', dataset: '-D', ids: '-I')
74
- unless o_i[:api_key].nil?
75
- ENV["#{o_i[:universe].to_s.upcase}_API_KEY"] = o_i[:api_key]
76
- end
77
-
78
- $stderr.puts "Dataset: #{o_i[:dataset]}" unless o_i[:q]
79
- $stderr.puts 'Loading project.' unless o_i[:q]
80
- p = MiGA::Project.load(o_i[:project])
81
- raise "Impossible to load project: #{o_i[:project]}" if p.nil?
82
-
83
- next if o_i[:ignore_dup] and not p.dataset(o_i[:dataset]).nil?
84
-
85
- $stderr.puts 'Locating remote dataset.' unless o_i[:q]
86
- rd = MiGA::RemoteDataset.new(o_i[:ids], o_i[:db], o_i[:universe])
87
-
88
- if o[:get_md]
89
- $stderr.puts 'Updating dataset.' unless o_i[:q]
90
- d = p.dataset(o_i[:dataset])
91
- next if d.nil?
92
- md = add_metadata(o_i, d).metadata.data
93
- rd.update_metadata(d, md)
94
- else
95
- $stderr.puts 'Creating dataset.' unless o_i[:q]
96
- dummy_d = MiGA::Dataset.new(p, o_i[:dataset])
97
- md = add_metadata(o_i, dummy_d).metadata.data
98
- md[:metadata_only] = true if o[:only_md]
99
- dummy_d.remove!
100
- rd.save_to(p, o_i[:dataset], !o_i[:query], md)
101
- p.add_dataset(o_i[:dataset])
102
- end
103
-
104
- $stderr.puts 'Done.' unless o_i[:q]
105
- end
data/actions/init.rb DELETED
@@ -1,301 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- # @package MiGA
4
- # @license Artistic-2.0
5
-
6
- require 'shellwords'
7
-
8
- o = {q: true, mytaxa: nil,
9
- config: File.expand_path('.miga_modules', ENV['HOME']),
10
- ask: false, auto: false, dtype: 'bash'}
11
- OptionParser.new do |opt|
12
- opt_banner(opt)
13
- opt.on('-c', '--config PATH',
14
- 'Path to the Bash configuration file.',
15
- "By default: #{o[:config]}."){ |v| o[:config] = v }
16
- opt.on('--[no-]mytaxa',
17
- 'Should I try setting up MyTaxa its dependencies?',
18
- 'By default: interactive (true if --auto).'
19
- ){ |v| o[:mytaxa] = v }
20
- opt.on('--daemon-type STRING',
21
- 'Type of daemon launcher, one of: bash, qsub, msub, slurm.',
22
- "By default: interactive (#{o[:dtype]} if --auto)."
23
- ){ |v| o[:dtype]=v }
24
- opt.on('--ask-all', 'If set, asks for the location of all software.',
25
- 'By default, only the locations missing in PATH are requested.'
26
- ){ |v| o[:ask] = v }
27
- opt.on('--auto', 'If set, accepts all defaults as answers.'
28
- ){ |v| o[:auto] = v }
29
- opt_common(opt, o)
30
- end.parse!
31
- $auto_answer = o[:auto]
32
-
33
- def ask_user(q, d = nil, ans = nil, force = false)
34
- $stderr.print "#{q}#{" (#{ans.join(" / ")})" unless ans.nil?}" +
35
- "#{" [#{d}]" unless d.nil?} > "
36
- if $auto_answer and not force
37
- $stderr.puts ''
38
- else
39
- o = gets.chomp
40
- end
41
- o = d if o.nil? or o.empty?
42
- unless ans.nil? or ans.include? o
43
- $stderr.puts 'Answer not recognized.'
44
- return ask_user(q, d, ans)
45
- end
46
- o
47
- end
48
-
49
- ##=> Main <=
50
-
51
- miga = MiGA::MiGA.root_path
52
- $stderr.puts <<BANNER
53
- ===[ Welcome to MiGA, the Microbial Genome Atlas ]===
54
-
55
- I'm the initialization script, and I'll sniff around your computer to
56
- make sure you have all the requirements for MiGA data processing.
57
-
58
- BANNER
59
-
60
- if ask_user(
61
- 'Would you like to see all the requirements before starting?',
62
- 'no', %w(yes no)) == 'yes'
63
- File.open(File.expand_path('utils/requirements.txt', miga), 'r') do |fh|
64
- fh.each_line{ |ln| $stderr.puts ln }
65
- end
66
- end
67
-
68
- rc_path = File.expand_path('.miga_rc', ENV['HOME'])
69
- if File.exist? rc_path
70
- if ask_user(
71
- 'I found a previous configuration. Do you want to continue?',
72
- 'yes', %w(yes no)) == 'no'
73
- $stderr.puts 'OK, see you soon!'
74
- exit 0
75
- end
76
- end
77
- rc_fh = File.open(rc_path, 'w')
78
- rc_fh.puts <<BASH
79
- #!/bin/bash
80
- # `miga init` made this on #{Time.now}
81
-
82
- BASH
83
-
84
- # Check bash configuration file
85
- unless File.exist? o[:config]
86
- o[:config] = ask_user(
87
- 'Is there a script I need to load at startup?', o[:config])
88
- end
89
- if File.exist? o[:config]
90
- o[:config] = File.expand_path o[:config]
91
- $stderr.puts "Found bash configuration script: #{o[:config]}."
92
- rc_fh.puts "MIGA_STARTUP='#{o[:config]}'"
93
- rc_fh.puts '. "$MIGA_STARTUP"'
94
- end
95
- $stderr.puts ''
96
-
97
- # Check for software requirements
98
- $stderr.puts 'Looking for requirements:'
99
- if o[:mytaxa].nil?
100
- o[:mytaxa] = ask_user(
101
- 'Should I include MyTaxa modules?', 'yes', %w(yes no)) == 'yes'
102
- end
103
- rc_fh.puts 'export MIGA_MYTAXA="no"' unless o[:mytaxa]
104
- paths = {}
105
- rc_fh.puts 'MIGA_PATH=""'
106
- File.open(File.expand_path('utils/requirements.txt', miga), 'r') do |fh|
107
- fh.each_line do |ln|
108
- next if $. < 3
109
- r = ln.chomp.split(/\t+/)
110
- next if r[0] =~ /\(opt\)$/ and not o[:mytaxa]
111
- $stderr.print "Testing #{r[0]}#{" (#{r[3]})" if r[3]}... "
112
- path = nil
113
- loop do
114
- if File.exist? o[:config]
115
- d_path = File.dirname(`. "#{o[:config]}" && which "#{r[1]}"`)
116
- else
117
- d_path = File.dirname(`which "#{r[1]}"`)
118
- end
119
- if o[:ask] or d_path == '.'
120
- path = ask_user('Where can I find it?', d_path, nil, true)
121
- else
122
- path = d_path
123
- $stderr.puts path
124
- end
125
- if File.executable? File.expand_path(r[1], path)
126
- if d_path != path
127
- rc_fh.puts "MIGA_PATH=\"#{path}:$MIGA_PATH\" # #{r[1]}"
128
- end
129
- break
130
- end
131
- $stderr.print "I cannot find #{r[1]}. "
132
- end
133
- paths[r[1]] = File.expand_path(r[1], path).shellescape
134
- end
135
- end
136
- rc_fh.puts 'export PATH="$MIGA_PATH$PATH"'
137
- $stderr.puts ''
138
-
139
- # Check for other files
140
- if o[:mytaxa]
141
- $stderr.puts 'Looking for MyTaxa databases:'
142
- mt = File.dirname paths["MyTaxa"]
143
- $stderr.print 'Looking for scores... '
144
- unless Dir.exist? File.expand_path('db', mt)
145
- $stderr.puts "no.\nExecute 'python2 #{mt}/utils/download_db.py'."
146
- exit 1
147
- end
148
- $stderr.puts 'yes.'
149
- $stderr.print 'Looking for diamond db... '
150
- unless File.exist? File.expand_path('AllGenomes.faa.dmnd', mt)
151
- $stderr.puts "no.\nDownload " +
152
- "'http://enve-omics.ce.gatech.edu/data/public_mytaxa/" +
153
- "AllGenomes.faa.dmnd' into #{mt}."
154
- exit 1
155
- end
156
- $stderr.puts ''
157
- end
158
-
159
- # Check for R packages
160
- $stderr.puts 'Looking for R packages:'
161
- %w(enveomics.R ape cluster vegan).each do |pkg|
162
- $stderr.print "Testing #{pkg}... "
163
- `echo "library('#{pkg}')" | #{paths["R"].shellescape} --vanilla -q 2>&1`
164
- if $?.success?
165
- $stderr.puts 'yes.'
166
- else
167
- $stderr.puts 'no, installing.'
168
- $stderr.print '' +
169
- `echo "install.packages('#{pkg}', repos='http://cran.rstudio.com/')" \
170
- | #{paths["R"].shellescape} --vanilla -q 2>&1`
171
- `echo "library('#{pkg}')" | #{paths["R"].shellescape} --vanilla -q 2>&1`
172
- raise "Unable to auto-install R package #{pkg}." unless $?.success?
173
- end
174
- end
175
- $stderr.puts ''
176
-
177
- # Check for Ruby gems
178
- $stderr.puts 'Looking for Ruby gems:'
179
- %w(sqlite3 daemons json).each do |pkg|
180
- $stderr.print "Testing #{pkg}... "
181
- `#{paths["ruby"].shellescape} -r "#{pkg}" -e "" 2>/dev/null`
182
- if $?.success?
183
- $stderr.puts 'yes.'
184
- else
185
- $stderr.puts 'no, installing.'
186
- # This hackey mess is meant to ensure the test and installation are done
187
- # on the configuration Ruby, not on the Ruby currently executing the init
188
- # action
189
- $stderr.print `#{paths['ruby'].shellescape} \
190
- -r rubygems -r rubygems/gem_runner \
191
- -e "Gem::GemRunner.new.run %w(install --user #{pkg})" 2>&1`
192
- raise "Unable to auto-install Ruby gem #{pkg}." unless $?.success?
193
- end
194
- end
195
- $stderr.puts ''
196
-
197
- # Configure daemon
198
- $stderr.puts 'Default daemon configuration:'
199
- daemon_f = File.expand_path('.miga_daemon.json', ENV['HOME'])
200
- unless File.exist?(daemon_f) and ask_user(
201
- 'A template daemon already exists, do you want to preserve it?',
202
- 'yes', %w(yes no)) == 'yes'
203
- v = {created:Time.now.to_s, updated:Time.now.to_s}
204
- v[:type] = ask_user('Please select the type of daemon you want to setup',
205
- o[:dtype], %w(bash qsub msub slurm))
206
- case v[:type]
207
- when 'bash'
208
- v[:latency] = ask_user('How long should I sleep? (in seconds)', '30').to_i
209
- v[:maxjobs] = ask_user('How many jobs can I launch at once?', '6').to_i
210
- v[:ppn] = ask_user('How many CPUs can I use per job?', '2').to_i
211
- $stderr.puts 'Setting up internal daemon defaults.'
212
- $stderr.puts 'If you don\'t understand this just leave default values:'
213
- v[:cmd] = ask_user(
214
- "How should I launch tasks?\n %1$s: script path, %2$s: variables, " +
215
- "%3$d: CPUs, %4$s: log file, %5$s: task name.\n",
216
- "%2$s '%1$s' > '%4$s' 2>&1")
217
- v[:var] = ask_user(
218
- "How should I pass variables?\n %1$s: keys, %2$s: values.\n",
219
- "%1$s=%2$s")
220
- v[:varsep] = ask_user('What should I use to separate variables?', ' ')
221
- v[:alive] = ask_user(
222
- "How can I know that a process is still alive?\n %1$s: PID, " +
223
- "output should be 1 for running and 0 for non-running.\n",
224
- "ps -p '%1$s'|tail -n+2|wc -l")
225
- v[:kill] = ask_user(
226
- "How should I terminate tasks?\n %s: process ID.", "kill -9 '%s'")
227
- when 'slurm'
228
- queue = ask_user('What queue should I use?', nil, nil, true)
229
- v[:latency] = ask_user(
230
- 'How long should I sleep? (in seconds)', '150').to_i
231
- v[:maxjobs] = ask_user('How many jobs can I launch at once?', '300').to_i
232
- v[:ppn] = ask_user('How many CPUs can I use per job?', '2').to_i
233
- $stderr.puts 'Setting up internal daemon defaults.'
234
- $stderr.puts 'If you don\'t understand this just leave default values:'
235
- v[:cmd] = ask_user(
236
- "How should I launch tasks?\n %1$s: script path, %2$s: variables, " +
237
- "%3$d: CPUs, %4$d: log file, %5$s: task name.\n",
238
- "%2$s sbatch --partition='#{queue}' --export=ALL " +
239
- "--nodes=1 --ntasks-per-node=%3$d --output='%4$s' --job-name='%5$s' " +
240
- "--mem=9G --time=12:00:00 %1$s | perl -pe 's/.* //'")
241
- v[:var] = ask_user(
242
- "How should I pass variables?\n %1$s: keys, %2$s: values.\n",
243
- "%1$s=%2$s")
244
- v[:varsep] = ask_user('What should I use to separate variables?', ' ')
245
- v[:alive] = ask_user(
246
- "How can I know that a process is still alive?\n %1$s: job id, " +
247
- "output should be 1 for running and 0 for non-running.\n",
248
- "squeue -h -o %%t -j '%1$s' | grep '^PD\\|R\\|CF\\|CG$' " +
249
- "| tail -n 1 | wc -l")
250
- v[:kill] = ask_user(
251
- "How should I terminate tasks?\n %s: process ID.", "scancel '%s'")
252
- else # [qm]sub
253
- queue = ask_user('What queue should I use?', nil, nil, true)
254
- v[:latency] = ask_user(
255
- 'How long should I sleep? (in seconds)', '150').to_i
256
- v[:maxjobs] = ask_user('How many jobs can I launch at once?', '300').to_i
257
- v[:ppn] = ask_user('How many CPUs can I use per job?', '2').to_i
258
- $stderr.puts 'Setting up internal daemon defaults.'
259
- $stderr.puts 'If you don\'t understand this just leave default values:'
260
- v[:cmd] = ask_user(
261
- "How should I launch tasks?\n %1$s: script path, %2$s: variables, " +
262
- "%3$d: CPUs, %4$d: log file, %5$s: task name.\n",
263
- "#{v[:type]} -q '#{queue}' -v '%2$s' -l nodes=1:ppn=%3$d %1$s " +
264
- "-j oe -o '%4$s' -N '%5$s' -l mem=9g -l walltime=12:00:00 | grep .")
265
- v[:var] = ask_user(
266
- "How should I pass variables?\n %1$s: keys, %2$s: values.\n",
267
- "%1$s=%2$s")
268
- v[:varsep] = ask_user('What should I use to separate variables?', ',')
269
- if v[:type] == 'qsub'
270
- v[:alive] = ask_user(
271
- "How can I know that a process is still alive?\n %1$s: job id, " +
272
- "output should be 1 for running and 0 for non-running.\n",
273
- "qstat -f '%1$s'|grep ' job_state ='|perl -pe 's/.*= //'|grep '[^C]'"+
274
- "|tail -n1|wc -l|awk '{print $1}'")
275
- v[:kill] = ask_user(
276
- "How should I terminate tasks?\n %s: process ID.", "qdel '%s'")
277
- else
278
- v[:alive] = ask_user(
279
- "How can I know that a process is still alive?\n %1$s: job id, " +
280
- "output should be 1 for running and 0 for non-running.\n",
281
- "checkjob '%1$s'|grep '^State:'|perl -pe 's/.*: //'" +
282
- "|grep 'Deferred\\|Hold\\|Idle\\|Starting\\|Running\\|Blocked'"+
283
- "|tail -n1|wc -l|awk '{print $1}'")
284
- v[:kill] = ask_user(
285
- "How should I terminate tasks?\n %s: process ID.", "canceljob '%s'")
286
- end
287
- end
288
- File.open(daemon_f, 'w'){ |fh| fh.puts JSON.pretty_generate(v) }
289
- end
290
- $stderr.puts ''
291
-
292
- rc_fh.puts <<FOOT
293
-
294
- MIGA_CONFIG_VERSION='#{MiGA::MiGA.VERSION}'
295
- MIGA_CONFIG_LONGVERSION='#{MiGA::MiGA.LONG_VERSION}'
296
- MIGA_CONFIG_DATE='#{Time.now}'
297
-
298
- FOOT
299
-
300
- $stderr.puts 'Configuration complete. MiGA is ready to work!'
301
- $stderr.puts ''
data/actions/ln.rb DELETED
@@ -1,47 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- # @package MiGA
4
- # @license Artistic-2.0
5
-
6
- o = {q:true, info:false, force:false, method: :hardlink }
7
- OptionParser.new do |opt|
8
- opt_banner(opt)
9
- opt_object(opt, o, [:project, :dataset_opt])
10
- opt.on("-Q", "--project-target PATH",
11
- "(Mandatory) Path to the project where to link the dataset."
12
- ){ |v| o[:project2]=v }
13
- opt.on("-f", "--force",
14
- "Forces linking, even if dataset's preprocessing is incomplete."
15
- ){ |v| o[:force]=v }
16
- opt.on("-s", "--symlink",
17
- "Creates symlinks instead of the default hard links."
18
- ){ o[:method] = :symlink }
19
- opt.on("-c", "--copy",
20
- "Creates copies instead of the default hard links."){ o[:method] = :copy }
21
- opt_filter_datasets(opt, o)
22
- opt_common(opt, o)
23
- end.parse!
24
-
25
- ##=> Main <=
26
- opt_require(o, project:"-P", project2:"-Q")
27
-
28
- $stderr.puts "Loading project." unless o[:q]
29
- p = MiGA::Project.load(o[:project])
30
- raise "Impossible to load project: #{o[:project]}" if p.nil?
31
- q = MiGA::Project.load(o[:project2])
32
- raise "Impossible to load project: #{o[:project2]}" if q.nil?
33
-
34
- $stderr.puts "Listing dataset." unless o[:q]
35
- if o[:dataset].nil?
36
- ds = p.datasets
37
- else
38
- ds = [p.dataset(o[:dataset])]
39
- end
40
- ds = filter_datasets!(ds, o)
41
- ds.each do |d|
42
- next unless o[:force] or d.done_preprocessing?
43
- puts d.name
44
- q.import_dataset(d, o[:method])
45
- end
46
-
47
- $stderr.puts "Done." unless o[:q]
data/actions/ls.rb DELETED
@@ -1,61 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- # @package MiGA
4
- # @license Artistic-2.0
5
-
6
- o = {q:true, info:false, processing:false, silent:false, tabular:false}
7
- OptionParser.new do |opt|
8
- opt_banner(opt)
9
- opt_object(opt, o, [:project, :dataset_opt])
10
- opt_filter_datasets(opt, o)
11
- opt.on("-i", "--info",
12
- "Print additional information on each dataset."){ |v| o[:info]=v }
13
- opt.on("-p", "--processing",
14
- "Print information on processing advance."){ |v| o[:processing]=v }
15
- opt.on("-m", "--metadata STRING",
16
- "Print name and metadata field only. If set, ignores -i and assumes --tab."
17
- ){ |v| o[:datum]=v }
18
- opt.on("--tab",
19
- "Returns a tab-delimited table."){ |v| o[:tabular] = v }
20
- opt.on("-s", "--silent",
21
- "No output and exit with non-zero status if the dataset list is empty."
22
- ){ |v| o[:silent] = v }
23
- opt_common(opt, o)
24
- end.parse!
25
-
26
- ##=> Main <=
27
- opt_require(o, project:"-P")
28
- o[:q] = true if o[:silent]
29
-
30
- $stderr.puts "Loading project." unless o[:q]
31
- p = MiGA::Project.load(o[:project])
32
- raise "Impossible to load project: #{o[:project]}" if p.nil?
33
-
34
- $stderr.puts "Listing datasets." unless o[:q]
35
- if o[:dataset].nil?
36
- ds = p.datasets
37
- elsif MiGA::Dataset.exist? p, o[:dataset]
38
- ds = [p.dataset(o[:dataset])]
39
- else
40
- ds = []
41
- end
42
- ds = filter_datasets!(ds, o)
43
- exit(1) if o[:silent] and ds.empty?
44
-
45
- if not o[:datum].nil?
46
- ds.each do |d|
47
- v = d.metadata[ o[:datum] ]
48
- puts "#{d.name}\t#{v.nil? ? '?' : v}"
49
- end
50
- elsif o[:info]
51
- puts MiGA::MiGA.tabulate(
52
- MiGA::Dataset.INFO_FIELDS, ds.map{ |d| d.info }, o[:tabular])
53
- elsif o[:processing]
54
- comp = ["-","done","queued"]
55
- puts MiGA::MiGA.tabulate([:name] + MiGA::Dataset.PREPROCESSING_TASKS,
56
- ds.map{ |d| [d.name] + d.profile_advance.map{ |i| comp[i] } }, o[:tabular])
57
- else
58
- ds.each{|d| puts d.name}
59
- end
60
-
61
- $stderr.puts "Done." unless o[:q]