miga-base 0.7.16.7 → 0.7.18.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: affb99901724300ab46e2e7b1ef3f4e227f4288bef3a0634c8caa5f86b10b345
4
- data.tar.gz: fc51e29cc954afd22ae6ab59c59393165a3ea64455807bac8e2300660f2c2673
3
+ metadata.gz: 78b020bda6120b5a284100b9544d7a488d385630f21b5d6d5e2bb9b365c832fb
4
+ data.tar.gz: d8332711fdf86308726700654ee8db91bb097f022a26e1240e62d107efa833a1
5
5
  SHA512:
6
- metadata.gz: f8c9f1c7c307e408080ea80a4496eba48fe9d05220b6e636490958358c5a19da1f2ee86971c4b02b8200fcd8d5f01ec949b62dfbfce7b70b68da8ce940c7f5bf
7
- data.tar.gz: cc4fed1bdef75754f4110dc768cc87b4766e6f7cf91166fdfa321657952066831b39b6da10aa598929a0155c93f555ba7ed0c0afe3dc118a92a7c954ce271551
6
+ metadata.gz: 8b766a40acf37193204e6369e07f2ecba1a67aa6141133654ab69f89b2ba9a63a959bd3f5b7fd21c56542eea7e3174cc342b05dc76b73ed27c6a2b273c546ca9
7
+ data.tar.gz: 6f8d791b9bf443a1eb43198625ebce48501ebaa3680be0c4cef55931b176fd2365407865e6e43cd549217ada6c67e5c86b156bd0816b80f91c388af812af810c
data/bin/miga CHANGED
@@ -4,6 +4,8 @@
4
4
  # @license Artistic-2.0
5
5
 
6
6
  $LOAD_PATH.push File.expand_path('../../lib', __FILE__)
7
+ $LOAD_PATH.push File.expand_path('../../lib', File.realpath(__FILE__))
8
+
7
9
  require 'miga'
8
10
  require 'miga/cli'
9
11
 
@@ -73,6 +73,7 @@ class MiGA::Cli::Action::Daemon < MiGA::Cli::Action
73
73
  end
74
74
 
75
75
  def perform
76
+ cli.operation or raise 'Please specify a daemon operation'
76
77
  p = cli.load_project
77
78
  d = MiGA::Daemon.new(p, cli[:json])
78
79
  dopts = %i[latency maxjobs nodelist ppn shutdown_when_done]
@@ -1,5 +1,5 @@
1
1
  require 'miga/cli/action'
2
- require 'sqlite3'
2
+ require 'miga/sqlite'
3
3
 
4
4
  class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
5
5
  end
@@ -10,9 +10,7 @@ module MiGA::Cli::Action::Doctor::Base
10
10
  # tables saving +metric+ (:ani or :aai) and call +blk+ if the
11
11
  # file is corrupt or doesn't contain the expected structure
12
12
  def check_sqlite3_database(db_file, metric, &blk)
13
- SQLite3::Database.new(db_file) do |conn|
14
- conn.execute("select count(*) from #{metric}").first
15
- end
13
+ MiGA::SQLite.new(db_file).run("select count(*) from #{metric}")
16
14
  rescue SQLite3::SQLException, SQLite3::CorruptException
17
15
  blk.call
18
16
  end
@@ -107,11 +105,7 @@ module MiGA::Cli::Action::Doctor::Base
107
105
  dist = dataset.result(:distances) or return
108
106
  path = dist.file_path(:aai_db) or return
109
107
 
110
- o = []
111
- SQLite3::Database.new(path) do |conn|
112
- o = conn.execute('select seq2 from aai').map(&:first)
113
- end
114
- o
108
+ MiGA::SQLite.new(path).run('select seq2 from aai').map(&:first)
115
109
  end
116
110
 
117
111
  ##
@@ -120,34 +114,20 @@ module MiGA::Cli::Action::Doctor::Base
120
114
  def save_bidirectional(a, b)
121
115
  each_database_file(a) do |db_file, metric, result|
122
116
  data = nil
123
- SQLite3::Database.new(db_file) do |conn|
124
- data =
125
- conn.execute(
126
- "select seq1, seq2, #{metric}, sd, n, omega " +
127
- "from #{metric} where seq2 = ? limit 1", b.name
128
- ).first
129
- end
117
+ data = MiGA::SQLite.new(db_file).run(
118
+ "select seq1, seq2, #{metric}, sd, n, omega " +
119
+ "from #{metric} where seq2 = ? limit 1", b.name
120
+ ).first
130
121
  next if data.nil? || data.empty?
131
122
 
132
123
  db_file_b = File.join(File.dirname(db_file), "#{b.name}.db")
133
124
  next unless File.exist?(db_file_b)
134
125
 
135
126
  data[0], data[1] = data[1], data[0]
136
- SQLite3::Database.new(db_file_b) do |conn|
137
- attempts = 0
138
- begin
139
- attempts += 1
140
- conn.execute(
141
- "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
142
- "values(?, ?, ?, ?, ?, ?)", data
143
- )
144
- rescue SQLite3::BusyException => e
145
- raise "Cannot populate #{db_file_b}: #{e.message}" if attempts > 3
146
-
147
- sleep(1)
148
- retry
149
- end
150
- end
127
+ MiGA::SQLite.new(db_file_b).run(
128
+ "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
129
+ "values(?, ?, ?, ?, ?, ?)", data
130
+ )
151
131
  end
152
132
  end
153
133
  end
@@ -37,9 +37,9 @@ class MiGA::Cli::Action::Run < MiGA::Cli::Action
37
37
  cli[:thr] ||= ENV['CORES'].to_i unless ENV['CORES'].nil?
38
38
  cli[:result] = File.basename(cli[:result].to_s, '.bash').to_sym
39
39
  end
40
- cli[:project] = nil if cli[:project].empty?
41
- cli[:dataset] = nil if cli[:dataset].empty?
42
- cli[:result] = nil if cli[:result].empty?
40
+ %i[project dataset result].each do |i|
41
+ cli[i] = nil if cli[i].nil? || cli[i].empty?
42
+ end
43
43
 
44
44
  # Unset dataset if the requested result is for projects
45
45
  if (MiGA::Project.RESULT_DIRS.keys + [:p]).include? cli[:result]
@@ -184,7 +184,7 @@ module MiGA::Cli::Action::Wf
184
184
  def call_cli(cmd)
185
185
  cmd << '-v' if cli[:verbose]
186
186
  MiGA::MiGA.DEBUG "Cli::Action::Wf.call_cli #{cmd}"
187
- MiGA::Cli.new(cmd.map(&:to_s)).launch
187
+ MiGA::Cli.new(cmd.map(&:to_s)).launch(true)
188
188
  end
189
189
 
190
190
  def run_daemon
@@ -73,7 +73,7 @@ class MiGA::Daemon < MiGA::MiGA
73
73
  say 'MiGA:%s launched' % project.name
74
74
  say '-----------------------------------'
75
75
  miga_say "Saving log to: #{output_file}" unless show_log?
76
- recalculate_status!
76
+ queue_maintenance
77
77
  load_status
78
78
  say 'Configuration options:'
79
79
  say @runopts.to_s
@@ -92,8 +92,7 @@ class MiGA::Daemon < MiGA::MiGA
92
92
  flush!
93
93
  if (loop_i % 12).zero?
94
94
  purge!
95
- # TEMPORARILY DISABLED:
96
- # recalculate_status!
95
+ queue_maintenance
97
96
  end
98
97
  save_status
99
98
  sleep(latency)
@@ -101,9 +100,13 @@ class MiGA::Daemon < MiGA::MiGA
101
100
  true
102
101
  end
103
102
 
104
- def recalculate_status!
105
- say 'Recalculating status for all datasets'
106
- project.each_dataset(&:recalculate_status)
103
+ ##
104
+ # Queue maintenance tasks as an analysis job
105
+ def queue_maintenance
106
+ return if bypass_maintenance?
107
+
108
+ say 'Queueing maintenance tasks'
109
+ queue_job(:maintenance)
107
110
  end
108
111
 
109
112
  ##
@@ -271,6 +274,11 @@ class MiGA::Daemon < MiGA::MiGA
271
274
  # Avoid single datasets hogging resources
272
275
  @jobs_to_run.rotate! rand(jobs_to_run.size)
273
276
 
277
+ # Prioritize project-wide jobs
278
+ project_jobs = @jobs_to_run.select { |i| i[:ds].nil? }
279
+ @jobs_to_run.delete_if { |i| i[:ds].nil? }
280
+ @jobs_to_run.prepend(*project_jobs)
281
+
274
282
  # Launch as many +jobs_to_run+ as possible
275
283
  while (hostk = next_host)
276
284
  break if jobs_to_run.empty?
@@ -309,11 +317,13 @@ class MiGA::Daemon < MiGA::MiGA
309
317
  job[:hostk] = hostk
310
318
  job[:cmd] = job[:cmd].miga_variables(host: nodelist[hostk])
311
319
  job[:pid] = spawn job[:cmd]
312
- Process.detach job[:pid] unless [nil, '', 0].include?(job[:pid])
320
+ MiGA::MiGA.DEBUG "Detaching PID: #{job[:pid]}"
321
+ Process.detach(job[:pid]) unless [nil, '', 0].include?(job[:pid])
313
322
  when 'bash'
314
323
  # Local job
315
324
  job[:pid] = spawn job[:cmd]
316
- Process.detach job[:pid] unless [nil, '', 0].include?(job[:pid])
325
+ MiGA::MiGA.DEBUG "Detaching PID: #{job[:pid]}"
326
+ Process.detach(job[:pid]) unless [nil, '', 0].include?(job[:pid])
317
327
  else
318
328
  # Schedule cluster job (qsub, msub, slurm)
319
329
  job[:pid] = `#{job[:cmd]}`.chomp
@@ -17,7 +17,7 @@ module MiGA::Daemon::Base
17
17
  if !force && v == 0 && k != :verbosity
18
18
  raise "Daemon's #{k} cannot be set to zero"
19
19
  end
20
- when :shutdown_when_done, :show_log
20
+ when :shutdown_when_done, :show_log, :bypass_maintenance
21
21
  v = !!v
22
22
  when :nodelist
23
23
  if v =~ /^\$/
@@ -63,6 +63,12 @@ module MiGA::Daemon::Base
63
63
  !!runopts(:shutdown_when_done)
64
64
  end
65
65
 
66
+ ##
67
+ # Should the daemon ignore regular maintenance steps?
68
+ def bypass_maintenance?
69
+ !!runopts(:bypass_maintenance)
70
+ end
71
+
66
72
  ##
67
73
  # Returns the level of verbosity for the daemon as an Integer, or 1 if unset.
68
74
  # Verbosity levels are:
@@ -7,7 +7,7 @@ require 'miga/metadata'
7
7
  require 'miga/dataset/result'
8
8
  require 'miga/dataset/status'
9
9
  require 'miga/dataset/hooks'
10
- require 'sqlite3'
10
+ require 'miga/sqlite'
11
11
 
12
12
  ##
13
13
  # Dataset representation in MiGA
@@ -190,8 +190,7 @@ class MiGA::Dataset < MiGA::MiGA
190
190
  r = result(ref_project ? :taxonomy : :distances)
191
191
  return nil if r.nil?
192
192
 
193
- db = SQLite3::Database.new(r.file_path(:aai_db))
194
- db.execute(
193
+ MiGA::SQLite.new(r.file_path(:aai_db)).run(
195
194
  'SELECT seq2, aai FROM aai WHERE seq2 != ? ' \
196
195
  'GROUP BY seq2 ORDER BY aai DESC LIMIT ?', [name, how_many]
197
196
  )
@@ -1,4 +1,4 @@
1
- require 'sqlite3'
1
+ require 'miga/sqlite'
2
2
  require 'miga/result'
3
3
  require 'miga/dataset/base'
4
4
  require 'miga/common/with_result'
@@ -151,13 +151,13 @@ module MiGA::Dataset::Result
151
151
  db = r.file_path(db_type)
152
152
  next if db.nil? || !File.size?(db)
153
153
 
154
- sqlite_db = SQLite3::Database.new db
154
+ sqlite_db = MiGA::SQLite.new(db)
155
155
  table = db_type[-6..-4]
156
- val = sqlite_db.execute "select seq2 from #{table}"
156
+ val = sqlite_db.run("select seq2 from #{table}")
157
157
  next if val.empty?
158
158
 
159
159
  (val.map(&:first) - ref).each do |extra|
160
- sqlite_db.execute "delete from #{table} where seq2=?", extra
160
+ sqlite_db.run("delete from #{table} where seq2=?", extra)
161
161
  end
162
162
  end
163
163
  end
@@ -156,10 +156,12 @@ class MiGA::RemoteDataset < MiGA::MiGA
156
156
  return nil unless metadata[:ncbi_asm]
157
157
 
158
158
  ncbi_asm_id = self.class.ncbi_asm_acc2id metadata[:ncbi_asm]
159
- doc = MiGA::Json.parse(
160
- self.class.download(:ncbi_summary, :assembly, ncbi_asm_id, :json),
161
- symbolize: false, contents: true
162
- )
159
+ txt = nil
160
+ 3.times do
161
+ txt = self.class.download(:ncbi_summary, :assembly, ncbi_asm_id, :json)
162
+ txt.empty? ? sleep(1) : break
163
+ end
164
+ doc = MiGA::Json.parse(txt, symbolize: false, contents: true)
163
165
  @_ncbi_asm_json_doc = doc['result'][ doc['result']['uids'].first ]
164
166
  end
165
167
 
@@ -0,0 +1,49 @@
1
+ # @package MiGA
2
+ # @license Artistic-2.0
3
+
4
+ require 'sqlite3'
5
+
6
+ ##
7
+ # SQLite3 wrapper for MiGA.
8
+ class MiGA::SQLite < MiGA::MiGA
9
+ class << self
10
+ ##
11
+ # Default parsing options. Supported +opts+ keys:
12
+ # - +:busy_attempts+: Number of times to retry when database is busy
13
+ # (default: 3)
14
+ def default_opts(opts = {})
15
+ opts[:busy_attempts] ||= 3
16
+ opts
17
+ end
18
+ end
19
+
20
+ ##
21
+ # Options hash
22
+ attr :opts
23
+
24
+ ##
25
+ # Database absolute path
26
+ attr :path
27
+
28
+ ##
29
+ # Create MiGA::SQLite with database in +path+ (without opening a connection)
30
+ # and options +opts+ (see +.default_opts+)
31
+ def initialize(path, opts = {})
32
+ @opts = MiGA::SQLite.default_opts(opts)
33
+ @path = File.absolute_path(path)
34
+ end
35
+
36
+ ##
37
+ # Executes +cmd+ and returns the result
38
+ def run(*cmd)
39
+ busy_attempts ||= 0
40
+ conn = SQLite3::Database.new(path)
41
+ conn.execute(*cmd)
42
+ rescue SQLite3::BusyException => e
43
+ busy_attempts += 1
44
+ raise "Database busy #{path}: #{e.message}" if busy_attempts >= 3
45
+
46
+ sleep(1)
47
+ retry
48
+ end
49
+ end
@@ -8,7 +8,7 @@ module MiGA
8
8
  # - Float representing the major.minor version.
9
9
  # - Integer representing gem releases of the current version.
10
10
  # - Integer representing minor changes that require new version number.
11
- VERSION = [0.7, 16, 7]
11
+ VERSION = [0.7, 18, 0]
12
12
 
13
13
  ##
14
14
  # Nickname for the current major.minor version.
@@ -16,7 +16,7 @@ module MiGA
16
16
 
17
17
  ##
18
18
  # Date of the current gem release.
19
- VERSION_DATE = Date.new(2020, 11, 22)
19
+ VERSION_DATE = Date.new(2020, 12, 27)
20
20
 
21
21
  ##
22
22
  # Reference of MiGA.
@@ -0,0 +1,9 @@
1
+ #!/bin/bash
2
+ # Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES
3
+ set -e
4
+ SCRIPT="maintenance"
5
+ # shellcheck source=scripts/miga.bash
6
+ . "$MIGA/scripts/miga.bash" || exit 1
7
+
8
+ miga doctor --only status -P "$PROJECT" -t "$CORES" -v
9
+
@@ -12,12 +12,15 @@ cd "$DIR"
12
12
  miga date > "miga-project.start"
13
13
 
14
14
  # Execute doctor
15
+ echo "# Doctor"
15
16
  miga doctor -P "$PROJECT" -t "$CORES" -v
16
17
 
17
18
  # Index taxonomy
19
+ echo "# Index taxonomy"
18
20
  miga tax_index -P "$PROJECT" -i "miga-project.taxonomy.json" --ref --active
19
21
 
20
22
  # Index metadata
23
+ echo "# Index metadata"
21
24
  ruby -I "$MIGA/lib" \
22
25
  "$MIGA/utils/index_metadata.rb" "$PROJECT" "miga-project.metadata.db"
23
26
 
@@ -93,7 +93,7 @@ class DaemonTest < Test::Unit::TestCase
93
93
  0 => /-{20}\n/,
94
94
  1 => /MiGA:#{p.name} launched/,
95
95
  2 => /-{20}\n/,
96
- 6 => /Probing running jobs\n/
96
+ 8 => /Probing running jobs\n/
97
97
  }.each { |k, v| assert_match(v, l[k], "unexpected line: #{k}") }
98
98
  ensure
99
99
  begin
@@ -160,8 +160,9 @@ class DaemonTest < Test::Unit::TestCase
160
160
  assert_equal(0, d1.jobs_running.size)
161
161
  assert_equal(0, d1.jobs_to_run.size)
162
162
  capture_stderr { d1.in_loop }
163
+ # 3 dataset jobs + 1 maintenance job:
163
164
  assert_equal(1, d1.jobs_running.size)
164
- assert_equal(2, d1.jobs_to_run.size)
165
+ assert_equal(3, d1.jobs_to_run.size)
165
166
  end
166
167
 
167
168
  def test_maxjobs_runopts
@@ -172,7 +173,7 @@ class DaemonTest < Test::Unit::TestCase
172
173
  assert_equal(0, d1.jobs_to_run.size)
173
174
  capture_stderr { d1.in_loop }
174
175
  assert_equal(2, d1.jobs_running.size)
175
- assert_equal(1, d1.jobs_to_run.size)
176
+ assert_equal(2, d1.jobs_to_run.size)
176
177
  end
177
178
 
178
179
  def test_load_status
@@ -232,6 +233,7 @@ class DaemonTest < Test::Unit::TestCase
232
233
  end
233
234
 
234
235
  def test_shutdown_when_done
236
+ daemon.runopts(:bypass_maintenance, true)
235
237
  daemon.runopts(:shutdown_when_done, true)
236
238
  out = capture_stderr { assert { !daemon.in_loop } }.string
237
239
  assert_match(/Nothing else to do/, out)
@@ -317,4 +319,20 @@ class DaemonTest < Test::Unit::TestCase
317
319
  out = capture_stderr { d1.in_loop }.string
318
320
  assert_match(/Daemon loop start/, out)
319
321
  end
322
+
323
+ def test_bypass_maintenance
324
+ # Default (run maintenance)
325
+ d = daemon(0)
326
+ d.runopts(:latency, 0, true)
327
+ capture_stderr { d.in_loop }
328
+ assert_equal(1, d.jobs_running.size)
329
+ assert_equal(:maintenance, d.jobs_running.first[:job])
330
+
331
+ # Bypassing maintenance
332
+ d = daemon(1)
333
+ d.runopts(:latency, 0, true)
334
+ d.runopts(:bypass_maintenance, true)
335
+ capture_stderr { d.in_loop }
336
+ assert_equal([], d.jobs_running)
337
+ end
320
338
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: miga-base
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.16.7
4
+ version: 0.7.18.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Luis M. Rodriguez-R
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-11-22 00:00:00.000000000 Z
11
+ date: 2020-12-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: daemons
@@ -196,6 +196,7 @@ files:
196
196
  - lib/miga/result/dates.rb
197
197
  - lib/miga/result/source.rb
198
198
  - lib/miga/result/stats.rb
199
+ - lib/miga/sqlite.rb
199
200
  - lib/miga/tax_dist.rb
200
201
  - lib/miga/tax_index.rb
201
202
  - lib/miga/taxonomy.rb
@@ -211,6 +212,7 @@ files:
211
212
  - scripts/essential_genes.bash
212
213
  - scripts/haai_distances.bash
213
214
  - scripts/init.bash
215
+ - scripts/maintenance.bash
214
216
  - scripts/miga.bash
215
217
  - scripts/mytaxa.bash
216
218
  - scripts/mytaxa_scan.bash