miga-base 1.1.0.0 → 1.1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4076b3b3a4a4143ac9100ce4d58fada7615f68ad3e6174445510655f62904867
-  data.tar.gz: '0975a5feb4c9eb71a474be87dd14b58297ef1aa7bd8612c20f1ce65febbdf980'
+  metadata.gz: 70955c93dc93a73a0ac28d3aaa8f75b4d0f2e0e5dc8797d1fc2bf57e969a8fbe
+  data.tar.gz: 89ba5a42a7a12a104a12f9116f46b0e107bc82b22731622403d09c2e2b7459c7
 SHA512:
-  metadata.gz: ebcb7fe28d415ca9709433975585518eb1ecd8e8270c584b6579da222e4d3733cc20d810787c3f764f6a6136e1a6f09b7cb6b1c00114c3ea9c0885370654f3a7
-  data.tar.gz: '082bd856ed21487e5de709e2067f1d3453f824e0ece7a77716c6fbe70d88a16c4d295196d5c6133e5667142e25f55f7e48e4a785afd160a14e8195a9b7efa6c2'
+  metadata.gz: d57d59d5cb439119dda8f4c8ff90cd3d2d253ff631117c3005b60f321014e61d584c04655f6861de29807cfa2a9c569f2e0e35fde6931cc7140982512a35f13c
+  data.tar.gz: a9c1090876f87c32376b477363f95410786a2d2da73f72be3d81a2fcfe28c6f1dc3829b741b1ad4986ded59bdc06999ad775739d8ecbd7694b4c6e40077905ec

data/lib/miga/daemon/base.rb

@@ -12,9 +12,10 @@ module MiGA::Daemon::Base
     k = k.to_sym
     unless v.nil?
       case k
-      when :latency, :maxjobs, :ppn, :format_version, :verbosity
+      when :latency, :maxjobs, :ppn, :ppn_project, :format_version, :verbosity,
+           :skip_maintenance
         v = v.to_i
-        if !force && v == 0 && k != :verbosity
+        if !force && v == 0 && !%i[verbosity skip_maintenance].include?(k)
           raise "Daemon's #{k} cannot be set to zero"
         end
       when :shutdown_when_done, :show_log, :bypass_maintenance
@@ -51,9 +52,23 @@ module MiGA::Daemon::Base
   end
 
   ##
-  # Returns Integer indicating the number of CPUs per job
-  def ppn
-    runopts(:ppn)
+  # Returns the number of times maintenance should be skipped before running
+  def skip_maintenance
+    runopts(:skip_maintenance) || 0
+  end
+
+  ##
+  # Returns the running option +opt+ in jobs for +what+. +what+ can be
+  # +:dataset+ or +:projects+
+  def runopts_for(opt, what)
+    runopts(:"#{opt}_#{what}") || runopts(opt)
+  end
+
+  ##
+  # Returns Integer indicating the number of CPUs per job, in jobs for +what+.
+  # See also #runopts_for
+  def ppn(what = :dataset)
+    runopts_for(:ppn, what)
   end
 
   ##

data/lib/miga/daemon.rb

@@ -93,7 +93,7 @@ class MiGA::Daemon < MiGA::MiGA
     flush!
     if (loop_i % 12).zero?
       purge!
-      queue_maintenance
+      queue_maintenance if (loop_i % (12 * (skip_maintenance + 1))).zero?
     end
     save_status
     sleep(latency)
@@ -231,10 +231,11 @@ class MiGA::Daemon < MiGA::MiGA
   # Construct the command for the given job definition with current
   # daemon settings
   def job_cmd(to_run)
+    what = to_run[:ds].nil? ? :project : :dataset
     vars = {
       'PROJECT' => project.path,
-      'RUNTYPE' => runopts(:type),
-      'CORES' => ppn,
+      'RUNTYPE' => runopts_for(:type, what),
+      'CORES' => ppn(what),
       'MIGA' => MiGA::MiGA.root_path
     }
     vars['DATASET'] = to_run[:ds].name unless to_run[:ds].nil?
@@ -246,13 +247,13 @@ class MiGA::Daemon < MiGA::MiGA
               ),
       vars: vars.map do |k, v|
               runopts(:var).miga_variables(key: k, value: v)
-            end.join(runopts(:varsep)),
-      cpus: ppn,
+            end.join(runopts_for(:varsep, what)),
+      cpus: ppn(what),
       log: File.join(log_dir, "#{to_run[:ds_name]}.log"),
       task_name: to_run[:task_name],
       miga: File.join(MiGA::MiGA.root_path, 'bin/miga').shellescape
     }
-    runopts(:cmd).miga_variables(var_hsh)
+    runopts_for(:cmd, what).miga_variables(var_hsh)
   end
 
   ##

data/lib/miga/version.rb

@@ -12,7 +12,7 @@ module MiGA
   # - String indicating release status:
   #   - rc* release candidate, not released as gem
   #   - [0-9]+ stable release, released as gem
-  VERSION = [1.1, 0, 0].freeze
+  VERSION = [1.1, 2, 1].freeze
 
   ##
   # Nickname for the current major.minor version.
@@ -20,7 +20,7 @@ module MiGA
 
   ##
   # Date of the current gem relese.
-  VERSION_DATE = Date.new(2021, 10, 28)
+  VERSION_DATE = Date.new(2021, 11, 7)
 
   ##
   # References of MiGA

data/utils/FastAAI/FastAAI

@@ -583,6 +583,24 @@ class input_file:
 		self.initial_state = "protein+HMM"
 		
 		self.verbose = verbosity
+		
+		#r_scripts_loc = os.path.dirname(sys.modules['metapop'].__file__) + "/metapop_r/"
+		#"00.Libraries/01.SCG_HMMs/Complete_SCG_DB.hmm"
+		self.hmm_path = None
+		try:
+			#Try to locate the data bundled as it would be with a pip/conda install.
+			script_path = os.path.dirname(sys.modules['fastAAI_HMM_models'].__file__)
+			hmm_complete_model = script_path + '/00.Libraries/01.SCG_HMMs/Complete_SCG_DB.hmm'
+			self.hmm_path = str(hmm_complete_model)
+			#Check that the file exists or fail to the except.
+			fh = open(self.hmm_path)
+			fh.close()
+		except:
+			#Look in the same dir as the script; old method/MiGA friendly
+			script_path = Path(__file__)
+			script_dir = script_path.parent
+			hmm_complete_model = script_dir / "00.Libraries/01.SCG_HMMs/Complete_SCG_DB.hmm"
+			self.hmm_path = str(hmm_complete_model)
 	
 	#Functions for externally setting status and file paths of particular types
 	def set_genome(self, path):
@@ -2701,6 +2719,7 @@ def merge_db_thread_starter(rev_index, per_db_accs):
 	accs_per_db = per_db_accs
 	
 	
+	
 def merge_db(recipient, donors, verbose, threads):
 	#Prettier on the CLI
 
@@ -2828,111 +2847,109 @@ def merge_db(recipient, donors, verbose, threads):
 		
 	all_accessions = list(all_accessions)
 	
-	acc_args = [(acc, donor_dbs, recipient) for acc in all_accessions]
-	
-	if not os.path.exists("FastAAI_temp"):
-		os.mkdir("FastAAI_temp")
 	
 	print("")
 	print("Formatting data to add to database. Started at", curtime())
 	
-	if verbose:
-		print("")
-		count = 0
-		total_counts = len(acc_args)
-		try:
-			percentage = (count/total_counts)*100
-			sys.stdout.write("Completion".rjust(3)+ ' |'+('#'*int(percentage/2)).ljust(50)+'| ' + ('%.2f'%percentage).rjust(7)+'% ( ' + str(count) + " of " + str(total_counts) + ' done at ' + curtime() + " )\n")
-			sys.stdout.flush()			
-		except:
-			#It's not really a big deal if the progress bar cannot be printed.
-			pass
-	
-	last_pct = 0
-	
-	pool = multiprocessing.Pool(threads, initializer=merge_db_thread_starter, initargs = (reverse_genome_indices, accs_per_db,))
+	temp_dir = tempfile.mkdtemp()
+	try:
+		acc_args = [(acc, donor_dbs, recipient, temp_dir) for acc in all_accessions]
 	
-	quiverfull = []
-	for result in pool.imap_unordered(pull_and_merge_accession, acc_args):
-		acc = result[0]
-		child = result[1]
-		#sub_gak = result[2]
-		
-		quiverfull.append([acc, child])
-		#gaks.extend(sub_gak)
-		
 		if verbose:
-			count += 1
+			print("")
+			count = 0
+			total_counts = len(acc_args)
 			try:
 				percentage = (count/total_counts)*100
-				log_time = curtime()
-				sys.stdout.write('\033[A')
-				sys.stdout.flush()
 				sys.stdout.write("Completion".rjust(3)+ ' |'+('#'*int(percentage/2)).ljust(50)+'| ' + ('%.2f'%percentage).rjust(7)+'% ( ' + str(count) + " of " + str(total_counts) + ' done at ' + curtime() + " )\n")
 				sys.stdout.flush()			
 			except:
 				#It's not really a big deal if the progress bar cannot be printed.
 				pass
-	
-	pool.close()
-	pool.join()
-	
-	print("")
-	print("Adding data to final database. Started at", curtime())
-	
-	if verbose:
-		print("")
 		
-		count = 0
-		total_counts = len(acc_args)
-		try:
-			percentage = (count/total_counts)*100
-			sys.stdout.write("Completion".rjust(3)+ ' |'+('#'*int(percentage/2)).ljust(50)+'| ' + ('%.2f'%percentage).rjust(7)+'% ( ' + str(count) + " of " + str(total_counts) + ' done at ' + curtime() + " )\n")
-			sys.stdout.flush()
-		except:
-			#It's not really a big deal if the progress bar cannot be printed.
-			pass
-	
-	last_pct = 0
-	
-	recipient.activate_connection()
-	genome_list_update_sql = "INSERT OR REPLACE INTO genome_index VALUES (?, ?, ?)"
-	genome_reindex = []
-	for g in joint_genome_index:
-		genome_reindex.append((g, joint_genome_index[g], joint_genome_counts[joint_genome_index[g]]))
+		last_pct = 0
 		
-	recipient.cursor.executemany(genome_list_update_sql, genome_reindex)
-	recipient.connection.commit()
-	
-	del genome_reindex
-	
-	for result in quiverfull:
-		acc = result[0]
-		child = result[1]
+		pool = multiprocessing.Pool(threads, initializer=merge_db_thread_starter, initargs = (reverse_genome_indices, accs_per_db,))
 		
-		recipient.add_child_to_parent(acc, child, genomes_too = True, update_gak = True)
+		quiverfull = []
+		for result in pool.imap_unordered(pull_and_merge_accession, acc_args):
+			acc = result[0]
+			child = result[1]
+			#sub_gak = result[2]
+			
+			quiverfull.append([acc, child])
+			#gaks.extend(sub_gak)
+			
+			if verbose:
+				count += 1
+				try:
+					percentage = (count/total_counts)*100
+					log_time = curtime()
+					sys.stdout.write('\033[A')
+					sys.stdout.flush()
+					sys.stdout.write("Completion".rjust(3)+ ' |'+('#'*int(percentage/2)).ljust(50)+'| ' + ('%.2f'%percentage).rjust(7)+'% ( ' + str(count) + " of " + str(total_counts) + ' done at ' + curtime() + " )\n")
+					sys.stdout.flush()			
+				except:
+					#It's not really a big deal if the progress bar cannot be printed.
+					pass
+		
+		pool.close()
+		pool.join()
+		
+		print("")
+		print("Adding data to final database. Started at", curtime())
 		
 		if verbose:
-			count += 1
+			print("")
+			
+			count = 0
+			total_counts = len(acc_args)
 			try:
 				percentage = (count/total_counts)*100
-				log_time = curtime()
-				sys.stdout.write('\033[A')
-				sys.stdout.flush()
 				sys.stdout.write("Completion".rjust(3)+ ' |'+('#'*int(percentage/2)).ljust(50)+'| ' + ('%.2f'%percentage).rjust(7)+'% ( ' + str(count) + " of " + str(total_counts) + ' done at ' + curtime() + " )\n")
 				sys.stdout.flush()
 			except:
 				#It's not really a big deal if the progress bar cannot be printed.
 				pass
-	
-	os.rmdir("FastAAI_temp")
-	''' We're only ever increasing the DB size, so we don't actually need to vacuum it.
-	if recip_check != "created":
-		print("")
-		print("Cleaning up the database after the update. This may take a while.")
-		recipient.connection.execute("VACUUM")
-		recipient.connection.close()
-	'''
+		
+		last_pct = 0
+		
+		recipient.activate_connection()
+		genome_list_update_sql = "INSERT OR REPLACE INTO genome_index VALUES (?, ?, ?)"
+		genome_reindex = []
+		for g in joint_genome_index:
+			genome_reindex.append((g, joint_genome_index[g], joint_genome_counts[joint_genome_index[g]]))
+			
+		recipient.cursor.executemany(genome_list_update_sql, genome_reindex)
+		recipient.connection.commit()
+		
+		del genome_reindex
+		
+		for result in quiverfull:
+			acc = result[0]
+			child = result[1]
+			
+			recipient.add_child_to_parent(acc, child, genomes_too = True, update_gak = True)
+			
+			if verbose:
+				count += 1
+				try:
+					percentage = (count/total_counts)*100
+					log_time = curtime()
+					sys.stdout.write('\033[A')
+					sys.stdout.flush()
+					sys.stdout.write("Completion".rjust(3)+ ' |'+('#'*int(percentage/2)).ljust(50)+'| ' + ('%.2f'%percentage).rjust(7)+'% ( ' + str(count) + " of " + str(total_counts) + ' done at ' + curtime() + " )\n")
+					sys.stdout.flush()
+				except:
+					#It's not really a big deal if the progress bar cannot be printed.
+					pass
+	except:
+		#Error
+		shutil.rmtree(temp_dir)
+	finally:
+		#Success
+		shutil.rmtree(temp_dir)
+		
 	print("\nDatabases merged!")
 	
 	return None
@@ -2947,14 +2964,14 @@ def pull_and_merge_accession(args):
 		accession_inverter[accession_index[acc]] = sql_friendly_accession
 	
 	#joint_genome_index, accession_index, accession_inverter, accs_per_db are global already.
-	acc, donor_dbs, recipient = args[0], args[1], args[2]
+	acc, donor_dbs, recipient, temp = args[0], args[1], args[2], args[3]
 	
 	acc_name = accession_inverter[acc]
 	acc_name_gens = acc_name + "_genomes"
 	
 	query_sql = "SELECT * FROM " + acc_name
 	
-	temp_db = fastaai_database("FastAAI_temp/"+acc_name+".db")
+	temp_db = fastaai_database(os.path.normpath(temp+"/"+acc_name+".db"))
 	temp_db.activate_connection()
 	
 	create_command = "CREATE TABLE IF NOT EXISTS " + acc_name + " (kmer INTEGER PRIMARY KEY, genomes array)"
@@ -3627,4 +3644,4 @@ def main():
 if __name__ == "__main__":
 	main()
 
-	
+	

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 1.1.0.0
+  version: 1.1.2.1
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-28 00:00:00.000000000 Z
+date: 2021-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons