miga-base 1.1.1.0 → 1.1.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a666ed2219c2e1573dff12caca0b841d4c649bad62f4f5c0d274fc2c5407b178
-  data.tar.gz: 9e6a46b35a83b8f2349bb2a676f6c2fba7e375b52156734b636084d68c4626af
+  metadata.gz: 4603bf75db8f82a1a30c1d6da1ab045139b13bf1eec5fd2d5be5c79fb2ac0442
+  data.tar.gz: 24324b35a885453c01fada6fb07527616b3a952ab4ccb9b9c818c5d06e580acd
 SHA512:
-  metadata.gz: 61abc23642093894f068e1edc6252fb7a4e95035284f5469a0c955aebdadbdd2d62a76f395c933c7e819498e9a3b7685dd7257afe6263bc0b51111b141d00a13
-  data.tar.gz: 72f7024d6d5594a794dd5d7ca68d7b5896d28109a2800189981c9ba1fed85a3ad2e51303a547991fc998360426a5e85fe03dc9e7a61d38520744b6ed2dece8b9
+  metadata.gz: d9e555d736e3987d4afc45707307b7f95d2164cb8db822ad639e0f71e919c67972bff76aefdc7b0a1a1e7120e3100b863a8dde14470066ac7d4ab0b5be5c20f7
+  data.tar.gz: 54d1823817cb2d4220d885e84e941cf5e4df88f850824cf3837ed716879aec4733f2d2df98bedc03aa29878a1e5bd7641b015c76738eed841d08046c9f50d1bb

data/lib/miga/daemon/base.rb

@@ -12,9 +12,10 @@ module MiGA::Daemon::Base
     k = k.to_sym
     unless v.nil?
       case k
-      when :latency, :maxjobs, :ppn, :ppn_project, :format_version, :verbosity
+      when :latency, :maxjobs, :ppn, :ppn_project, :format_version, :verbosity,
+           :skip_maintenance
         v = v.to_i
-        if !force && v == 0 && k != :verbosity
+        if !force && v == 0 && !%i[verbosity skip_maintenance].include?(k)
           raise "Daemon's #{k} cannot be set to zero"
         end
       when :shutdown_when_done, :show_log, :bypass_maintenance
@@ -50,6 +51,12 @@ module MiGA::Daemon::Base
     runopts(:nodelist)
   end
 
+  ##
+  # Returns the number of times maintenance should be skipped before running
+  def skip_maintenance
+    runopts(:skip_maintenance) || 0
+  end
+
   ##
   # Returns the running option +opt+ in jobs for +what+. +what+ can be
   # +:dataset+ or +:projects+

data/lib/miga/daemon.rb

@@ -93,7 +93,7 @@ class MiGA::Daemon < MiGA::MiGA
     flush!
     if (loop_i % 12).zero?
       purge!
-      queue_maintenance
+      queue_maintenance if (loop_i % (12 * (skip_maintenance + 1))).zero?
     end
     save_status
     sleep(latency)

data/lib/miga/dataset.rb

@@ -107,7 +107,7 @@ class MiGA::Dataset < MiGA::MiGA
     metadata[:warn] = "Inactive: #{reason}" unless reason.nil?
     metadata[:inactive] = true
     metadata.save
-    project.recalculate_tasks('Reference dataset inactivated') if ref?
+    project.recalculate_tasks("Reference dataset inactivated: #{name}") if ref?
     pull_hook :on_inactivate
   end
 
@@ -117,7 +117,7 @@ class MiGA::Dataset < MiGA::MiGA
     metadata[:inactive] = nil
     metadata[:warn] = nil if metadata[:warn] && metadata[:warn] =~ /^Inactive: /
     metadata.save
-    project.recalculate_tasks('Reference dataset activated') if ref?
+    project.recalculate_tasks("Reference dataset activated: #{name}") if ref?
     pull_hook :on_activate
   end
 

data/lib/miga/project/dataset.rb

@@ -52,7 +52,9 @@ module MiGA::Project::Dataset
       @metadata[:datasets] << name
       @dataset_names_hash = nil # Ensure loading even if +do_not_save+ is true
       save
-      recalculate_tasks('New reference dataset added') if d.ref? && d.active?
+      if d.ref? && d.active?
+        recalculate_tasks("Reference dataset added: #{d.name}")
+      end
       pull_hook(:on_add_dataset, name)
     end
     dataset(name)
@@ -66,7 +68,9 @@ module MiGA::Project::Dataset
 
     self.metadata[:datasets].delete(name)
     save
-    recalculate_tasks('Reference dataset unlinked') if d.ref? && d.active?
+    if d.ref? && d.active?
+      recalculate_tasks("Reference dataset unlinked: #{d.name}")
+    end
     pull_hook(:on_unlink_dataset, name)
     d
   end

data/lib/miga/version.rb

@@ -12,7 +12,7 @@ module MiGA
   # - String indicating release status:
   #   - rc* release candidate, not released as gem
   #   - [0-9]+ stable release, released as gem
-  VERSION = [1.1, 1, 0].freeze
+  VERSION = [1.1, 2, 2].freeze
 
   ##
   # Nickname for the current major.minor version.
@@ -20,7 +20,7 @@ module MiGA
 
   ##
   # Date of the current gem relese.
-  VERSION_DATE = Date.new(2021, 10, 29)
+  VERSION_DATE = Date.new(2021, 11, 9)
 
   ##
   # References of MiGA

data/utils/FastAAI/FastAAI

@@ -583,6 +583,24 @@ class input_file:
 		self.initial_state = "protein+HMM"
 		
 		self.verbose = verbosity
+		
+		#r_scripts_loc = os.path.dirname(sys.modules['metapop'].__file__) + "/metapop_r/"
+		#"00.Libraries/01.SCG_HMMs/Complete_SCG_DB.hmm"
+		self.hmm_path = None
+		try:
+			#Try to locate the data bundled as it would be with a pip/conda install.
+			script_path = os.path.dirname(sys.modules['fastAAI_HMM_models'].__file__)
+			hmm_complete_model = script_path + '/00.Libraries/01.SCG_HMMs/Complete_SCG_DB.hmm'
+			self.hmm_path = str(hmm_complete_model)
+			#Check that the file exists or fail to the except.
+			fh = open(self.hmm_path)
+			fh.close()
+		except:
+			#Look in the same dir as the script; old method/MiGA friendly
+			script_path = Path(__file__)
+			script_dir = script_path.parent
+			hmm_complete_model = script_dir / "00.Libraries/01.SCG_HMMs/Complete_SCG_DB.hmm"
+			self.hmm_path = str(hmm_complete_model)
 	
 	#Functions for externally setting status and file paths of particular types
 	def set_genome(self, path):
@@ -2701,6 +2719,7 @@ def merge_db_thread_starter(rev_index, per_db_accs):
 	accs_per_db = per_db_accs
 	
 	
+	
 def merge_db(recipient, donors, verbose, threads):
 	#Prettier on the CLI
 
@@ -2828,111 +2847,109 @@ def merge_db(recipient, donors, verbose, threads):
 		
 	all_accessions = list(all_accessions)
 	
-	acc_args = [(acc, donor_dbs, recipient) for acc in all_accessions]
-	
-	if not os.path.exists("FastAAI_temp"):
-		os.mkdir("FastAAI_temp")
 	
 	print("")
 	print("Formatting data to add to database. Started at", curtime())
 	
-	if verbose:
-		print("")
-		count = 0
-		total_counts = len(acc_args)
-		try:
-			percentage = (count/total_counts)*100
-			sys.stdout.write("Completion".rjust(3)+ ' |'+('#'*int(percentage/2)).ljust(50)+'| ' + ('%.2f'%percentage).rjust(7)+'% ( ' + str(count) + " of " + str(total_counts) + ' done at ' + curtime() + " )\n")
-			sys.stdout.flush()			
-		except:
-			#It's not really a big deal if the progress bar cannot be printed.
-			pass
-	
-	last_pct = 0
-	
-	pool = multiprocessing.Pool(threads, initializer=merge_db_thread_starter, initargs = (reverse_genome_indices, accs_per_db,))
+	temp_dir = tempfile.mkdtemp()
+	try:
+		acc_args = [(acc, donor_dbs, recipient, temp_dir) for acc in all_accessions]
 	
-	quiverfull = []
-	for result in pool.imap_unordered(pull_and_merge_accession, acc_args):
-		acc = result[0]
-		child = result[1]
-		#sub_gak = result[2]
-		
-		quiverfull.append([acc, child])
-		#gaks.extend(sub_gak)
-		
 		if verbose:
-			count += 1
+			print("")
+			count = 0
+			total_counts = len(acc_args)
 			try:
 				percentage = (count/total_counts)*100
-				log_time = curtime()
-				sys.stdout.write('\033[A')
-				sys.stdout.flush()
 				sys.stdout.write("Completion".rjust(3)+ ' |'+('#'*int(percentage/2)).ljust(50)+'| ' + ('%.2f'%percentage).rjust(7)+'% ( ' + str(count) + " of " + str(total_counts) + ' done at ' + curtime() + " )\n")
 				sys.stdout.flush()			
 			except:
 				#It's not really a big deal if the progress bar cannot be printed.
 				pass
-	
-	pool.close()
-	pool.join()
-	
-	print("")
-	print("Adding data to final database. Started at", curtime())
-	
-	if verbose:
-		print("")
 		
-		count = 0
-		total_counts = len(acc_args)
-		try:
-			percentage = (count/total_counts)*100
-			sys.stdout.write("Completion".rjust(3)+ ' |'+('#'*int(percentage/2)).ljust(50)+'| ' + ('%.2f'%percentage).rjust(7)+'% ( ' + str(count) + " of " + str(total_counts) + ' done at ' + curtime() + " )\n")
-			sys.stdout.flush()
-		except:
-			#It's not really a big deal if the progress bar cannot be printed.
-			pass
-	
-	last_pct = 0
-	
-	recipient.activate_connection()
-	genome_list_update_sql = "INSERT OR REPLACE INTO genome_index VALUES (?, ?, ?)"
-	genome_reindex = []
-	for g in joint_genome_index:
-		genome_reindex.append((g, joint_genome_index[g], joint_genome_counts[joint_genome_index[g]]))
+		last_pct = 0
 		
-	recipient.cursor.executemany(genome_list_update_sql, genome_reindex)
-	recipient.connection.commit()
-	
-	del genome_reindex
-	
-	for result in quiverfull:
-		acc = result[0]
-		child = result[1]
+		pool = multiprocessing.Pool(threads, initializer=merge_db_thread_starter, initargs = (reverse_genome_indices, accs_per_db,))
 		
-		recipient.add_child_to_parent(acc, child, genomes_too = True, update_gak = True)
+		quiverfull = []
+		for result in pool.imap_unordered(pull_and_merge_accession, acc_args):
+			acc = result[0]
+			child = result[1]
+			#sub_gak = result[2]
+			
+			quiverfull.append([acc, child])
+			#gaks.extend(sub_gak)
+			
+			if verbose:
+				count += 1
+				try:
+					percentage = (count/total_counts)*100
+					log_time = curtime()
+					sys.stdout.write('\033[A')
+					sys.stdout.flush()
+					sys.stdout.write("Completion".rjust(3)+ ' |'+('#'*int(percentage/2)).ljust(50)+'| ' + ('%.2f'%percentage).rjust(7)+'% ( ' + str(count) + " of " + str(total_counts) + ' done at ' + curtime() + " )\n")
+					sys.stdout.flush()			
+				except:
+					#It's not really a big deal if the progress bar cannot be printed.
+					pass
+		
+		pool.close()
+		pool.join()
+		
+		print("")
+		print("Adding data to final database. Started at", curtime())
 		
 		if verbose:
-			count += 1
+			print("")
+			
+			count = 0
+			total_counts = len(acc_args)
 			try:
 				percentage = (count/total_counts)*100
-				log_time = curtime()
-				sys.stdout.write('\033[A')
-				sys.stdout.flush()
 				sys.stdout.write("Completion".rjust(3)+ ' |'+('#'*int(percentage/2)).ljust(50)+'| ' + ('%.2f'%percentage).rjust(7)+'% ( ' + str(count) + " of " + str(total_counts) + ' done at ' + curtime() + " )\n")
 				sys.stdout.flush()
 			except:
 				#It's not really a big deal if the progress bar cannot be printed.
 				pass
-	
-	os.rmdir("FastAAI_temp")
-	''' We're only ever increasing the DB size, so we don't actually need to vacuum it.
-	if recip_check != "created":
-		print("")
-		print("Cleaning up the database after the update. This may take a while.")
-		recipient.connection.execute("VACUUM")
-		recipient.connection.close()
-	'''
+		
+		last_pct = 0
+		
+		recipient.activate_connection()
+		genome_list_update_sql = "INSERT OR REPLACE INTO genome_index VALUES (?, ?, ?)"
+		genome_reindex = []
+		for g in joint_genome_index:
+			genome_reindex.append((g, joint_genome_index[g], joint_genome_counts[joint_genome_index[g]]))
+			
+		recipient.cursor.executemany(genome_list_update_sql, genome_reindex)
+		recipient.connection.commit()
+		
+		del genome_reindex
+		
+		for result in quiverfull:
+			acc = result[0]
+			child = result[1]
+			
+			recipient.add_child_to_parent(acc, child, genomes_too = True, update_gak = True)
+			
+			if verbose:
+				count += 1
+				try:
+					percentage = (count/total_counts)*100
+					log_time = curtime()
+					sys.stdout.write('\033[A')
+					sys.stdout.flush()
+					sys.stdout.write("Completion".rjust(3)+ ' |'+('#'*int(percentage/2)).ljust(50)+'| ' + ('%.2f'%percentage).rjust(7)+'% ( ' + str(count) + " of " + str(total_counts) + ' done at ' + curtime() + " )\n")
+					sys.stdout.flush()
+				except:
+					#It's not really a big deal if the progress bar cannot be printed.
+					pass
+	except:
+		#Error
+		shutil.rmtree(temp_dir)
+	finally:
+		#Success
+		shutil.rmtree(temp_dir)
+		
 	print("\nDatabases merged!")
 	
 	return None
@@ -2947,14 +2964,14 @@ def pull_and_merge_accession(args):
 		accession_inverter[accession_index[acc]] = sql_friendly_accession
 	
 	#joint_genome_index, accession_index, accession_inverter, accs_per_db are global already.
-	acc, donor_dbs, recipient = args[0], args[1], args[2]
+	acc, donor_dbs, recipient, temp = args[0], args[1], args[2], args[3]
 	
 	acc_name = accession_inverter[acc]
 	acc_name_gens = acc_name + "_genomes"
 	
 	query_sql = "SELECT * FROM " + acc_name
 	
-	temp_db = fastaai_database("FastAAI_temp/"+acc_name+".db")
+	temp_db = fastaai_database(os.path.normpath(temp+"/"+acc_name+".db"))
 	temp_db.activate_connection()
 	
 	create_command = "CREATE TABLE IF NOT EXISTS " + acc_name + " (kmer INTEGER PRIMARY KEY, genomes array)"
@@ -3627,4 +3644,4 @@ def main():
 if __name__ == "__main__":
 	main()
 
-	
+	

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 1.1.1.0
+  version: 1.1.2.2
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-29 00:00:00.000000000 Z
+date: 2021-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons