seqtrimnext 2.0.29

data/lib/seqtrimnext/plugins/plugin_low_quality.rb

@@ -0,0 +1,394 @@
+require "plugin"  
+
+########################################################
+# Author: Almudena Bocinos Rioboo                      
+# 
+# Defines the main methods that are necessary to execute PluginLowQuality. See the main method called execute.
+
+#                                                     
+# Inherit: Plugin
+########################################################
+
+class PluginLowQuality < Plugin   
+  
+    
+      
+      def create_sum_window(qual,ini,index_window_end)
+      
+        # puts "--------index w #{index_window_end}" 
+        sum=[] 
+        i=ini  
+        # puts "#{i} #{index_window_end}"
+        while (i<=index_window_end) # initialize sum                                  
+          sum[i]=0 
+          i += 1
+        end
+        # puts " contenido de sum" + sum.join.to_s  + " i index_window_end  window #{i} #{index_window_end} #{@window}" 
+      
+        i=ini
+        while (i<ini+@window)
+      
+          sum[ini] += qual[i] 
+          i+=1
+        end                                           
+      
+      
+        i=ini+1 
+      
+        while (i<=index_window_end)            
+      
+          sum[i]=sum[i-1]-qual[i-1]+qual[i+@window-1]
+          i+=1
+      
+        end   
+      
+        # puts '2____' + sum.join(',') + 'pos sum' + ini.to_s    
+      
+        return sum 
+      
+      end   
+      
+      def find_bounds_high_quality(sum,ini,index_window_end) 
+      
+        new_start = -1
+        new_end = -1
+        
+      # puts " ini #{ini} iwe #{index_window_end}"
+      # puts "ini #{ini} index_window_end #{index_window_end} sum[ini] #{sum[ini]} cut_off #{@cut_off} suma #{sum.size} " 
+       if (ini>index_window_end) 
+           temp_start= ini
+           # new_start, new_end = temp_start, index_window_end 
+           new_end = index_window_end # para que no crea que no hay alta calidad, sino que hemos sobrepasado el indice final de la ventana
+             # new_start, new_end = index_window_end, index_window_end 
+       end   
+      # puts " temp_start #{temp_start}" if (ini>index_window_end)
+      temp_start=((ini<=index_window_end) && (sum[ini]>=@cut_off))? ini : -1    
+      
+        i=ini+1
+        while (i<=index_window_end)
+          if (sum[i]>=@cut_off)  
+            if (temp_start<0)
+               temp_start=i  #just in! 
+               # puts "just in ---- #{sum[i]}>= cut off #{@cut_off} pos #{temp_start}"   
+            end
+      
+          else 
+              # puts "sum #{sum[i]} < cut off "
+              if(temp_start>=0)              #just out!   
+                # puts "update #{sum[i]}< cut off #{@cut_off} pos #{i}.if #{i-1} - #{temp_start} > #{new_end} - #{new_start}"
+                if (((i-1-temp_start)>=(new_end-new_start)))   
+                  new_start,new_end=temp_start,i-1 
+                  # puts "just out ---- new start,new_end = #{temp_start}, #{i-1}  index_window_end = #{index_window_end}"   
+                end
+                temp_start= -1 
+              end
+          end
+          i+=1  
+      
+      
+        end 
+        # puts "4 temp_start #{temp_start} new_start #{new_start} new-end #{new_end}"  
+      
+        if (temp_start != -1)   # finished while ok           
+          # puts "4 #{index_window_end} - #{temp_start} > #{new_end} - #{new_start}"
+            if ((index_window_end- temp_start) >= (new_end-new_start)) #put the end of the window at the end of sequence
+                new_start, new_end = temp_start, index_window_end     #-1
+            end
+        end  
+      
+        # puts "5 temp_start #{temp_start} new_start #{new_start} new-end #{new_end}"   
+        
+        # puts  " newstart  #{new_start} newend #{new_end}" 
+       
+        return new_start,new_end 
+       
+      
+      end  
+      
+      def cut_fine_bounds_short(qual,new_start,new_end)
+      
+          i=0                    
+          # puts " qual[new_start+i] new_start #{new_start} i #{i} = #{new_start+i} qual.size #{qual.size}"
+          while (i<@window)
+            if (qual[new_start+i]>=@low)
+              break
+            end    
+            i+=1
+          end  
+          new_start +=i 
+          # puts "#{new_start} ***********"
+      
+          i=@window -1
+          while (i>=0)  
+            if (qual[new_end+i]>=@low)    
+              break            
+            end
+            i-=1            
+          end     
+          new_end += i
+          # puts "6a new_start #{new_start} new-end #{new_end}"     
+          
+           # puts " #{new_start} #{new_end} .o.o.o.o.o.o.o.o2 short"    
+          return new_start, new_end  
+      
+      end  
+      
+      
+      # cuts fine the high quality bounds
+      def cut_fine_bounds(qual,new_start,new_end)   
+        # puts "  ççççççççççççççç #{new_start+@window} >= #{new_end} " 
+        # puts " #{new_start} #{new_end} .o.o.o.o.o.o.o.o1"
+        # cut it fine
+      
+         one_ok = 0         
+      
+          i=@window-1
+          # puts " qual[new_start+i] new_start #{new_start} i #{i} = #{new_start+i} qual.size #{qual.size}"
+          while (i>=0) 
+              if (qual[new_start+i] < @low) 
+                  break if one_ok
+              else 
+                  one_ok = 1
+              end    
+              i-=1
+          end
+          new_start += i+1
+          oneOk = 0  
+          i=0
+          while (i<@window) 
+              if (qual[new_end+i] < @low) 
+                  break if oneOk
+              else 
+                  oneOk = 1
+              end  
+              i+=1
+          end
+          new_end += i-1 
+          # puts "6b  new_start #{new_start} new-end #{new_end}"  
+      
+        # puts " #{new_start} #{new_end} .o.o.o.o.o.o.o.o2"
+        return new_start, new_end
+      
+      end
+      
+      def find_high_quality(qual,ini=0)  
+      
+        # puts qual.class.to_s + qual.size.to_s + 'size,' + @window.to_s + ' window, '+ qual.join(',')  + 'size' + qual.size.to_s
+        
+        update=false
+        # if @window>qual.length-ini     #search in the last window although has a low size
+        #     @window=qual.length-ini   
+        #      # puts ' UPDATE WINDOW  Y CUT OFF ' + @window.to_s
+        #      @cut_off=@window*@low   
+        #      update=true
+        #   end          
+                   
+        if (ini==0 or update)
+          #index_window_start = ini
+          @index_window_end = qual.size- @window #don't sub 1, or will lost the last nucleotide of the sequence -1;
+          #TODO En seqtrim de Juan iwe, que en nuestro seqtrim se llama index_window_end, está perdiendo 2 nucleótidos de la última ventana calculada 
+      
+      
+          @sum = create_sum_window(qual,ini,@index_window_end) 
+          # puts "SUMA #{@sum.join(' ')}"   
+        end              
+              
+        new_start, new_end = find_bounds_high_quality(@sum,ini,@index_window_end) 
+        # puts " #{new_start} #{new_end} .o.o.o.o.o.o.o.o1"
+      
+        if (new_start>=0)
+          if (new_start+@window >= new_end)
+             # puts "cfs"     
+            new_start, new_end = cut_fine_bounds_short(qual,new_start,new_end)
+            # puts "cfs"
+      
+          else  
+            # puts "cf"
+            new_start, new_end = cut_fine_bounds(qual,new_start,new_end) 
+            # puts "cf"
+          end 
+        end 
+        
+         # puts " #{new_start} #{new_end} .o.o.o.o.o.o.o.o2" 
+      
+        return new_start,new_end #+1
+      
+      
+      end
+      
+      
+      def add_action_before_high_qual(p_begin,p_end,actions,seq,start)
+      
+        action_size = p_begin-1
+        if action_size>=(@window/2)  
+      
+      
+          # puts "action_SIZE1 #{action_size} > #{@window/2}"
+      
+          if ( (p_begin>0) && (action_size>0) )  #if there is action before the high qual part 
+            # it's created an action before of the high quality part
+            a = seq.new_action(start ,p_begin-1,"ActionLowQuality") # adds the ActionInsert to the sequence before adding the actionMid
+            # puts " new low qual start: #{start}  = #{a.start_pos} end: #{p_begin} -1 = #{a.end_pos}"
+            actions.push a   
+          end 
+        end             
+      end  
+      
+      def add_action_after_high_qual(p_begin,p_end,actions,seq)
+      
+        action_size = seq.insert_end-p_end
+        if action_size>=(@window/2)
+      
+      
+           # puts "action_SIZE2 #{action_size} > #{@window/2}"
+      
+           if ((p_end<seq.seq_fasta.size-1) && (action_size>0) )  #if there is action before the high qual part 
+             # it's created an action before of the high quality part
+             a = seq.new_action(p_end-seq.insert_start+1,seq.seq_fasta.size-1,"ActionLowQuality") # adds the ActionInsert to the sequence before adding the actionMid
+      
+             actions.push a   
+           end 
+         end 
+      end
+       
+                                                                                   
+
+
+  
+  ######################################################################
+  #---------------------------------------------------------------------
+ 
+  # Begins the plugin1's execution whit the sequence "seq"
+  # Creates an action by each subsequence with low quality to eliminate it 
+  # A subsequence has low quality if (the add of all its qualitis < subsequence_size*20)  
+  # Creates the qualities windows from the sequence, looks for the subsequence with high quality 
+  # and mark, with an action, the before part to the High Quality Subsequence like a low quality part 
+  # Finally  mark, with an action, the after part to the High Quality Subsequence like a low quality part 
+  #----------------------------------------------------------------- 
+   
+ def execute(seqs)
+   seqs.each do |s|
+       exec_seq(s)
+    end
+  end
+
+
+  def exec_seq(seq)
+
+     if ((self.class.to_s=='PluginLowQuality') && seq.seq_qual.nil? ) 
+       $LOG.error " Quality File haven't been provided. It's impossible to execute " + self.class.to_s     
+     elsif (seq.seq_qual.size>0)
+       $LOG.info "[#{self.class.to_s}, seq: #{seq.seq_name}]: checking low quality of the sequence"    
+
+       @low=@params.get_param('min_quality').to_i
+
+       if @params.get_param('window_width').to_i>seq.seq_fasta.length   
+         @window=seq.seq_fasta.length   
+       
+       else 
+         @window=@params.get_param('window_width').to_i  
+       
+       end 
+       @cut_off=@window*@low   
+                                           
+       type='ActionLowQuality' 
+       low_qual=0
+       actions=[] 
+       
+       p_begin,p_end =0,-1 # positions from high quality bounds    
+                                
+       # @stats[:low_qual]={}    
+       # @stats['low_qual']={} 
+       
+       
+       while ((p_begin>=0)  && (p_end + 1 < seq.seq_qual.size) ) 
+         
+         
+         p_begin_old,p_end_old= p_begin, p_end
+         p_begin,p_end = find_high_quality(seq.seq_qual,p_end+1)  
+         # entra=(p_begin>0) or (p_end_old<0)
+         #         
+         # puts "high ini fin #{p_begin} #{p_end} ini-old fin-old  #{p_begin_old} #{p_end_old} __ ___ ___ ___1"     
+         
+         if ((p_begin>0) && (p_begin-p_end_old-1>=@window/2)) #if we have found the high quality part, and  the low quality part has enough size 
+            # it's created an action before of the high quality part 
+            add_action_before_high_qual(p_begin,p_end,actions,seq,p_end_old+1) 
+            # puts "low1 ini fin  #{p_end_old+1} #{p_begin-1} = #{p_begin-1-p_end_old-1+1}"     
+            low_qual = p_begin-1-p_end_old-1 + 1 
+            
+            add_stats('low_qual',low_qual)
+            # @stats[:low_qual]={low_qual => 1} 
+             
+         end
+         
+         # puts "-----ññññ----- high quality  #{p_begin}   #{p_end}+#{seq.insert_start} seq size #{seq.seq_fasta.size}"   
+
+       end
+        
+       # puts "high [#{p_begin}, #{p_end}] old [#{p_begin_old}, #{p_end_old}] size #{seq.seq_qual.size}"   
+       if ((p_begin>=0) && (p_end+1<seq.seq_qual.size))  #if we have found the high quality part 
+          
+          # it's created an action after of the high quality part     
+          add_action_after_high_qual(p_begin,p_end,actions,seq) 
+          # puts "low2 ini fin #{p_end+1} #{seq.seq_fasta.size-1}  = #{seq.seq_fasta.size-1-p_end-1+1}"
+          low_qual = seq.seq_fasta.size-1 - p_end-seq.insert_start-1 + 1
+          # if @stats[:low_qual][low_qual].nil?
+          #              @stats[:low_qual][low_qual] = 0
+          #           end
+          #           @stats[:low_qual][low_qual] += 1  
+          add_stats('low_qual',low_qual) 
+          # @stats[:low_qual]={low_qual => 1}    
+       end                                     
+
+       # puts "-----ññññ----- high quality  #{p_begin}   #{p_end}"  
+   
+       
+       if p_end<0 and p_end_old #add action low qual to all the part      
+         a = seq.new_action(p_end_old+1 ,seq.seq_fasta.size-1,"ActionLowQuality") # adds the ActionInsert to the sequence before adding the actionMid
+         # puts "new low qual start: #{p_end_old+1} end: #{seq.seq_fasta.size-1} = #{seq.seq_fasta.size-1 - p_end_old-1 + 1}" 
+         low_qual = seq.seq_fasta.size-1 - p_end_old-1 + 1 
+         
+          # if @stats[:low_qual][low_qual].nil?
+          #             @stats[:low_qual][low_qual] = 0
+          #          end
+          #          @stats[:low_qual][low_qual] += 1   
+         add_stats('low_qual',low_qual) 
+         # @stats[:low_qual]={'low_qual' => 1} 
+          
+         actions.push a
+       end
+       
+       # puts "------- ADDING ACTIONs LOW QUAL #{actions.size}"    
+       seq.add_actions(actions)
+     end       
+
+   end 
+   
+  #-----------------------------------------------------------------
+  
+  
+  ######################################################################
+  #---------------------------------------------------------------------
+
+  #Returns an array with the errors due to parameters are missing 
+  def self.check_params(params) 
+    
+    errors=[]
+    
+   	comment='Minimum quality value for every nucleotide'
+		default_value = 20
+		params.check_param(errors,'min_quality','Integer',default_value,comment)
+		
+   	comment='Quality window for scanning low quality segments'
+		default_value = 15
+		params.check_param(errors,'window_width','Integer',default_value,comment)
+   
+
+    
+    return errors
+  end
+  
+  
+  private :find_high_quality
+  
+end

data/lib/seqtrimnext/plugins/plugin_mids.rb

@@ -0,0 +1,231 @@
+require "plugin"
+require 'recover_mid'
+include RecoverMid
+
+########################################################
+# Author: Almudena Bocinos Rioboo                      
+# 
+# Defines the main methods that are necessary to execute PluginMids                                                     
+# Inherit: Plugin
+########################################################
+
+class PluginMids < Plugin   
+  SIZE_SEARCH_MID=20  
+  MAX_MID_ERRORS = 2
+  #MIN_MID_SIZE = 7  # very important, don't touch 
+  # DB_MID_SIZE = 10  # DONE read formatted db and save the mid sizes    
+            
+  
+  
+  #Begins the plugin1's execution to warn that there are contaminants in the sequence "seq"    
+  def execute(seqs)
+    blasts= do_blasts(seqs)
+    
+    seqs.each_with_index do |s,i|
+      exec_seq(s,blasts.querys[i])
+    end
+  end
+  
+  def do_blasts(seqs)
+     # find MIDS  with less results than max_target_seqs value 
+     blast = BatchBlast.new("-db #{@params.get_param('mids_db')}",'blastn'," -task blastn-short    -perc_identity #{@params.get_param('blast_percent_mids')} -max_target_seqs 4 ")  #get mids 
+     $LOG.info('BLAST:'+blast.get_blast_cmd)
+
+     fastas=[]
+     
+     seqs.each do |seq|
+      fastas.push ">"+seq.seq_name
+      fastas.push seq.seq_fasta[0..SIZE_SEARCH_MID]
+     end
+     
+     # fastas=fastas.join("\n")
+     
+     blast_table_results = blast.do_blast(fastas)
+     
+     # puts blast_table_results.inspect
+    
+     return blast_table_results
+  end
+  
+  
+  def exec_seq(seq,blast_query)
+    if blast_query.query_id != seq.seq_name
+      raise "Blast and seq names does not match, blast:#{blast_query.query_id} sn:#{seq.seq_name}"
+    end
+
+     
+     $LOG.info "[#{self.class.to_s}, seq: #{seq.seq_name}]: looking for mids into the sequence"   
+      
+
+     # blast_table_results = blast.do_blast(seq.seq_fasta[0..SIZE_SEARCH_MID])             # execute blast to find mids
+     # blast_table_results.inspect
+     
+     actions=[]
+     file_tag='no_MID'
+     
+     key_size=0
+     mid_size=0
+     
+     key_already_found=!seq.get_actions(ActionKey).empty?
+     
+     mid_errors=[]    #number of MIDs with 1 error, and number of MIDs with 2 errors
+     mid_id=[] #number of MIDs from each type
+     mid_found = false
+     
+     if !blast_query.hits.empty? # mid found 
+       
+       # blast_query.hits.sort!{|h1,h2| h1.q_beg <=> h2.q_beg}
+       # puts blast_query.count.to_s + "============== #{blast_query.hits[0].inspect}"
+       # blast_table_results.inspect
+                                  
+       # select first sorted mid
+       mid=blast_query.hits[0]
+       
+       # find a not reversed mid
+       if mid.reversed
+         
+         blast_query.hits.each do |hit|
+          if !hit.reversed # take the first non-reversed one
+             mid = hit
+             break
+          end
+         end
+
+       end
+       
+       # puts "DOES THE MID HAVE ENOUGHT SIZE? #{mid.q_end-mid.q_beg+1} >= #{MIN_MID_SIZE}?"
+       mid_size=mid.q_end-mid.q_beg+1
+       
+       db_mid=@params.get_mid(mid.subject_id)
+       db_mid_size = db_mid.size  #get mid's size from DB   
+       
+       mid_initial_pos=mid.q_beg-mid.s_beg
+       has_full_key=false
+       if @params.get_param('sequencing_key')
+       	has_full_key = !seq.seq_fasta.index(@params.get_param('sequencing_key')).nil?
+       end
+       
+       if mid.reversed
+       	  # discard mid
+       elsif (mid.gaps+mid.mismatches > MAX_MID_ERRORS) # number of ERRORS and GAPs is higher than MAX_MID_ERRORS, 
+          # discard mid
+       elsif (mid.q_beg<3) # if found mid starts below 3, then discard it 
+         # discard mid
+       elsif (has_full_key && (mid_initial_pos >=6))
+         # discard mid
+       elsif (!has_full_key && (mid_initial_pos >=7))
+         # discard mid
+       elsif (mid_size >= db_mid_size-1)  # MID found and MID's size is enought, THEN create key and mid
+ 			 		 
+		       key_beg,key_end=[0,mid.q_beg-1]
+		       key_size=mid.q_beg
+		      
+		       # Create an ActionKey before the ActionMid
+		       if key_size>0 && !key_already_found                                    
+		         a = seq.new_action(key_beg,key_end,"ActionKey") # adds the actionKey to the sequence
+		         actions.push a 
+		       end      
+         
+ 					 #Create an ActionMid 
+           a = seq.new_action(mid.q_beg,mid.q_end,"ActionMid") # adds the ActionMids to the sequence   
+           a.message = mid.subject_id
+           a.tag_id = mid.subject_id
+           file_tag = mid.subject_id
+           actions.push a
+         
+           mid_found = true
+         
+      elsif (mid_size >= db_mid_size-3)
+        # To recover a MID it must start or end in one edge
+        if (mid.s_beg==0) || (mid.s_end==mid_size)
+
+					new_q_beg, new_q_end, recovered_size,recovered_mid = recover_mid(mid, db_mid, seq.seq_fasta[0..SIZE_SEARCH_MID])
+					
+					$LOG.debug("Recover mid: #{recovered_mid} valid (#{recovered_size} >= #{10-1}) = #{recovered_size>=10-1}, #{seq.seq_fasta[new_q_beg..new_q_end]}")
+
+					if recovered_size >= db_mid_size-1
+						mid_size = recovered_size
+				
+						# if MID found and MID's size is enought to recover a MID, THEN create an action key and mid 
+						key_beg,key_end=[0,new_q_beg-1]
+						key_size=new_q_beg
+					
+						$LOG.debug "RECOVER OUTPUT: #{new_q_beg} #{new_q_end} #{recovered_size}"
+
+						#  if key_size > 4(or max_size_key) then seq.seq_rejected 
+
+						# Create an ActionKey before the ActionMid
+						if key_size>0 && !key_already_found
+							a = seq.new_action(key_beg,key_end,"ActionKey") # adds the actionKey to the sequence
+							actions.push a 
+						end      
+
+						#Create an ActionMid to a recovered mid
+						a = seq.new_action(new_q_beg,new_q_end,"ActionMid") # adds the ActionMids to the sequence 
+						a.message = "Recovered " + mid.subject_id
+						a.tag_id = mid.subject_id
+            file_tag = mid.subject_id
+						actions.push a
+						add_stats('recovered_mid_id',mid.subject_id)
+
+						mid_found = true
+			    end
+		    end
+		  end
+    end
+     
+    if !mid_found && !key_already_found # MID not found, take only the key
+       mid_size=0
+       key_beg,key_end=[0,3]
+       key_size=4
+       a = seq.new_action(key_beg,key_end,'ActionKey') # adds the actionKey to the sequence
+       actions.push a   
+     end 
+     
+     #Add actions  
+     seq.add_actions(actions)
+     
+     seq.add_file_tag(1, file_tag, :both)
+     # seq.add_file_tag(2,'sequence')
+     
+     if (mid_found) # MID without errors
+      
+       add_stats('mid_id',mid.subject_id)
+       add_stats('mid_id','total')
+
+       #save MID count by ID
+       add_stats(mid.subject_id,mid_size)
+      
+      	if (mid.gaps+mid.mismatches > 0)
+				add_stats('mid_with_errors',mid.gaps+mid.mismatches)
+      	end
+       
+     end
+     
+     if !key_already_found
+       add_stats('key_size',key_size)
+       add_stats('mid_size',mid_size)
+     end
+  end
+ 
+  #Returns an array with the errors due to parameters are missing 
+  def self.check_params(params)
+    errors=[]
+    
+    	comment='Blast E-value used as cut-off when searching for MIDs'
+		default_value = 1e-10
+		params.check_param(errors,'blast_evalue_mids','Float',default_value,comment)
+		
+		comment='Minimum required identity (%) for a reliable MID'
+		default_value = 95
+		params.check_param(errors,'blast_percent_mids','Integer',default_value,comment)
+    
+    comment='Path for MID database'
+		default_value = File.join($FORMATTED_DB_PATH,'mids.fasta')
+		params.check_param(errors,'mids_db','DB',default_value,comment)
+
+    return errors
+  end
+  
+  
+end