seqtrimnext 2.0.51 → 2.0.52

data/lib/seqtrimnext/classes/extract_stats.rb

@@ -11,20 +11,20 @@ require "scbi_math"
 
 class ExtractStats
 
-  def initialize(sequence_reader,params)
+  def initialize(sequence_readers,params)
 
     @sequence_lengths = []         #array of sequences lengths
     @length_frequency = []      #number of sequences of each size (frequency)
     @keys={}  #found keys
     @params = params
-    @use_qual=sequence_reader.with_qual?
+    @use_qual=sequence_readers.first.with_qual?
     # @params.get_param('use_qual')
 
     @totalnt=0
     @qv=[]
 
 
-    @sequence_lengths_stats, @length_frequency_stats, @quality_stats = extract_stats_from_sequences(sequence_reader)
+    @sequence_lengths_stats, @length_frequency_stats, @quality_stats = extract_stats_from_sequences(sequence_readers)
 
 
     set_params_and_results
@@ -37,30 +37,33 @@ class ExtractStats
 
   end
 
-  def extract_stats_from_sequences(sequence_reader)
+  def extract_stats_from_sequences(sequence_readers)
+    sequence_readers.each do |sequence_reader|
 
-    sequence_reader.each do |name_seq,fasta_seq,qual|
-      l = fasta_seq.length
 
-      @totalnt+=l
+      sequence_reader.each do |name_seq,fasta_seq,qual|
+        l = fasta_seq.length
 
-      #save all lengths
-      @sequence_lengths.push l
+        @totalnt+=l
 
-      # add key value
-      add_key(fasta_seq[0..3].upcase)
+        #save all lengths
+        @sequence_lengths.push l
 
-      # add fasta length
-      @length_frequency[fasta_seq.length] = (@length_frequency[fasta_seq.length] || 1 ) + 1
+        # add key value
+        add_key(fasta_seq[0..3].upcase)
 
-      #extract qv values
-      extract_qv_from_sequence(qual) if @use_qual
+        # add fasta length
+        @length_frequency[fasta_seq.length] = (@length_frequency[fasta_seq.length] || 1 ) + 1
 
-      # print some progress info
-      if (sequence_reader.num_seqs % 10000==0)
-        puts "Calculating stats: #{sequence_reader.num_seqs}"
-      end
+        #extract qv values
+        extract_qv_from_sequence(qual) if @use_qual
+
+        # print some progress info
+        if (sequence_reader.num_seqs % 10000==0)
+          puts "Calculating stats: #{sequence_reader.num_seqs}"
+        end
 
+      end
     end
 
     length_stats = ScbiNArray.to_na(@sequence_lengths)
@@ -74,9 +77,9 @@ class ExtractStats
 
     ## PLOT RESULTS
     if !File.exists?('graphs')
-    	Dir.mkdir('graphs')
+      Dir.mkdir('graphs')
     end
-    
+
 
     x = []
     y = []
@@ -107,10 +110,10 @@ class ExtractStats
   def plot_qualities
 
     if !File.exists?('graphs')
-    	Dir.mkdir('graphs')
+      Dir.mkdir('graphs')
     end
     minimum_qual_value = @params.get_param('min_quality').to_i
-    
+
     # get qualities values
     x=[]
     y=[]

data/lib/seqtrimnext/classes/params.rb

@@ -127,7 +127,7 @@ class Params
 
         # line doesn't finish in *
         if (line[0]!='>'[0]) && (!(line =~ /\*$/))
-          
+
           #puts line
           # puts line,line[0]
           if line =~ />([^\.]+)\.\.\.\s/
@@ -189,12 +189,12 @@ class Params
 
   def get_fasta(list,name,type)
     res = list[name]
-    
+
     if res.nil?
       $LOG.error("Error. The #{type}: #{name} was not correctly loaded")
       raise "Error. The #{type}: #{name} was not found in loaded #{name}s: #{list.map{|k,v| k}}."
     end
-    
+
     return res
   end
 
@@ -231,165 +231,151 @@ class Params
       line = Regexp.last_match[2].to_i
       method = Regexp.last_match[3]
       plugin=File.basename(file,File.extname(file))
-      
-      # puts "CALLER: #{plugin}"
-      # puts [file, line, method]
 
     end
     
   end
   
-  def set_param(param,value,comment=nil)
-                         plugin=get_plugin
+  def set_param(param,value,comment = nil)
+     plugin=get_plugin
 
-                         @params[param] = value
+     @params[param] = value
 
-                         if get_comment(plugin,param).nil?
-                           set_comment(plugin,param,comment)
-                         end
+     if get_comment(plugin,param).nil?
+       set_comment(plugin,param,comment)
+     end
 
 
-                       end
-  
-  # def set_order(plugin,param)
-  #      
-  #      if @param_order[plugin].nil?
-  #        @param_order[plugin]=[]
-  #    end
-  #    
-  #    if !@param_order[plugin].index(param)
-  #      @param_order[plugin].push param
-  #    end
-  # end
-    
+   end
   
   def get_comment(plugin,param)
-                         res = nil
-                         if @plugin_comments[plugin]
-                           res =@plugin_comments[plugin][param]
-                         end
-                         return res
-                       end
+     res = nil
+     if @plugin_comments[plugin]
+       res =@plugin_comments[plugin][param]
+     end
+     return res
+   end
   
   
   def set_comment(plugin,param,comment)
-                         if !comment.is_a?(Array) && !comment.nil?
-                           comment=comment.split("\n").compact.map{|l| l.strip}
-                         end
+     if !comment.is_a?(Array) && !comment.nil?
+       comment=comment.split("\n").compact.map{|l| l.strip}
+     end
 
-                         if @plugin_comments[plugin].nil?
-                           @plugin_comments[plugin]={}
-                         end
+     if @plugin_comments[plugin].nil?
+       @plugin_comments[plugin]={}
+     end
 
-                         old_comment=''
-                         # remove from other plugins
-                         @plugin_comments.each do |plugin_name,comments|
-                           if comments.keys.include?(param) && plugin_name!=plugin
-                             old_comment=comments[param]
-                             comments.delete(param)
-                           end
-                         end
+     old_comment=''
+     # remove from other plugins
+     @plugin_comments.each do |plugin_name,comments|
+       if comments.keys.include?(param) && plugin_name!=plugin
+         old_comment=comments[param]
+         comments.delete(param)
+       end
+     end
 
-                         if comment.nil?
-                           comment=old_comment
-                         end
+     if comment.nil?
+       comment=old_comment
+     end
 
-                         # @comments[param]=(comment || [''])
-                         @plugin_comments[plugin][param]=(comment || [''])
-                         # puts @plugin_comments.keys.to_json
+     # @comments[param]=(comment || [''])
+     @plugin_comments[plugin][param]=(comment || [''])
+     # puts @plugin_comments.keys.to_json
 
-                         # remove empty comments
+     # remove empty comments
 
-                         @plugin_comments.reverse_each do |plugin_name,comments|
-                           if comments.empty?
-                             @plugin_comments.delete(plugin_name)
-                           end
-                         end
+     @plugin_comments.reverse_each do |plugin_name,comments|
+       if comments.empty?
+         @plugin_comments.delete(plugin_name)
+       end
+     end
+
+   end
 
-                       end
-       
   
   def set_mid(param,value)
-                         @mids[param] = value
-                       end
-  #attr_accessor :h  # to accede to the atribute 'h' from out of this class
+    @mids[param] = value
+  end
   
   # Returns true if exists the parameter and nil if don't
   def exists?(param_name)
-                         return !@params[param_name].nil?
-                       end
-  
+     return !@params[param_name].nil?
+   end
+ 
   def check_plugin_list_param(errors,param_name)
-                         # get plugin list
-                         pl_list=get_param(param_name)
+     # get plugin list
+     pl_list=get_param(param_name)
 
-                         # puts pl_list,param_name
-                         list=pl_list.split(',')
+     # puts pl_list,param_name
+     list=pl_list.split(',')
 
-                         list.map!{|e| e.strip}
+     list.map!{|e| e.strip}
 
-                         # puts "Lista:",list.join(',')
+     # puts "Lista:",list.join(',')
 
 
-                         # always the pluginExtractInserts at the end
-                         list.delete('PluginExtractInserts')
-                         list << 'PluginExtractInserts'
+     # always the pluginExtractInserts at the end
+     list.delete('PluginExtractInserts')
+     list << 'PluginExtractInserts'
 
-                         set_param(param_name,list.join(','))
-                         # if !list.include?('PluginExtractInserts')
-                         #   raise "PluginExtractInserts do not exists"
-                         #
-                         # end
+     set_param(param_name,list.join(','))
+     # if !list.include?('PluginExtractInserts')
+     #   raise "PluginExtractInserts do not exists"
+     #
+     # end
 
 
 
-                       end
+   end
 
   # def split_databases(db_param_name)
   def check_db_param(errors,db_param_name)
-                         # expand database paths
-                         dbs= get_param(db_param_name).gsub('"','').split(/\s+/)
-                         # puts "ALGO"*20
-                         # puts "INPUT DATABASES:\n"+dbs.join(',')
-
-                         procesed_dbs=[]
-                         #
-                         # TODO - chequear aqui que la db no esta vacia y que esta formateada.
-                         dbs.reverse_each {|db_p|
-                           db=File.expand_path(db_p)
-
-                           if !File.exists?(db)
-                             path=File.join($FORMATTED_DB_PATH,db_p)
-                           else
-                             path=db
-                           end
-
-
-                           if Dir.glob(path+'*.n*').entries.empty?
-                             puts "DB file #{path} not formatted"
-
-                             if File.writable_real?(path)
-                               cmd = "makeblastdb -in #{path} -parse_seqids -dbtype nucl"
-                               system(cmd)
-                             else
-                               raise "Can't format database. We don't have write permissions in: #{path}"
-                             end
-                           end
-
-                           procesed_dbs << path
-
-                           if !File.exists?(path)
-                             raise "DB File #{path} does not exists"
-                             # exit
-                           end
-                         }
-
-                         db_paths = '"'+procesed_dbs.join(' ')+'"'
-
-                         set_param(db_param_name,db_paths)
-
-                         puts "USED DATABASES\n"+db_paths
-                       end
+     if !get_param(db_param_name).empty?
+       # expand database paths
+       dbs= get_param(db_param_name).gsub('"','').split(/\s+/)
+       # puts "ALGO"*20
+       # puts "INPUT DATABASES:\n"+dbs.join(',')
+
+       procesed_dbs=[]
+       #
+       # TODO - chequear aqui que la db no esta vacia y que esta formateada.
+       dbs.reverse_each {|db_p|
+         db=File.expand_path(db_p)
+
+         if !File.exists?(db)
+           path=File.join($FORMATTED_DB_PATH,db_p)
+         else
+           path=db
+         end
+
+
+         if Dir.glob(path+'*.n*').entries.empty?
+           puts "DB file #{path} not formatted"
+
+           if File.writable_real?(path)
+             cmd = "makeblastdb -in #{path} -parse_seqids -dbtype nucl"
+             system(cmd)
+           else
+             raise "Can't format database. We don't have write permissions in: #{path}"
+           end
+         end
+
+         procesed_dbs << path
+
+         if !File.exists?(path)
+           raise "DB File #{path} does not exists"
+           # exit
+         end
+       }
+
+       db_paths = '"'+procesed_dbs.join(' ')+'"'
+
+       set_param(db_param_name,db_paths)
+
+       puts "USED DATABASES\n"+db_paths
+     end
+   end
   
   
   def self.generate_sample_params

data/lib/seqtrimnext/classes/plugin_manager.rb

@@ -29,7 +29,6 @@ class PluginManager
       
       # keeps a list of rejected sequences
       
-
       rejected_seqs = []
 
       @plugin_names.each do |plugin_name|
@@ -51,11 +50,10 @@ class PluginManager
         
         # Creates an instance of the respective plugin stored in "plugin_name",and asociate it to the sequence 'seq'
         plugin_class = Object.const_get(plugin_name)
-        # puts "ANTES *************" + seq.to_text_seq_fasta
+
         plugin_execution=plugin_class.new(running_seqs,@params)
-        #puts plugin_name+':'+ plugin_execution.stats.to_json
-        running_seqs.stats[plugin_name] = plugin_execution.stats
 
+        running_seqs.stats[plugin_name] = plugin_execution.stats
         
         # puts running_seqs.stats.to_json
         plugin_execution=nil

data/lib/seqtrimnext/classes/seqtrim.rb

@@ -175,22 +175,14 @@ class Seqtrim
     # it is the server part
   if !only_workers then
 
-    sequence_reader = nil
     cd_hit_input_file = nil
     
     # TODO - FIX seqtrim to not iterate two times over input, so STDIN can be used
-    
+    sequence_readers=[]
+
     # open sequence reader and expand input files paths
     if options[:fastq]
       
-      if options[:fastq]=='-'
-        seqs_path = STDIN
-      else
-        seqs_path = File.expand_path(options[:fastq])
-      end
-      
-      cd_hit_input_file = seqs_path
-      
       # choose fastq quality format
       format=:sanger
       
@@ -203,13 +195,23 @@ class Seqtrim
         format = :sanger
       end
       
+      seqs_path=''
+      
       $LOG.info("Used FastQ format for input files: #{format}")
+      # iterate files
+      options[:fastq].each do |fastq_file|
+        
+        if fastq_file=='-'
+          seqs_path = STDIN
+        else
+          seqs_path = File.expand_path(fastq_file)
+        end
+        
+        sequence_readers << FastqFile.new(seqs_path,'r',format, true)
+        
+      end
       
-      sequence_reader = FastqFile.new(seqs_path,'r',format, true)
-      # cd_hit_input_file = 'cd-hit-input.fasta'
       cd_hit_input_file = seqs_path
-      # $LOG.info "Converting input file for cd-hit-454"
-      # $LOG.info "Conversion done"
       
     else
 
@@ -217,7 +219,7 @@ class Seqtrim
       cd_hit_input_file = seqs_path
       
       qual_path =  File.expand_path(options[:qual]) if qual_path
-      sequence_reader = FastaQualFile.new(options[:fasta],options[:qual],true)
+      sequence_readers << FastaQualFile.new(options[:fasta],options[:qual],true)
 
     end
 
@@ -250,12 +252,11 @@ class Seqtrim
 	    	$LOG.error "Plugin check failed"
 
 	    	# save used params to file
-      params.save_file('used_params.txt')
+        params.save_file('used_params.txt')
 
       exit
     end
     
-    
     if !Dir.exists?(OUTPUT_PATH)
       Dir.mkdir(OUTPUT_PATH)
     end
@@ -263,8 +264,7 @@ class Seqtrim
     # Extract global stats
     if params.get_param('generate_initial_stats')=='true'
       $LOG.info "Calculatings stats"
-      
-      ExtractStats.new(sequence_reader,params)
+      ExtractStats.new(sequence_readers,params)
     else
       $LOG.info "Skipping calculatings stats phase."
     end
@@ -274,14 +274,6 @@ class Seqtrim
     params.save_file(File.join(OUTPUT_PATH,'used_params.txt'))
     
     piro_on = (params.get_param('next_generation_sequences')=='true')
-    
-      # format blast database with truncated file
-      #MakeBlastDb.format_db(es.truncated_file_path,File.basename(es.truncated_file_path,File.extname(es.truncated_file_path)),'./') if piro_on
-      
-      # leer mids
-      # params.load_mids(File.join($FORMATTED_DB_PATH,'mids.fasta'))
-      # params.load_ab_adapters(File.join($FORMATTED_DB_PATH,'adapters_ab.fasta'))
-      # params.load_linkers(File.join($FORMATTED_DB_PATH,'linkers.fasta'))
 
       params.load_mids(params.get_param('mids_db'))
       params.load_ab_adapters(params.get_param('adapters_ab_db'))
@@ -334,7 +326,7 @@ class Seqtrim
       else
   			$LOG.info 'Starting server'
       	        
- 				SeqtrimWorkManager.init_work_manager(sequence_reader, params,chunk_size,use_json,options[:skip_output])
+ 				SeqtrimWorkManager.init_work_manager(sequence_readers, params,chunk_size,use_json,options[:skip_output])
 				
         begin
   				cpus=1
@@ -371,7 +363,10 @@ class Seqtrim
 				server.start_server
         
         # close sequence reader
-				sequence_reader.close
+				sequence_readers.each do |file|
+				  file.close
+				end
+				
 				$LOG.info 'Closing server'
 			end
 			

data/lib/seqtrimnext/classes/sequence.rb

@@ -20,8 +20,8 @@ class Sequence
       
       @seq_rejected_by_message=''
       
-      @ns_present = ns_present? 
-      @xs_present = xs_present?    
+      @ns_present = ns_present?
+      @xs_present = xs_present?
       
       
       

data/lib/seqtrimnext/classes/sequence_group.rb

@@ -10,6 +10,7 @@ class SequenceGroup
     @seqs=seqs
     @output_text={}
     @output_files={}
+    
   end
   
   
@@ -31,6 +32,13 @@ class SequenceGroup
         yield seq
       end
   end
+  
+  def each_slice(n)
+      @seqs.each_slice(n) do |seqs|
+        yield seqs
+      end
+  end
+  
 
   def each_with_index
       @seqs.each_with_index do |seq,i|
@@ -46,7 +54,19 @@ class SequenceGroup
   end
   
   def add(array)
-    @seqs += array
+    @seqs = @seqs + array
+
+    # sort by tuple_id and order in tuple
+    @seqs.sort! do |a,b|
+      comp = (a.tuple_id <=> b.tuple_id)
+      comp.zero? ? (a.order_in_tuple <=> b.order_in_tuple) : comp
+    end
+
+    # print
+    # @seqs.each do |s|
+    #   puts "TID:#{s.tuple_id}, OIT: #{s.order_in_tuple}"
+    # end
+    
   end
   
   def count

data/lib/seqtrimnext/classes/sequence_with_action.rb

@@ -14,14 +14,13 @@ include Term::ANSIColor
 class SequenceWithAction < Sequence    
   SHOW_QUAL = false
   SHOW_FINAL_INSERTS=true
-  attr_accessor :actions,:seq_fasta_orig, :seq_qual_orig ,:insert_start , :insert_end, :stats , :insert_start_last , :insert_end_last
+
+  attr_accessor :actions,:seq_fasta_orig, :seq_qual_orig ,:insert_start , :insert_end, :stats , :insert_start_last , :insert_end_last, :order_in_tuple, :tuple_id, :tuple_size
     
   # Creates an instance with the structure to storage the actions associated to a sequence  
   def initialize(seq_name,seq_fasta,seq_qual, seq_comment = '')
     super
-    #Tried
-    #if @ns_present then $LOG.debug "The sequence #{seq_name} has N's" else $LOG.debug "The sequence #{seq_name} hasn't N's" end
-    #if @xs_present then $LOG.debug "The sequence #{seq_name} has X's" else $LOG.debug "The sequence #{seq_name} hasn't X's" end
+
     @actions = []
     @seq_fasta_orig = seq_fasta
     @seq_fasta = seq_fasta
@@ -32,18 +31,31 @@ class SequenceWithAction < Sequence
     @insert_start = 0
     @insert_end = seq_fasta.length-1 
     
-    
-    #@seq_qual = seq_qual
-    
     @stats={}
+    @comments=[]
     
     @file_tags=[]
     
+    # for paired ends
+    @order_in_tuple=0
+    @tuple_id=0
+    @tuple_size=0
+    @file_tag_tuple_priority=0
+    
+  end
+  
+  def add_comment(comment)
+    @comments.push comment
+  end
+  
+  def get_comment_line
+    return ([@seq_rejected_by_message]+@comments).compact.join(';')
   end
   
   # add a file tag to sequence
-  def add_file_tag(tag_level, tag_value, tag_type)
+  def add_file_tag(tag_level, tag_value, tag_type, priority=0)
     @file_tags<< {:level => tag_level, :name => tag_value, :type=> tag_type}
+    @file_tag_tuple_priority=priority
   end
   
   # join file tags into a path
@@ -71,7 +83,7 @@ class SequenceWithAction < Sequence
     
     # puts "#{dirname}, #{filename}"
     
-    return [dirname,filename]
+    return [dirname,filename,@file_tag_tuple_priority]
     
   end
   
@@ -328,16 +340,16 @@ class SequenceWithAction < Sequence
       output_res=[]
       
       if @seq_rejected
-        output_res<< " Sequence #{seq_name} had the next actions: ".bold.underline + " REJECTED:  #{@seq_rejected_by_message}".red    
+        output_res<< "[#{@tuple_id},#{@order_in_tuple}] Sequence #{seq_name} had the next actions: ".bold.underline + " REJECTED:  #{@seq_rejected_by_message}".red    
         # puts  @seq_name.bold + bold + ' REJECTED BECAUSE ' +@seq_rejected_by_message.bold if @seq_rejected 
       else
-        output_res<< " Sequence #{seq_name} had the next actions: ".bold.underline  
+        output_res<< "[#{@tuple_id},#{@order_in_tuple}] Sequence #{seq_name} had the next actions: ".bold.underline  
         
-      end    
+      end
       
       n=1
       withMessage = ["ActionIsContaminated","ActionVectors","ActionBadAdapter","ActionLeftAdapter","ActionRightAdapter"]  
-      color = red                      
+      color = red
       
       @actions.sort!{|e,f| e.start_pos<=>f.start_pos}.each do |a|     
         a_type=a.action_type