seqtrimnext 2.0.29

data/lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb

@@ -0,0 +1,290 @@
+
+#finds the classes that were in the folder 'classes'
+
+ROOT_PATH=File.dirname(File.dirname(File.dirname(__FILE__)))
+
+$: << File.expand_path(File.join(ROOT_PATH, 'classes'))
+$: << File.expand_path(File.join(ROOT_PATH, 'classes','blast'))
+
+#finds the classes that were in the folder 'plugins'
+$: << File.expand_path(File.join(ROOT_PATH, 'plugins'))
+
+#finds the classes that were in the folder 'plugins'
+$: << File.expand_path(File.join(ROOT_PATH, 'actions'))
+
+#finds the classes that were in the folder 'utils'
+$: << File.expand_path(File.join(ROOT_PATH, 'utils'))
+
+$: << File.expand_path(File.join(ROOT_PATH, 'classes','em_classes'))
+
+$: << File.expand_path(ROOT_PATH)
+
+$SEQTRIM_PATH = ROOT_PATH
+
+if ENV['BLASTDB']# && Dir.exists?(ENV['BLASTDB'])
+  $FORMATTED_DB_PATH = ENV['BLASTDB']
+  $DB_PATH = File.dirname($FORMATTED_DB_PATH)
+else
+  $FORMATTED_DB_PATH = File.expand_path(File.join(ROOT_PATH, "DB",'formatted'))
+  $DB_PATH = File.expand_path(File.join(ROOT_PATH, "DB"))  
+end
+
+ENV['BLASTDB']=$FORMATTED_DB_PATH
+
+OUTPUT_PATH='output_files'
+
+
+# $: << File.expand_path('~/progs/ruby/gems/scbi_mapreduce/lib')
+
+require 'scbi_mapreduce'
+require 'params'
+require 'action_manager'
+require 'plugin_manager'
+# require 'sequence_with_action'
+# 
+require 'scbi_fastq'
+require 'sequence_group'
+
+class SeqtrimWorker <  ScbiMapreduce::Worker
+
+    def process_object(obj)
+          running_seqs=SequenceGroup.new(obj)
+          
+         # execute plugins
+         @plugin_manager.execute_plugins(running_seqs)
+         
+         # add output data
+          add_output_data(running_seqs)
+         
+			   return running_seqs
+		end
+		
+    def receive_initial_config(obj)
+
+				# Reads the parameters
+				$WORKER_LOG.info "Params received"
+#				@params = Params.new(params_path)
+				@params = obj
+				
+				@use_qual=@params.get_param('use_qual')
+				@use_json=@params.get_param('use_json')
+    end
+
+		def starting_worker
+        # $WORKER_LOG.level = Logger::ERROR
+				$WORKER_LOG.info "Loading actions"
+				@action_manager = ActionManager.new
+
+				$WORKER_LOG.info "Loading plugins"
+				@plugin_list = @params.get_param('plugin_list') # puts in plugin_list the plugins's array
+				$WORKER_LOG.info "PLUGIN LIST:" + @plugin_list
+				
+				@plugin_manager = PluginManager.new(@plugin_list,@params) # creates an instance from PluginManager. This must storage the plugins and load it    
+		
+		rescue Exception => e
+			puts (e.message+ e.backtrace.join("\n"))
+		
+		end
+		
+		
+		def closing_worker
+			 
+		end		
+		
+		
+		def add_output_data(obj)
+      obj.output_text=[]
+      
+		  obj.each do |seq|
+		    	obj.output_text << seq.to_text
+          write_seq_to_files(obj.output_files,seq, obj.stats)
+		  end
+		  
+      # @remove seqs since they are not needed anymore to write output files
+      obj.remove_all_seqs
+	  end
+	  
+	  def add_stat(stats,key,subkey,value,count=1)
+	    
+	    stats[key]={} if !stats[key]
+      stats[key][subkey]={} if !stats[key][subkey]
+      stats[key][subkey][value]=0 if !stats[key][subkey][value]
+      
+  	  stats[key][subkey][value]+=count
+    end
+
+  	def write_seq_to_files(files,seq, stats)
+      # puts stats.to_json
+      
+      dir_name,file_name=seq.get_file_tag_path
+      # puts File.join(dir_name,'sequences_'+file_name)
+      
+      # get current inserts
+      inserts = seq.get_inserts
+      
+      # qualities are optional
+      if @use_qual
+        qual_inserts = seq.get_qual_inserts
+ 	    end
+      
+      # save json if necessary
+      if @use_json
+        json_file(files)<< seq.to_json
+      end
+      
+      # find mids
+      mid = seq.get_actions(ActionMid).first
+      
+      if (seq.seq_rejected)           # sequence rejected
+      
+      	#save to rejected sequences 
+        message = seq.seq_rejected_by_message 
+        rejected_output_file(files)<<('>'+seq.seq_name+ ' ' + message)
+        
+        add_stat(stats,'sequences','rejected',seq.seq_rejected_by_message)
+        add_stat(stats,'sequences','count','rejected')
+    	  
+
+      elsif (inserts.empty?)  #sequence with no inserts
+        message = 'No valid inserts found'
+        rejected_output_file(files)<<('>'+seq.seq_name+ ' ' + message)
+        
+        add_stat(stats,'sequences','rejected',message)
+        add_stat(stats,'sequences','count','rejected')
+        
+			elsif (inserts.count == 2) # sequence with two inserts  = PAIRED SEQUENCES
+        add_stat(stats,'sequences','count','output_seqs_paired')
+
+        # TODO - Add this stats to full stats
+        # @@full_stats.add_stats({'sequences' => {'paired' => {'count' => 1}}})
+  	    
+				if (mid.nil? || (mid.message=='no_MID') ) # without mid
+					mid_id = 'no_MID'
+					mid_message = ' No MID found'
+				else
+					mid_id = mid.tag_id
+					mid_message=''
+					if mid_id != mid_message
+						mid_message = ' '+mid.message
+					end
+				end
+
+        # fasta_file = get_paired_file(mid_id)
+
+        n="#{seq.seq_name}_left"
+        c="template=#{seq.seq_name} dir=R library=#{mid_id}"
+        f=inserts[0].reverse.tr('actgACTG','tgacTGAC')
+        q=[]
+        if @use_qual
+            q=qual_inserts[0].reverse			    
+   	    end   
+
+        paired_file(files,dir_name,file_name)<<FastqFile.to_fastq(n,f,q,c)
+        
+        
+        n="#{seq.seq_name}_right"
+        c="template=#{seq.seq_name} dir=F library=#{mid_id}"
+        f=inserts[1]
+        q=[]
+        if @use_qual
+	          q=qual_inserts[1]
+   	    end
+        
+        paired_file(files,dir_name,file_name)<<FastqFile.to_fastq(n,f,q,c)
+        
+        
+      elsif (inserts.count == 1) # sequence with one insert
+
+				if (mid.nil? || (mid.message=='no_MID') ) # without mid
+					mid_id = 'no_MID'
+					mid_message = ' No MID found'
+				else
+					mid_id = mid.tag_id
+					mid_message=''
+					if mid_id != mid_message
+						mid_message = ' '+mid.message
+					end
+				end
+
+       # save fasta and qual in no MID file
+        has_low_complexity = seq.get_actions(ActionLowComplexity)
+        
+        if has_low_complexity.empty?
+          add_stat(stats,'sequences','count','output_seqs')
+          
+          # fasta_file = get_sequence_file(mid_id)
+          # sff_file=get_sffinfo_file(mid_id)
+          fasta_file=sequence_file(files,dir_name,file_name)
+          sff_file=sffinfo_file(files,dir_name,file_name)
+        else
+          add_stat(stats,'sequences','count','output_seqs_low_complexity')
+          
+          # fasta_file = get_low_complexity_file(mid_id)
+          # sff_file=get_low_sffinfo_file(mid_id)
+          fasta_file=low_complexity_file(files,dir_name,file_name)
+          sff_file=low_sffinfo_file(files,dir_name,file_name)
+        end
+        
+        q=[]
+        if @use_qual
+	         q=qual_inserts[0]
+				end
+        
+        n=seq.seq_name
+        c=mid_message
+        f=inserts[0]
+        
+        fasta_file << FastqFile.to_fastq(n,f,q,c)
+        
+        inserts_pos = seq.get_actions(ActionInsert)
+        
+        sff_file<< "#{n} #{inserts_pos[0].start_pos+1} #{inserts_pos[0].end_pos+1}"
+        
+      end
+  	  
+	  end
+	  
+	  
+	  # ACCESS TO FILES
+
+	  def json_file(files)
+      return get_file(files,File.join(OUTPUT_PATH,'results.json'))
+	  end
+	  
+	  def rejected_output_file(files)
+      return get_file(files,File.join(OUTPUT_PATH,'rejected.txt'))
+	  end
+	  
+	  
+	  def sequence_file(files, dir_name, file_name)
+      return get_file(files,File.join(OUTPUT_PATH,dir_name,'sequences_'+file_name+'.fastq'))
+	  end 	
+	  
+    def paired_file(files, dir_name, file_name)
+      return get_file(files,File.join(OUTPUT_PATH,dir_name,'paired_'+file_name+'.fastq'))
+    end
+
+    def low_complexity_file(files, dir_name, file_name)
+      return get_file(files,File.join(OUTPUT_PATH,dir_name,'low_complexity_'+file_name+'.fastq'))
+    end
+	  
+	  def sffinfo_file(files, dir_name, file_name)
+      return get_file(files,File.join(OUTPUT_PATH,dir_name,'sff_info_'+file_name+'.txt'))
+    end
+
+	  def low_sffinfo_file(files, dir_name, file_name)
+      return get_file(files,File.join(OUTPUT_PATH,dir_name,'low_complexity_sff_info_'+file_name+'.txt'))
+    end
+	  
+	  def get_file(files,fn)
+	    res=files[fn]
+	    
+	    if !res
+	      files[fn]=[]
+	      res=files[fn]
+      end
+      
+      return res
+    end
+	  
+end

data/lib/seqtrimnext/classes/extract_stats.rb

@@ -0,0 +1,255 @@
+######################################
+# Author:: Almudena Bocinos Rioboo
+# Extract stats like mean of sequence's length
+######################################
+
+# $: << '/Users/dariogf/progs/ruby/gems/scbi_plot/lib'
+# $: << '/Users/dariogf/progs/ruby/gems/scbi_math/lib'
+
+require 'scbi_plot'
+require "scbi_math"
+
+class ExtractStats
+
+  def initialize(sequence_reader,params)
+
+    @sequence_lengths = []         #array of sequences lengths
+    @length_frequency = []      #number of sequences of each size (frequency)
+    @keys={}  #found keys
+    @params = params
+    @use_qual=sequence_reader.with_qual?
+    # @params.get_param('use_qual')
+
+    @totalnt=0
+    @qv=[]
+
+
+    @sequence_lengths_stats, @length_frequency_stats, @quality_stats = extract_stats_from_sequences(sequence_reader)
+
+
+    set_params_and_results
+
+    plot_lengths
+
+    plot_qualities if @use_qual
+
+    print_global_stats
+
+  end
+
+  def extract_stats_from_sequences(sequence_reader)
+
+    sequence_reader.each do |name_seq,fasta_seq,qual|
+      l = fasta_seq.length
+
+      @totalnt+=l
+
+      #save all lengths
+      @sequence_lengths.push l
+
+      # add key value
+      add_key(fasta_seq[0..3].upcase)
+
+      # add fasta length
+      @length_frequency[fasta_seq.length] = (@length_frequency[fasta_seq.length] || 1 ) + 1
+
+      #extract qv values
+      extract_qv_from_sequence(qual) if @use_qual
+
+      # print some progress info
+      if (sequence_reader.num_seqs % 10000==0)
+        puts "Calculating stats: #{sequence_reader.num_seqs}"
+      end
+
+    end
+
+    length_stats = ScbiNArray.to_na(@sequence_lengths)
+    length_frequency_stats = ScbiNArray.to_na(@length_frequency.map{|e| e || 0})
+    quality_stats = ScbiNArray.to_na(@qv) if @use_qual
+
+    return [length_stats, length_frequency_stats, quality_stats]
+  end
+
+  def plot_lengths
+
+    ## PLOT RESULTS
+    if !File.exists?('graphs')
+    	Dir.mkdir('graphs')
+    end
+    
+
+    x = []
+    y = []
+
+    x =(0..@length_frequency.length-1).collect.to_a
+    y = @length_frequency.map{|e| e || 0}
+
+    file_name = 'graphs/size_stats.png'
+
+    p=ScbiPlot::Lines.new(file_name,'Stats of sequence sizes')
+    p.x_label= "Sequence length"
+    p.y_label= "Number of sequences"
+
+    p.add_x(x)
+
+    p.add_series('sizes', y,'impulses',2)
+
+    p.add_vertical_line('Mode',@length_frequency_stats.fat_mode[0])
+
+    p.add_vertical_line('L',@params.get_param('min_sequence_size_raw').to_i)
+    p.add_vertical_line('H',@params.get_param('max_sequence_size_raw').to_i)
+
+    p.do_graph
+
+
+  end
+
+  def plot_qualities
+
+    if !File.exists?('graphs')
+    	Dir.mkdir('graphs')
+    end
+    minimum_qual_value = @params.get_param('min_quality').to_i
+    
+    # get qualities values
+    x=[]
+    y=[]
+    min=[]
+    max=[]
+    qual_limit=[]
+
+    @qv.each_with_index do |e,i|
+      x << i
+      y << (e[:tot]/e[:nseq])
+      min << (e[:min])
+      max << (e[:max])
+      qual_limit << minimum_qual_value
+        # puts "#{i}: #{e[:tot]/e[:nseq]}"
+      end
+
+    # make plot of qualities
+
+    file_name='graphs/qualities.png'
+
+  	 p=ScbiPlot::Lines.new(file_name,'Stats of sequence qualities')
+     p.x_label= "Nucleotide position"
+     p.y_label= "Quality value"
+
+      p.add_x(x)
+
+      p.add_series('mean', y)
+      p.add_series('min', min)
+      p.add_series('max', max)
+      p.add_series('qual limit',qual_limit)
+
+
+      p.do_graph
+  end
+  
+   
+   def add_qv(q,i)
+     if !@qv[i]
+       @qv[i]={:max => 0, :min => 1000000, :nseq => 0, :tot => 0}
+     end
+     
+     # set max
+     @qv[i][:tot]+=q
+     @qv[i][:nseq]+=1
+     @qv[i][:min]=[@qv[i][:min],q].min
+     @qv[i][:max]=[@qv[i][:max],q].max
+     
+   end
+   
+   def extract_qv_from_sequence(qual)
+     qual.each_with_index do |q,i|
+       add_qv(q,i)
+     end
+   end
+
+  def add_key(key)
+    if @keys[key].nil?
+	    @keys[key]=1
+    else
+	    @keys[key]+=1
+    end
+  end
+
+  def get_max_key
+    return @keys.keys.sort{|e1,e2| @keys[e1]<=>@keys[e2]}.last
+  end
+    
+  def set_params_and_results
+
+    if @sequence_lengths.empty? 
+      puts "No sequences has been sucessfully readed " 
+      return
+    end
+    
+    
+    # set limiting parameters
+    
+    @params.set_param('sequencing_key',get_max_key)
+    @params.set_param('all_found_keys',@keys.to_json)
+
+    # sequence min size, is taken directly from params file
+    # max sequence limit is calculated here 
+    if (@sequence_lengths_stats.variance_coefficient<=10) or (@params.get_param('accept_very_long_sequences')=='true')
+
+      # high size limit is calculated with stats
+      @params.set_param('max_sequence_size_raw',(@sequence_lengths_stats.max+10).to_i)
+
+    else # > 10 %
+
+      # high size limit is calculated with stats
+      @params.set_param('max_sequence_size_raw',(@sequence_lengths_stats.mean+2*@sequence_lengths_stats.stddev).to_i)
+    end
+
+
+  end
+
+def print_global_stats
+
+if !@sequence_lengths_stats.nil?
+initial_stats={}
+initial_stats[:sequence_count] = @sequence_lengths_stats.size
+initial_stats[:smallest_sequence_size] = @sequence_lengths_stats.min
+initial_stats[:biggest_sequence_size] = @sequence_lengths_stats.max
+
+initial_stats[:min_sequence_size_raw]=@params.get_param('min_sequence_size_raw')
+initial_stats[:max_sequence_size_raw]=@params.get_param('max_sequence_size_raw')
+initial_stats[:coefficient_of_variance]=@sequence_lengths_stats.variance_coefficient
+initial_stats[:nucleotide_count]=@totalnt
+initial_stats[:mode_of_sizes]=@length_frequency_stats.fat_mode[0]
+initial_stats[:mean_of_sequence_sizes]=@sequence_lengths_stats.mean
+
+initial_stats[:qv]=@qv
+initial_stats[:used_key]=get_max_key
+initial_stats[:all_keys]=@keys
+
+File.open(File.join(OUTPUT_PATH,'initial_stats.json'),'w') do |f|
+  f.puts JSON.pretty_generate(initial_stats)
+end
+
+puts "_"*10+ " STATISTICS "+"_"*10
+puts "Total sequence count: #{@sequence_lengths_stats.size}"
+
+puts "Smallest sequence: #{initial_stats[:smallest_sequence_size]} nt"
+puts "Biggest sequence : #{initial_stats[:biggest_sequence_size]} nt"
+puts "Mean of sequence sizes : #{initial_stats[:mean_of_sequence_sizes]} nt"
+puts "Mode of sequence sizes : #{initial_stats[:mode_of_sizes]} nt"
+
+puts "Low size limit : #{initial_stats[:min_sequence_size_raw]} nt"
+puts "High size limit : #{initial_stats[:max_sequence_size_raw]} nt"
+
+puts "Coefficient of variation: #{initial_stats[:coefficient_of_variance]} %"
+puts "Total nucleotide count: #{initial_stats[:nucleotide_count]} nt"
+
+puts "_"*30
+
+
+end
+
+end
+
+
+end

data/lib/seqtrimnext/classes/gnu_plot_graph.rb

@@ -0,0 +1,140 @@
+require 'gnuplot'
+
+class GnuPlotGraph
+
+def initialize(file_name,x,y,title=nil)
+    $VERBOSE=true
+    Gnuplot.open do |gp|
+      # histogram
+      Gnuplot::Plot.new( gp ) do |plot|
+     
+        # plot.space= 5 # it's the free space between the first/last value and the begin/end of axis X
+        
+       #plot.set("xrange [#{xr_min}: #{xr_max}]") 
+				if !title
+				 title=file_name
+				end
+				
+        plot.title "#{title}"
+        plot.xlabel "length"
+        plot.ylabel "Number of sequences"
+        plot.set "key off" #leyend
+        
+        
+#        plot.set "style fill   solid 1.00 border -1"
+#        #plot.set "style histogram clustered gap 0 title offset character 0, 0, 0"
+#        plot.set "style data histograms"
+#        plot.set "boxwidth 0.2 absolute"
+        
+# For this next line, lw is linewidth (2-4)?
+#plot [XMIN:XMAX] 'myHistogramData' with boxes lw VALUE
+
+        contains_strings=false
+
+        x.each do |v|
+  	 	  	begin
+  		 	 	  r=Integer(v)
+  	 	  	rescue
+  		 	 	  contains_strings=true
+  		 	 	  break
+  	 	    end
+  	 	   end
+        
+        
+        if !contains_strings
+            # plot.set "xrange [*:*]"
+            # puts "INTEGER GRAPH"
+				    plot.style "fill  pattern 22  border -1"
+				    plot.set "boxwidth 0.2" # Probably 3-5.
+				     
+				    plot.data << Gnuplot::DataSet.new( [x, y] ) do |ds| 
+				      #ds.with=  " boxes lw 1"
+              # ds.using=""
+				      				      ds.with=  " imp lw 4"
+				    end         
+        
+        else #graph with strings in X axis
+            # puts "STRING GRAPH"        
+          plot.xlabel ""
+            
+          plot.set "style fill solid 1.00 border -1"
+          plot.set "style histogram clustered gap 1 title offset character 0, 0, 0"
+          plot.set "style data histogram"
+          plot.set "boxwidth 0.2 absolute"
+          if x.count>4 then
+            plot.set "xtics offset 0,graph 0 rotate 90"
+          end
+          # $VERBOSE=true
+          # plot.set "style data linespoints"
+          # plot.set "xtics border in scale 1,0.5 nomirror rotate by -45  offset character 0, 0, 0"
+          
+          # s = []
+          # # i=0
+          # x.each_with_index do |v,i|
+          #   #s.push "\"#{v}\""
+          #   s.push "#{v} #{i}"
+          #   
+          #   # i+=1
+          # end
+          # 
+          # 
+          # plot.set "xtics (#{s.join(',')})"
+          # puts "XTICKS: (#{s.join(',')})"
+          # puts "X:"
+          #           puts x.join(';')
+          #           puts "Y:"
+          #           puts y.join(';')
+          
+          # if more than 20 strings, then keep greater ones
+          
+          if x.count>20
+            # puts "original X:#{x.count}"
+            $VERBOSE=true            
+            h = {}
+            
+            x.each_with_index do |x1,i|
+              h[x1]=y[i]
+            end
+            
+            # puts h.to_json
+            x=[]
+            y=[]
+            
+            10.times do
+              ma=h.max_by{|k,v| v}
+              if ma
+                puts "MAX:",ma.join(' * '),"of",h.values.sort.join(',')
+                x.push ma[0]
+                y.push ma[1]
+                h.delete(ma[0])
+              end
+            end
+            
+            # puts "MAX 20 #{x.length}:#{x.join(';')}"
+            
+            # set key below
+            # plot.set "label 3 below" 
+            
+          end
+
+		      plot.data << Gnuplot::DataSet.new( [x,y] ) do |ds| 
+            ds.using = "2:xticlabels(1)"   #show the graph and use labels at x
+            # ds.using="2"
+		        #ds.with=  " boxes lw 1"
+		        # ds.using = "2 t 'Sequences' " #show the legend in the graph          
+		      end
+          
+	      end
+         
+        if !file_name.nil?
+          plot.terminal "png size 800,600"
+          plot.output "#{file_name}"
+        end
+      end
+      
+   end
+
+end
+
+
+end