seqtrimnext 2.0.29

data/lib/seqtrimnext/classes/scan_for_restr_site.rb

@@ -0,0 +1,138 @@
+#!/usr/bin/env ruby
+
+#########################################
+# Author:: Almudena Bocinos Rioboo
+# This class provided the methods to read the parameter's file and to create the structure where will be storaged the param's name and the param's numeric-value
+ #########################################
+class ScanForRestrSite
+  
+  #Creates the structure and start the reading of parameter's file
+  def initialize(sequence,rest)    
+    @seq_fasta=sequence 
+    @rest=rest      
+    puts "#{@seq_fasta}  , #{@rest}"  
+    res = execute
+    
+
+    res.each do |e| 
+        puts "#{e.join(',')}"
+    end
+    
+  # selects from res,the max good hit  
+    puts "--- MAX: --- "
+    
+    max = res.max{|e1,e2| e1[1]<=> e2[1]}
+                      
+    puts max.join(' ; ')                    
+                                             
+  # checks if the max one has the size of restriction with a margen error
+    margen = (@rest.size <= 4)? 0 : 1;  # <- don't change       
+    if ((max[1] !=  @rest.size) && (max[1] != @rest.size-margen)) 
+      puts "-the max good hit hasn't the size minimum: #{@rest.size} or #{@rest.size-margen} "
+      max=[]
+    end
+    
+    
+    
+    #read_file(path)   
+  end
+            
+  
+  
+  def execute    
+    r=[]
+                              # 
+                              # for (my $p=0; $p < $sL-$srfL; $p++){
+                              #     $os = $ns = $xs = 0;
+                              #     for ( my $i=0; $i < $srfL; $i++ ) {
+                              #         my $c  = substr($s, $i+$p, 1);  # ver si decrementar antes pos
+                              #         my $cc = substr($restrSite, $i, 1);
+                              #         if ($c eq $cc) {
+                              #             ++$os;
+                              #         } elsif ($c eq "N"){
+                              #             ++$ns;
+                              #         } else {
+                              #             ++$xs;
+                              #         }
+                              #     }
+                              #     $r[$p] = [$p, $os, $ns, $xs];
+                              #         print "$p, $os, $ns, $xs\n";
+                              # }                     
+    for p in 0..@seq_fasta.size-@rest.size 
+        os = 0; 
+        ns = 0; 
+        xs = 0;                    
+        puts "-------[#{p}]-#{@seq_fasta[p,@seq_fasta.size-p]}  , #{@rest}"  
+        
+       i=0
+      @rest.each_char do |cc|
+         c = @seq_fasta[i+p].chr
+         puts "(#{c}==#{cc})=>#{c==cc}"
+        if (c == cc)
+          os += 1
+        elsif (c == 'N')
+          ns += 1
+        else     
+          xs += 1
+        end
+        i+=1     
+        
+      end
+      r[p]=[p,os,ns,xs]
+       puts r[p].join(',')
+    end     
+    return r
+  end
+  
+  # Reads param's file
+  def read_file(path_fichero)
+    File.open(path_fichero).each_line do |line| 
+           
+      line.chomp! # delete end of line      
+      
+      if !line.empty?
+        if !(line =~ /^#/)   # if line is not a comment
+          # extract the parameter's name in params[0] and the parameter's value in params[1]
+          params = line.split(/\s*=\s*/)
+          
+          # storage in the hash the pair key/value, in our case will be name/numeric-value , 
+          # that are save in params[0] and params[1],  respectively
+          @h[params[0]] = params[1]
+          
+          $LOG.debug "read: #{params[1]}"
+        end # end if comentario
+      end #end if line
+    end #end each
+    $LOG.info "File Params have been readed"
+
+  end# end def
+
+  #  Prints the pair name/numeric-value for every parameter  
+  def print_parameters()
+    @h.each do |clave, valor|
+      
+      $LOG.debug  "The Parameter #{clave} have the value " +valor.to_s
+    end
+  end
+  
+  # Return the parameter's list in an array
+  def get_param(param)
+    #$LOG.debug "Get Param:  #{@h[param]}"
+    return @h[param]
+  end
+  
+  def set_param(param,value)
+    @h[param] = value
+  end
+  
+  #attr_accessor :h  # to accede to the atribute 'h' from out of this class
+  
+  # Returns true if exists the parameter and nil if don't
+  def exists?(param_name)
+    return !@h[param_name].nil?
+  end
+  
+end      
+scan = ScanForRestrSite.new("AaaaACGTACGT", "AGTAC")
+# scan = ScanForRestrSite.new("AaaaACGTAeCGT", "AGTAC")
+

data/lib/seqtrimnext/classes/scbi_stats.rb

@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+
+require 'json'
+
+class Array
+  
+  def sum
+    r=0
+    each do |e|
+      r+=e
+    end
+    return r
+  end
+  
+end
+
+class ScbiStats
+  
+  def initialize(values)
+    
+    @values=values
+    
+    
+  end
+  
+  def get_window_value(i,window_size=10)
+    start_pos=[0,i-window_size].max
+    
+    end_pos=[@values.length,i+window_size].min
+    # puts "#{@values[start_pos..end_pos]} => #{@values[start_pos..end_pos].sum}"
+    return @values[start_pos..end_pos].sum
+  end
+  
+  def fat_mode(window_size=10)
+    
+    fat_modes=[]
+    max_fat=0
+    
+    @values.length.times do |i|
+      fat=get_window_value(i)
+      
+      fat_modes << fat
+      
+      if fat_modes[max_fat] < fat
+        max_fat=i
+      end
+      
+    end
+    # puts fat_modes
+    return max_fat
+    # puts @values.length, @fat_modes.length
+  end
+  
+end
+
+
+# istat=JSON.parse(File.read('initial_stats.json'))
+# 
+# x=[]
+# istat['qv'].each do |qv|
+#   x<< qv['tot'].to_i
+#   
+# end
+# # Usage:
+# 
+# s=ScbiStats.new(x)
+# 
+# puts s.fat_mode

data/lib/seqtrimnext/classes/seqtrim.rb

@@ -0,0 +1,317 @@
+######################################
+# Author: Almudena Bocinos Rioboo
+# This is the main class. 
+######################################
+
+require 'extract_stats'
+
+# $: << File.expand_path('~/progs/ruby/gems/scbi_drb/lib')
+
+require 'scbi_mapreduce'
+require 'seqtrim_work_manager' 
+require 'action_manager'
+
+# SEQTRIM_VERSION_REVISION=27
+# SEQTRIM_VERSION_STAGE = 'b'
+# $SEQTRIM_VERSION = "2.0.0#{SEQTRIM_VERSION_STAGE}#{SEQTRIM_VERSION_REVISION}"
+
+class Seqtrim
+  
+  # First of all, reads the file's parameters, where are the values of all parameters and the 'plugin_list'  that specifies the order of execution from the plugins. 
+  #
+  # Secondly, loads the plugins in a folder .
+  #
+  # Thirdly, checks if parameter's file have the number of parameters necessary for every plugin that is going to be executed.
+  #
+  # After that, creates a thread's pool of a determinate number of workers, e.g. 10 threads,
+  # reads the sequences from files 'fasta' , until now without qualities,
+  # and executes the plugins over the sequences in the pool of threads
+  
+  
+  def get_cd_hit_cmd(cd_hit_input_file,workers,init_file_path)
+    
+    num_cpus_cdhit=1
+    cmd=''
+    
+    # if workers is an integer, reduce it by one in the server
+		begin
+		  Integer(workers)
+		  num_cpus_cdhit = workers
+		  cmd = "cd-hit-454 -i #{cd_hit_input_file} -o clusters.fasta -M #{num_cpus_cdhit*1000} -T #{num_cpus_cdhit} > cd-hit-454.out"
+	    
+	  rescue Exception => exception #not an integer, send via ssh to other machine
+      # puts exception
+	    worker_hash={};workers.map{|e| worker_hash[e] = (worker_hash[e]||0) +1}
+	    
+	    max_worker = worker_hash.sort_by{|k,v| -v}.first
+	    puts "Found these workers: #{worker_hash.sort_by{|k,v| -v}}"
+	    num_cpus_cdhit=max_worker[1]
+	    
+	    init=''
+	    cd=''
+
+
+	    cmd = "cd-hit-454 -i #{cd_hit_input_file} -o clusters.fasta -M #{num_cpus_cdhit*1000} -T #{num_cpus_cdhit} > cd-hit-454.out"
+	    		    
+      # worker is different to current machine, send over ssh
+	    if max_worker[0]!= workers[0]
+	       
+		    
+         if File.exists?(init_file_path)
+           init=". #{init_file_path}; "
+         end
+
+        pwd=`pwd`.chomp
+
+        cd =''
+
+        if File.exists?(pwd)
+          cd = "cd #{pwd}; "
+        end
+        cmd = "ssh #{max_worker[0]} \"#{init} #{cd} #{cmd}\""
+	    end
+    end
+    
+    return cmd
+  end
+  
+  def check_global_params(params)
+	  errors=[]
+	  
+    # check plugin list
+    comment='Plugins applied to every sequence, separated by commas. Order is important'
+   # default_value='PluginLowHighSize,PluginMids,PluginIndeterminations,PluginAbAdapters,PluginContaminants,PluginLinker,PluginVectors,PluginLowQuality'
+#    params.check_param(errors,'plugin_list','String',default_value,comment)
+    params.check_param(errors,'plugin_list','PluginList',nil,comment)
+
+    
+    comment='Should SeqTrimNext analysis be based on NGS? (if setting to false, a classic Sanger sequencing is considered)'
+	  default_value='true'
+	  params.check_param(errors,'next_generation_sequences','String',default_value,comment)
+
+    
+    comment='Remove duplicated (clonal) sequences (using CD-HIT 454)'
+	  default_value='true'
+	  params.check_param(errors,'remove_clonality','String',default_value,comment)
+
+
+		comment='Minimum insert size for every trimmed sequence'
+		default_value = 40
+		params.check_param(errors,'min_insert_size_trimmed','Integer',default_value,comment)
+		
+		comment='Minimum insert size for each end of paired-end reads; true paired-ends have both single-ends longer than this value'
+		default_value = 40
+		params.check_param(errors,'min_insert_size_paired','Integer',default_value,comment)
+		
+
+		comment='Do not reject unexpectedly long sequences found in the raw data'
+		default_value='true'
+		params.check_param(errors,'accept_very_long_sequences','String',default_value,comment)
+
+		comment='Seqtrim version'
+		default_value=Seqtrimnext::SEQTRIM_VERSION
+		params.check_param(errors,'seqtrim_version','String',default_value,comment)
+		
+		if !errors.empty?
+          $LOG.error 'Please, define the following global parameters in params file:'
+          errors.each do |error|
+            $LOG.error '   -' + error
+          end #end each
+        end #end if
+
+		return errors.empty?
+		
+  end
+  
+  
+  
+  def initialize(options)
+    # ,options[:fasta],options[:qual],,,,
+    params_path=options[:template]
+    
+    ip=options[:server_ip]
+    port=options[:port]
+    workers=options[:workers]
+    only_workers=options[:only_workers]
+    chunk_size = options[:chunk_size]
+    use_json = options[:json]
+       
+    
+    # it is the server part
+  if !only_workers then
+
+    sequence_reader = nil
+    cd_hit_input_file = nil
+    
+    # TODO - FIX seqtrim to not iterate two times over input, so STDIN can be used
+    
+    # open sequence reader and expand input files paths
+    if options[:fastq]
+      if options[:fastq]=='-'
+        seqs_path = STDIN
+      else
+        seqs_path = File.expand_path(options[:fastq])
+      end
+      cd_hit_input_file = seqs_path
+      sequence_reader = FastqFile.new(seqs_path,'r',:sanger, true)
+      # cd_hit_input_file = 'cd-hit-input.fasta'
+      cd_hit_input_file = seqs_path
+      # $LOG.info "Converting input file for cd-hit-454"
+      # $LOG.info "Conversion done"
+      
+    else
+
+      seqs_path = File.expand_path(options[:fasta])
+      cd_hit_input_file = seqs_path
+      
+      qual_path =  File.expand_path(options[:qual]) if qual_path
+      sequence_reader = FastaQualFile.new(options[:fasta],options[:qual],true)
+
+    end
+
+   
+    $LOG.info "Loading params"
+    # Reads the parameter's file
+    params = Params.new(params_path)
+
+    $LOG.info "Checking global params"
+    if !check_global_params(params)
+    		exit
+    end
+                                   
+    # Load actions
+    $LOG.info "Loading actions"
+    action_manager = ActionManager.new()
+
+		# load plugins 
+    plugin_list = params.get_param('plugin_list') # puts in plugin_list the plugins's array
+    $LOG.info "Loading plugins [#{plugin_list}]"    
+    
+    
+    plugin_manager = PluginManager.new(plugin_list,params) # creates an instance from PluginManager. This must storage the plugins and load it
+     
+     
+     
+		# load plugin params
+    $LOG.info "Check plugin params"
+    if !plugin_manager.check_plugins_params(params) then
+	    	$LOG.error "Plugin check failed"
+
+	    	# save used params to file
+      params.save_file('used_params.txt')
+
+      exit
+    end
+    
+    
+    if !Dir.exists?(OUTPUT_PATH)
+      Dir.mkdir(OUTPUT_PATH)
+    end
+
+    $LOG.info "Calculatings stats"
+    # Extract global stats
+    ExtractStats.new(sequence_reader,params)
+    
+    
+    # save used params to file
+    params.save_file(File.join(OUTPUT_PATH,'used_params.txt'))
+    
+    piro_on = (params.get_param('next_generation_sequences')=='true')
+    
+      # format blast database with truncated file
+      #MakeBlastDb.format_db(es.truncated_file_path,File.basename(es.truncated_file_path,File.extname(es.truncated_file_path)),'./') if piro_on
+      
+      # leer mids
+      params.load_mids(File.join($FORMATTED_DB_PATH,'mids.fasta'))
+      params.load_linkers(File.join($FORMATTED_DB_PATH,'linkers.fasta'))
+
+      
+      
+
+	    
+      #execute cd-hit
+      if params.get_param('remove_clonality')=='true'
+        
+        cmd=get_cd_hit_cmd(cd_hit_input_file,workers,File.join($SEQTRIM_PATH,'init_env'))
+        
+        $LOG.info "Executing cd-hit-454: #{cmd}"
+        
+        if !File.exists?('clusters.fasta.clstr')
+				  system(cmd)
+        end
+        
+        if File.exists?('clusters.fasta.clstr')
+	        params.load_repeated_seqs('clusters.fasta.clstr')
+        else
+          $LOG.error("Exiting due to not found clusters.fasta.clstr. Maybe cd-hit failed. Check cd-hit.out")
+          exit
+        end
+	    end
+      
+			
+		############ SCBI DRB ###########
+#			port = 50000
+#			ip = "10.250.255.6"
+#			port = 50000
+#			ip = "localhost"
+#
+#			workers=20
+#			only_workers=false
+				# launch work manager
+	
+
+  end # end only_workers
+
+			custom_worker_file = File.join(File.dirname(__FILE__), 'em_classes','seqtrim_worker.rb')
+      
+			$LOG.info "Workers:\n#{workers}"
+			
+      if only_workers then
+        
+        worker_launcher = ScbiMapreduce::WorkerLauncher.new(ip,port, workers, custom_worker_file, STDOUT)
+        worker_launcher.launch_workers_and_wait
+      else
+  			$LOG.info 'Starting server'
+      	        
+ 				SeqtrimWorkManager.init_work_manager(sequence_reader, params,chunk_size,use_json)
+				
+        begin
+  				cpus=1
+				
+  				if RUBY_PLATFORM.downcase.include?("darwin")
+            cpus=`hwprefs -cpu_count`.chomp.to_i
+  			  else
+  			    cpus=`grep processor /proc/cpuinfo |wc -l`.chomp.to_i
+  		    end
+        rescue
+          cpus=1
+        end
+				
+        # if workers is an integer, reduce it by one (because of the server)
+				begin
+				  Integer(workers)
+				  if workers>1 && workers<cpus
+				    workers-=1
+			    end
+			  rescue
+				  if workers.count>1 && workers.count<cpus
+			      workers.shift
+			    end
+		    end
+				
+				# launch processor server passing the ip, port and all required params
+        # server = Server.new(ip,port, workers, SeqtrimWorkManager,custom_worker_file, STDOUT,File.join($SEQTRIM_PATH,'init_env'))
+				server = ScbiMapreduce::Manager.new(ip,port, workers, SeqtrimWorkManager,custom_worker_file, STDOUT,'~/.seqtrimnext')
+				server.chunk_size=chunk_size
+				server.start_server
+        
+        # close sequence reader
+				sequence_reader.close
+				$LOG.info 'Closing server'
+			end
+			
+			############ SCBI DRB ###########
+
+  end
+  
+end # Seqtrim class

data/lib/seqtrimnext/classes/sequence.rb

@@ -0,0 +1,55 @@
+########################################################
+# Author: Almudena Bocinos Rioboo                      
+# 
+# Defines the class Sequence's attribute                                                     
+# 
+########################################################
+
+class Sequence
+  #storages the name and the contains from fasta sequence
+  def initialize(seq_name,seq_fasta,seq_qual, seq_comment = '')     
+      
+      @seq_fasta=seq_fasta
+      @seq_name=seq_name
+      @seq_qual=seq_qual           
+      @seq_comment = seq_comment
+      
+      @seq_rejected=false   
+      @seq_repeated=false
+      @seq_reversed=false 
+      
+      @seq_rejected_by_message=''
+      
+      @ns_present = ns_present? 
+      @xs_present = xs_present?    
+      
+      
+      
+      # puts "INIT SEQ >>>> #{seq_name} #{seq_specie}"
+      
+  end
+  
+  attr_accessor :seq_name, :seq_fasta, :seq_qual, :seq_comment , :seq_rejected, :seq_repeated , :seq_reversed 
+  attr_accessor :seq_rejected_by_message
+  
+  def ns_present?
+    return (@seq_fasta.index('N')  != nil)
+  end
+  
+  def xs_present?
+    return (@seq_fasta.index('X') != nil)
+  end
+  
+  def seq_is_long_enough(seq_min_length)
+    return (@seq_fasta.length>=seq_min_length)
+  end
+  
+  def to_fasta
+  		return ">"+@seq_name.to_s+"\n"+@seq_fasta  
+  end
+  
+  def to_qual
+  		return ">"+@seq_name.to_s+"\n"+"#{@seq_qual}"  
+  end
+  
+end

data/lib/seqtrimnext/classes/sequence_group.rb

@@ -0,0 +1,72 @@
+
+
+class SequenceGroup
+  
+  attr_accessor :stats,:output_text,:output_files
+  
+  
+  def initialize(seqs)
+    @stats={}
+    @seqs=seqs
+    @output_text={}
+    @output_files={}
+  end
+  
+  
+  def push(seq)
+    @seqs.push seq
+  end
+  
+  def delete(seq)
+    @seqs.delete(seq)
+  end
+  
+  def empty?
+    return @seqs.empty?
+  end
+  
+  
+  def each
+      @seqs.each do |seq|
+        yield seq
+      end
+  end
+
+  def each_with_index
+      @seqs.each_with_index do |seq,i|
+        yield seq,i
+      end
+  end
+
+  
+  def reverse_each
+      @seqs.reverse_each do |seq|
+        yield seq
+      end
+  end
+  
+  def add(array)
+    @seqs += array
+  end
+  
+  def count
+    return @seqs.count
+  end
+  
+  def include?(s)
+    return @seqs.include?(s)
+  end
+  
+  def remove_all_seqs
+    @seqs=[]
+  end
+  
+  # def job_identifier
+  #     return @seqs[0].seq_name
+  # end
+  
+  def inspect
+    return "Group with #{@seqs.count} sequences"
+  end
+  
+end