seqtrimnext 2.0.29

data/lib/seqtrimnext/actions/action_vectors.rb

@@ -0,0 +1,31 @@
+require "seqtrim_action"
+
+########################################################
+# Author: Almudena Bocinos Rioboo                      
+# 
+# Defines the main methods that are necessary to execute Plugin1                                                     
+# Inherit: Plugin
+########################################################
+
+class ActionVectors < SeqtrimAction
+    
+   def initialize(start_pos,end_pos)
+     super(start_pos,end_pos)    
+     @cut =true
+
+   end
+   
+# def apply_to(seq)
+#   
+#    # seq.seq_fasta = seq.seq_fasta.slice(start_pos,end_pos)
+#    $LOG.debug " Applying #{self.class} to seq #{seq.seq_name}. BEGIN: #{@start_pos}   END: #{@end_pos}  " 
+#   
+# end  
+
+  def apply_decoration(char)
+    return char.on_green
+  end      
+
+
+
+end

data/lib/seqtrimnext/actions/seqtrim_action.rb

@@ -0,0 +1,136 @@
+######################################
+# Author:: Almudena Bocinos Rioboo
+# This class creates the structures that storage the necessary values from an action
+######################################
+
+
+require 'term/ansicolor'
+include Term::ANSIColor
+
+class SeqtrimAction
+  
+  
+  attr_accessor :start_pos ,  :end_pos, :message, :cut , :reversed , :left_action , :right_action  , :found_definition, :tag_id, :informative
+  
+  #  Creates the nexts values from  an action: start_position, end_position, type
+  def initialize(start_pos,end_pos)
+    # puts " #{start_pos} #{end_pos} #{self.class.to_s}"   
+    
+    @start_pos = start_pos.to_i
+    @end_pos = end_pos.to_i        
+    
+    #@notes = '' 
+    @left_action = false
+    @right_action = false
+    @message = ''
+    @found_definition=[] #array when contaminant or vectors definitions are saved, each  separately
+    @cut = false 
+    @informative = false
+    @reversed = false
+    # puts " #{@start_pos} #{@end_pos} #{self.class.to_s}"  
+    @tag_id =''
+    
+  end
+  
+  def apply_to(seq)   
+       
+    $LOG.debug " Applying #{self.class} to seq #{seq.seq_name} . BEGIN: #{@start_pos}   END: #{@end_pos}  " 
+    
+  end
+  
+  def description
+    
+    return "Action Type: #{self.class} Begin: #{start_pos} End: #{end_pos}"
+  end
+  
+  def inspect
+    description
+  end
+  
+  
+  def contains?(pos)
+    return ((pos>=@start_pos) && (pos<=@end_pos))
+  end
+
+  
+  def contains_action?(start_pos,end_pos,margin=10)
+    #puts "#{start_pos}>=#{@start_pos-margin} && #{end_pos}<=#{@end_pos+margin} "
+		return ((start_pos>=@start_pos-margin) && (end_pos<=@end_pos+margin))
+  end
+  
+  def apply_decoration(char)
+    return char
+  end      
+  
+
+  
+  def decorate(char,pos)
+    if contains?(pos)
+      return apply_decoration(char)
+    else
+      return char
+    end
+  end
+  
+  def type
+    return self.class.to_s
+  end  
+  def action_type
+    
+    a_type='INFO'
+    if !@informative
+      a_type = 'LEFT'
+      if right_action
+        a_type = 'RIGHT'
+      end
+    end
+    return a_type
+    
+  end
+  
+  def to_human(camel_cased_word)
+    word = camel_cased_word.to_s.dup
+    word.gsub!(/::/, '/')
+    word.gsub!(/([A-Z]+)([A-Z][a-z])/,'\1 \2')
+    word.gsub!(/([a-z\d])([A-Z])/,'\1 \2')
+    word.tr!("-", " ")
+    word.downcase!
+    word.capitalize!
+    word
+  end
+  
+  def title
+   return to_human(type.gsub('Action',''))
+  end                                                      
+  
+  def near_left?(seq_fasta_size)
+    return ((@start_pos - 0 ) < (seq_fasta_size - @end_pos))
+  end
+  
+  
+  def left_action?(seq_fasta_size)
+     res = (@left_action || (!@right_action && near_left?(seq_fasta_size)))
+     # @left_action = res
+       
+     return res
+   end
+
+   def right_action?(seq_fasta_size)
+      res= (@right_action || !left_action?(seq_fasta_size))
+      # @right_action = res
+      return res
+   end 
+   
+  def to_hash
+     a = {}
+     
+     a[:type]=type
+     a[:start_pos]=@start_pos
+     a[:end_pos]=@end_pos
+     a[:message]=@message
+     
+     return a
+  end
+  
+  
+end

data/lib/seqtrimnext/classes/action_manager.rb

@@ -0,0 +1,47 @@
+#########################################
+# Author:: Almudena Bocinos Rioboo
+# This class provided the methods to apply actions to sequences
+ #########################################
+require 'seqtrim_action' 
+class ActionManager 
+  
+  #Storages the necessary plugins specified in 'plugin_list' and start the loading of plugins
+  def initialize 
+    
+    load_actions_from_files
+  end
+  
+  def self.new_action(start_pos,end_pos,action_type) 
+     action_class = Object.const_get(action_type)
+    # DONE mirar si la action_class es de verdad una action existente 
+     res = nil     
+     if !action_class.nil? && action_class.ancestors.include?(SeqtrimAction)
+       res= action_class.new(start_pos,end_pos)
+     else
+       #$LOG.error ' Error. Don´t exist the action: ' + action_class.to_s
+       puts ' Error. The action : ' + action_class.to_s+ ' does not exists'
+     end
+     return res
+  end
+  
+  
+  
+  
+  # Iterates by the files from the folder 'actions', and load it
+  def load_actions_from_files
+    ignore = ['.','..','seqtrim_action.rb']
+    #carpeta=Dir.open("progs/ruby/seqtrimii/actions")
+    actions_path = File.expand_path(File.join(File.dirname(__FILE__), "..","actions"))
+    if !File.exists?(actions_path)
+    	raise "Action folder does not exists"
+    end
+    carpeta=Dir.open(actions_path)
+
+    carpeta.entries.each do |action|
+      if !ignore.include?(action)
+          require action
+      end # end  if
+    end # end  each
+  end # end  def
+  
+end

data/lib/seqtrimnext/classes/em_classes/seqtrim_work_manager.rb

@@ -0,0 +1,335 @@
+require 'scbi_fasta'
+require 'scbi_fastq'
+# require 'work_manager'
+require 'graph_stats'
+require 'sequence_with_action'
+require 'sequence_group'
+
+# OUTPUT_PATH='output_files'
+STATS_PATH=File.join(OUTPUT_PATH,'stats.json')
+# TODO - Pasar secuencias en grupos, y hacer blast en grupos de secuencias.
+
+
+
+class SeqtrimWorkManager < ScbiMapreduce::WorkManager
+
+  def self.init_work_manager(sequence_reader, params, chunk_size = 100, use_json=false)
+    @@full_stats={}
+    @@params= params
+    @@exit = false
+
+    @@chunk_size = chunk_size
+
+
+    # puts "CHECKPOINT: #{self.checkpoint}\n"*20
+
+    checkpoint_exists=File.exists?('scbi_drb_checkpoint')
+
+    # @@use_qual = !qual_path.nil? and File.exists?(qual_path)
+    @@open_mode='w'
+    if checkpoint_exists
+      @@open_mode = 'a'
+      if File.exists?(STATS_PATH)
+        # load stats
+        text = File.read(STATS_PATH)
+
+        # wipe text
+        # text=text.grep(/^\s*[^#]/).to_s
+
+        # decode json
+        @@full_stats = JSON.parse(text)
+      end
+    end
+
+    #open input file
+    @@fqr=sequence_reader
+
+    # @@use_qual = @@fqr.with_qual?
+    # @@use_json = use_json
+
+    @@params.set_param('use_qual',@@fqr.with_qual?)
+    @@params.set_param('use_json',use_json)
+
+    @@use_json=use_json
+
+    @@fqr.rewind
+
+    # open output files
+
+    if !Dir.exists?(OUTPUT_PATH)
+      Dir.mkdir(OUTPUT_PATH)
+    end
+
+    @@files={}
+
+    # @@rejected_output_file=File.open(File.join(OUTPUT_PATH,'rejected.txt'),@@open_mode)
+
+    # seqs_with_errors
+    @@errors_file=File.open('errors.txt',@@open_mode)
+
+    if @@use_json
+      @@json_output=File.open('results.json',@@open_mode)
+    end
+
+    @@json_separator=''
+
+    @@paired_output_files={}
+
+    @@sequences_output_files={}
+
+    @@low_complexity_output_files={}
+
+    @@sffinfo_files={}
+
+    @@low_sffinfo_files={}
+
+  end
+
+  def self.end_work_manager
+
+    puts "FULL STATS:\n" +JSON.pretty_generate(@@full_stats)
+
+    f = File.open(STATS_PATH,'w')
+    f.puts JSON.pretty_generate(@@full_stats)
+    f.close
+
+    r=File.read(STATS_PATH)
+
+    stats=JSON::parse(r)
+
+
+    gs=GraphStats.new(stats)
+
+    #gs=GraphStats.new(@@full_stats)
+
+
+    #close all files
+
+    # @@fqr.close
+    if @@use_json
+      @@json_output.close
+    end
+    @@errors_file.close
+    # @@rejected_output_file.close
+
+    # @@paired_output_files.each do |k,file|
+    #   file.close
+    # end
+
+    @@files.each do |k,file|
+      file.close
+    end
+
+    # @@paired_qual_output_files.each do |k,file|
+    #            file.close
+    # end
+
+    # @@sequences_output_files.each do |k,file|
+    #   file.close
+    # end
+    #
+    # @@low_complexity_output_files.each do |k,file|
+    #   file.close
+    # end
+    #
+    # @@sffinfo_files.each do |k,file|
+    #   file.close
+    # end
+    #
+    # @@low_sffinfo_files.each do |k,file|
+    #   file.close
+    # end
+
+
+    # @@qual_output_files.each do |k,file|
+    #  file.close
+    # end
+
+    # more than one MID found
+    # if @@full_stats['mid_id'] && @@full_stats['mid_id'].count>1
+    #
+    # end
+
+    if File.exists?('scbi_drb_checkpoint')
+      File.delete('scbi_drb_checkpoint')
+    end
+  end
+
+
+  def error_received(worker_error, obj)
+    @@errors_file.puts "Error while processing object #{obj.inspect}\n" + worker_error.original_exception.message + ":\n" +worker_error.original_exception.backtrace.join("\n")
+    @@errors_file.puts "="*60
+  end
+
+  def too_many_errors_received
+    $LOG.error "Too many errors: #{@@error_count} errors on #{@@count} executed sequences, exiting before finishing"
+  end
+
+  def worker_initial_config
+    return @@params
+  end
+
+  def load_user_checkpoint(checkpoint)
+
+    # reset count stats since they are repeated by checkpointing
+
+    # if @@full_stats['sequences'] && @@full_stats['repeated']
+    #   @@full_stats['sequences']['count']['repeated']=0
+    # end
+    #
+    # if @@full_stats['sequences'] && @@full_stats['processed']
+    #   @@full_stats['sequences']['processed']['count']=0
+    # end
+    #
+    # if @@full_stats['sequences'] && @@full_stats['total']
+    #   @@full_stats['sequences']['total']['count']=0
+    # end
+
+    super
+    # return checkpoint
+  end
+
+  def save_user_checkpoint
+
+    f = File.open(STATS_PATH,'w')
+    f.puts JSON.pretty_generate(@@full_stats)
+    f.close
+
+  end
+
+
+  # read a work that will not be processed, only to skip until checkpoint
+  def trash_checkpointed_work
+
+    @@chunk_size.times do
+      begin
+        n,f,q,c = @@fqr.next_seq
+      end while (!n.nil? && @@params.repeated_seq?(n))
+
+      if n.nil?
+        break
+      end
+    end
+
+  end
+
+  def next_work
+
+    if @@exit
+      return nil
+    end
+
+    begin
+
+      n,f,q,c = @@fqr.next_seq
+
+      if !n.nil? && @@params.repeated_seq?(n)
+        @@full_stats.add_stats({'sequences' => {'count' => {'rejected' => 1}}})
+        @@full_stats.add_stats({'sequences' => {'rejected' => {'repeated' => 1}}})
+
+        get_file(File.join(OUTPUT_PATH,'rejected.txt')).puts('>'+n+ ' repeated')
+        
+      end
+      
+      if !n.nil?
+        @@full_stats.add_stats({'sequences' => {'count' => {'input_count' => 1}}})
+      end
+    end while (!n.nil? && @@params.repeated_seq?(n))
+
+    if !n.nil?
+      return SequenceWithAction.new(n,f.upcase,q,c)
+    else
+      return nil
+    end
+
+
+  end
+
+  # def next_work
+  #
+  #     if @@exit
+  #       return nil
+  #     end
+  #     group = SequenceGroup.new
+  #
+  #     @@chunk_size.times do
+  #       begin
+  #
+  #         n,f,q,c = @@fqr.next_seq
+  #
+  #         if !n.nil? && @@params.repeated_seq?(n)
+  #           @@full_stats.add_stats({'sequences' => {'count' => {'rejected' => 1}}})
+  #           @@full_stats.add_stats({'sequences' => {'rejected' => {'repeated' => 1}}})
+  #
+  #           get_file(File.join(OUTPUT_PATH,'rejected.txt')).puts('>'+n+ ' repeated')
+  #
+  #         end
+  #         if !n.nil?
+  #           @@full_stats.add_stats({'sequences' => {'count' => {'input_count' => 1}}})
+  #         end
+  #       end while (!n.nil? && @@params.repeated_seq?(n))
+  #
+  #       if !n.nil?
+  #         # @@full_stats.add_stats({'sequences' => {'count' => {'processed' => 1}}})
+  #         group.push SequenceWithAction.new(n,f.upcase,q,c)
+  #       else
+  #         break
+  #       end
+  #     end
+  #
+  #     # puts "Processing #{group.inspect}"
+  #
+  #     if group.empty?
+  #       return nil
+  #     else
+  #       return group
+  #     end
+  #
+  #   end
+
+  def work_received(obj)
+
+    res = obj
+
+    # collect stats
+    @@full_stats.add_stats(obj.stats)
+
+    # print output in screen
+    puts obj.output_text
+
+    # save results to files
+    save_files(obj)
+
+  end
+
+  def save_files(obj)
+    files=obj.output_files
+    files.each do |file_name,content|
+      f=get_file(file_name)
+      f.puts content
+    end
+  end
+
+  def get_file(file_name)
+    res_file = @@files[file_name]
+
+    # if file is not already open, create it
+    if res_file.nil?
+      # create dir if necessary
+      dir = File.dirname(file_name)
+      if !File.exists?(dir)
+        FileUtils.mkdir_p(dir)
+      end
+
+      # open file
+      res_file=File.open(file_name,@@open_mode)
+
+      # save it in hash for next use
+      @@files[file_name]=res_file
+    end
+
+    return res_file
+  end
+
+
+end