seqtrimnext 2.0.29

data/bin/seqtrimnext

@@ -0,0 +1,368 @@
+#!/usr/bin/env ruby
+#     SeqTrimNext: Next generation sequencing preprocessor
+#     Copyright (C) <2011>
+#     Authors: Almudena Bocinos Rioboo, Diego Dario Guerrero Fernandez,
+#              Rocio Bautista Moreno, Juan Falgueras Cano & M. Gonzalo Claros
+#              email: soporte@scbi.uma.es - http://www.scbi.uma.es
+#
+#     This program is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU Affero General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     This program is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU Affero General Public License for more details.
+#
+#     You should have received a copy of the GNU Affero General Public License
+#     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#= SEQTRIM II
+#
+#== Running
+#
+# Seqtrim can be run locally or in a parallel/distributted environment.
+#
+#=== Running locally
+#* list
+#
+#=== Running in a distributted environment
+#
+#== SEC 2
+#
+#=== SUB 2.1
+#
+
+
+# #finds the classes that were in the folder 'classes'
+# ROOT_PATH=File.dirname(__FILE__)
+# $: << File.expand_path(File.join(ROOT_PATH, 'classes'))
+#
+# #finds the classes that were in the folder 'plugins'
+# $: << File.expand_path(File.join(ROOT_PATH, 'plugins'))
+#
+#
+# #finds the classes that were in the folder 'plugins'
+# $: << File.expand_path(File.join(ROOT_PATH, 'actions'))
+#
+# #finds the classes that were in the folder 'utils'
+# $: << File.expand_path(File.join(ROOT_PATH, 'utils'))
+#
+# $: << File.expand_path(File.join(ROOT_PATH, 'classes','em_classes'))
+
+# to test scbi_drb gem locally
+# $: << File.expand_path('~/progs/ruby/gems/scbi_drb/lib/')
+
+# $: << File.expand_path(ROOT_PATH)
+
+$: << File.expand_path('~/progs/ruby/gems/seqtrimnext/lib/')
+
+require 'seqtrimnext'
+
+############ PATHS #######################
+$SEQTRIM_PATH = ROOT_PATH
+
+# if there is a BLASTDB environment var, then use it
+if ENV['BLASTDB']# && Dir.exists?(ENV['BLASTDB'])
+  $FORMATTED_DB_PATH = ENV['BLASTDB']
+  $DB_PATH = File.dirname($FORMATTED_DB_PATH)
+else # otherwise use ROOTPATH + DB
+  $FORMATTED_DB_PATH = File.expand_path(File.join(ROOT_PATH, "DB",'formatted'))
+  $DB_PATH = File.expand_path(File.join(ROOT_PATH, "DB"))
+end
+
+ENV['BLASTDB']=$FORMATTED_DB_PATH
+
+OUTPUT_PATH='output_files'
+
+
+# TODO - COMENTAR todas las clases y metodos para que salga la descripcion cuando hagas rdoc en el terminal
+
+#Checks install requeriments
+require 'install_requirements'
+
+ins = InstallRequirements.new
+if (!ins.check_install_requirements)
+  exit
+end
+
+require "logger"
+require 'optparse'
+require "global_match"
+require "seqtrim"
+require "params.rb"
+require "plugin.rb"
+require "sequence.rb"
+require "plugin_manager.rb"
+require "make_blast_db"
+require 'hash_stats'
+require 'list_db'
+require 'install_database'
+require 'socket'
+
+
+def show_additional_help
+
+  puts "\n"*3
+  puts "E.g.: processing a fastq sequences file"
+  puts "#{$0} -t genomics_454.txt -Q sequences.fastq"
+  puts "\n"*2
+
+  puts "E.g.: processing a fasta file with qual"
+  puts "#{$0} -t genomics_454.txt -f sequences.fasta -q sequences.qual"
+
+  templates = Dir.glob(File.join($SEQTRIM_PATH,'templates','*.txt')).map{|t| File.basename(t)}
+
+  puts "\n\n  ========================================================================================================"
+  puts "  Available templates to use with -t option (you can also use your own template):"
+  puts "  Templates at: #{File.join($SEQTRIM_PATH,'templates')}"
+  puts "  ========================================================================================================\n\n"
+
+  templates.map{|e| puts "      "+e}
+
+  puts "\n\n  ========================================================================================================"
+  puts "  Available databases to use in custom template files (you can also use your own database):"
+  puts "  Databases at: #{$DB_PATH}"
+  puts "  ========================================================================================================\n\n"
+
+  ListDb.list_databases($DB_PATH).map{|e| puts "      "+e}
+  #
+  # ip_list = Socket.ip_address_list.select{|e| e.ipv4?}.map{|e| e.ip_address}
+  #
+  # puts ip_list
+
+
+  exit
+
+end
+
+
+# Reads the parameters from console. For this is used ARGV, that is an array.
+options = {}
+
+optparse = OptionParser.new do |opts|
+
+  # Set a banner, displayed at the top
+  # of the help screen.
+  opts.banner = "Usage: #{$0} -t template_file \{-Q fastaQ_file | -f fasta_file -q qual_file\} [options]"
+
+  # Define the options, and what they do
+  #options[:server_ip] = '127.0.0.1'
+  options[:server_ip] = '0.0.0.0'
+  opts.on( '-s', '--server IP', 'Server ip. Can use a partial ip to select the apropriate interface' ) do |server_ip|
+
+    # get list of available ips
+    ip_list = Socket.ip_address_list.select{|e| e.ipv4?}.map{|e| e.ip_address}
+
+    ip=ip_list.select{|ip| ip.index(server_ip)==0}.first
+
+    if !ip
+      ip='0.0.0.0'
+      # $LOG.info("No available ip matching #{server_ip}")
+    end
+    # $ .info("Using ip #{ip}")
+    options[:server_ip] = ip
+  end
+
+  options[:port] = 0 #50000
+  opts.on( '-p', '--port PORT', 'Server port. If set to 0, an arbitrary empty port will be used') do |port|
+    options[:port] = port.to_i
+  end
+
+  options[:workers] = 2
+
+  opts.on( '-w', '--workers COUNT', 'Number of workers, or file containing machine names to launch workers with ssh' ) do |workers|
+    if File.exists?(workers)
+      # use workers file
+      options[:workers] = File.read(workers).split("\n").map{|w| w.chomp}
+    else
+      begin
+        options[:workers] = Integer(workers)
+      rescue
+        STDERR.puts "ERROR:Invalid workers parameter #{options[:workers]}"
+        exit
+      end
+
+    end
+    
+  end
+
+
+
+  options[:only_workers] = false
+  opts.on( '-o', '--only_workers', 'Only launch workers' ) do
+    options[:only_workers] = true
+  end
+
+  options[:check_db] = false
+  opts.on( '-c', '--check_databases', 'Check Blast databases and reformat if necessary' ) do
+    options[:check_db] = true
+  end
+
+  options[:install_db] = nil
+  opts.on( '-i', '--install_databases TYPE', 'Install base databases and reformat them if necessary') do |db_type|
+    options[:install_db] = db_type
+  end
+
+  options[:logfile] = STDOUT
+  opts.on( '-l', '--logfile FILE', 'Write log to FILE' ) do |file|
+    options[:logfile] = file
+  end
+
+  options[:fastq] = nil
+  opts.on( '-Q', '--fastq FILE', 'Fastq input file. Use - for <STDIN>' ) do |file|
+    options[:fastq] = file
+  end
+
+  options[:fasta] = nil
+  opts.on( '-f', '--fasta FILE', 'Fasta input file' ) do |file|
+    options[:fasta] = file
+  end
+
+  options[:qual] = nil
+  opts.on( '-q', '--qual FILE', 'Qual input file' ) do |file|
+    options[:qual] = file
+  end
+
+
+
+  options[:list_db] = nil
+  opts.on( '-L', '--list_db DB_NAME', 'List entries IDs in DB_NAME. Use "-L all" to view all available databases' ) do |value|
+    options[:list_db] = value
+  end
+
+  options[:gen_params] = false
+  opts.on( '-G', '--generate_template', 'Generates a sample template file with default parameters' ) do
+    options[:gen_params] = true
+  end
+
+  options[:template] = nil
+  opts.on( '-t', '--template TEMPLATE_FILE', 'Use TEMPLATE_FILE instead of default parameters' ) do |file|
+    options[:template] = file
+  end
+
+  options[:chunk_size] = 100
+  opts.on( '-g', '--group_size chunk_size', 'Group sequences in chunks of size <chunk_size>' ) do |cs|
+    options[:chunk_size] = cs.to_i
+  end
+
+
+  options[:json] = nil
+  opts.on( '-j', '--json', 'Save results in json file' ) do
+    options[:json] = true
+  end
+
+  # This displays the help screen, all programs are
+  # assumed to have this option.
+  opts.on_tail( '-h', '--help', 'Display this screen' ) do
+    puts opts
+    show_additional_help
+    exit
+  end
+end
+
+# parse options and remove from ARGV
+optparse.parse!
+
+if options[:list_db] then
+  # List database entries in a database
+  ListDb.new($DB_PATH,options[:list_db])
+  exit
+end
+
+if options[:gen_params] then
+  # Generates a sample params file in current directory
+  Params.generate_sample_params
+  exit
+end
+
+#set logger
+# system('rm logs/*')
+FileUtils.mkdir('logs') if !File.exists?('logs')
+
+$LOG = Logger.new(options[:logfile])
+$LOG.datetime_format = "%Y-%m-%d %H:%M:%S"
+#logger.level = Logger::INFO
+
+#DEBUG < INFO < WARN < ERROR < FATAL < UNKNOWN
+
+
+$LOG.info("Using BLASTDB: "+ $FORMATTED_DB_PATH)
+$LOG.info("Using options: "+ options.to_json)
+
+if options[:install_db] then
+  #install databases
+  InstallDatabase.new(options[:install_db],$DB_PATH)
+  
+  # reformat databases
+  MakeBlastDb.new($DB_PATH)
+  exit
+end
+
+if !File.exists?($FORMATTED_DB_PATH)
+  STDERR.puts "Database path not found: #{$FORMATTED_DB_PATH}. \n\n\nInstall databases to this path or set your BLASTDB environment variable (eg.: export BLASTDB=new_path)"
+  exit
+end
+
+
+if options[:check_db] then
+  # check and format blast databases
+  MakeBlastDb.new($DB_PATH)
+  exit
+end
+
+required_options = options[:template] && (options[:fastq] || (options[:fasta]))
+
+# if ((ARGV.count != 2) && (ARGV.count != 3)) # con esto vemos si hay argumentos,
+if (ARGV.count != 0) || (!required_options) # con esto vemos si hay argumentos,
+  puts "You must provide all required options"
+  puts ""
+  puts optparse.help
+  exit
+end
+
+# check for template
+if (!File.exists?(options[:template]))
+  if File.exists?(File.join($SEQTRIM_PATH,'templates',options[:template]))
+    options[:template] = File.join($SEQTRIM_PATH,'templates',options[:template])
+  else
+    $LOG.info "Params file: #{options[:template]} doesn't exists. \n\nYou can use your own template or specify one from this list:\n============================="
+    puts Dir.glob(File.join($SEQTRIM_PATH,'templates','*.txt')).map{|t| File.basename(t)}
+    exit
+  end
+end
+$LOG.info "Using params file: #{options[:template]}"
+
+# fastq file
+if (!options[:fastq].nil? && options[:fastq]!='-' && !File.exists?(options[:fastq]))
+  $LOG.error "Input file: #{options[:fasta]} doesn't exists"
+  exit
+end
+
+
+
+# fasta file
+if (!options[:fasta].nil? && !File.exists?(options[:fasta]))
+  $LOG.error "Input file: #{options[:fasta]} doesn't exists"
+  exit
+end
+
+# qual file
+if ((!options[:qual].nil?)&&(!File.exists?(options[:qual])))
+  $LOG.error "Input file: #{options[:qual]} doesn't exists"
+  exit
+end
+
+s = Seqtrim.new(options)
+
+#generate report
+
+
+if system("which generate_report.rb > /dev/null ")
+  cmd="generate_report.rb output_files 2> report_generation_errors.log"
+  $LOG.info "Generating report #{cmd}"
+  `#{cmd}`
+else
+  $LOG.info "If you want a detailed report in PDF format, install the optional seqtrimnext_report gem (gem install seqtrimnext_report)"
+end

data/bin/split_fastq.rb

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+
+require 'scbi_fastq'
+
+
+if ARGV.count < 3
+  puts "#{$0} FASTQ OUTPUT_NAME SPLIT_BY"
+  exit
+end
+
+
+  
+fastq = ARGV.shift
+output_name = ARGV.shift
+split_by = ARGV.shift.to_i
+
+
+file_index=1
+out=FastqFile.new("#{output_name}#{file_index}.fastq",'w')
+
+fqr=FastqFile.new(fastq)
+
+count = 0
+
+fqr.each do |seq_name,seq_fasta,seq_qual,comments|
+  
+  out.write_seq(seq_name,seq_fasta,seq_qual,comments)
+
+  count +=1
+
+  if (count % split_by) == 0 
+      
+    file_index +=1
+    out.close
+    out=FastqFile.new("#{output_name}#{file_index}.fastq",'w')
+
+  end
+end
+
+out.close
+fqr.close
+

data/bin/split_ilumina_paired.rb

@@ -0,0 +1,65 @@
+#!/usr/bin/env ruby
+
+# Splits a FastQ file with ilumina paired data into two separate files.
+
+require 'scbi_fastq'
+
+VERBOSE=false
+
+if !(ARGV.count==2 or ARGV.count==4)
+  puts "Usage: #{$0} paired.fastq output_name [pared1_tag paired2_tag]"
+  exit
+end
+
+p1_path=ARGV[0]
+output_base_name=ARGV[1]
+
+paired1_tag='/1'
+paired2_tag='/2'
+
+if (ARGV.count==4)
+  paired1_tag=ARGV[2]
+  paired2_tag=ARGV[3]
+end
+
+PAIRED1_TAG_RE=/#{Regexp.quote(paired1_tag)}$/
+PAIRED2_TAG_RE=/#{Regexp.quote(paired2_tag)}$/
+
+
+if !File.exists?(p1_path)
+  puts "File #{p1_path} doesn't exists"
+  exit
+end
+
+paired1_out = FastqFile.new(output_base_name+'_paired1.fastq','w',:sanger, true)
+paired2_out = FastqFile.new(output_base_name+'_paired2.fastq','w',:sanger, true)
+
+
+f_file = FastqFile.new(p1_path,'r',:sanger, true)
+
+f_file.each do |n,f,q,c|
+  
+  if n=~ PAIRED1_TAG_RE
+    paired1_out.write_seq(n,f,q,c)
+  elsif n=~ PAIRED2_TAG_RE
+    paired2_out.write_seq(n,f,q,c)
+  else
+    STDERR.puts "Aborting due to ERROR in file: #{n} doens't match neither left (#{paired1_tag}) nor right (#{paired2_tag}) tags"
+    exit
+  end
+  
+  if ((f_file.num_seqs%10000) == 0)
+    puts "Count: #{f_file.num_seqs}"
+  end
+  
+  
+end
+
+f_file.close
+
+paired1_out.close
+paired2_out.close
+
+
+
+

data/bin/split_paired.rb

@@ -0,0 +1,70 @@
+#!/usr/bin/env ruby
+
+
+require 'scbi_fasta'
+
+if ARGV.count!=3
+	puts "Usage: #{$0} fasta qual output_base_name"
+	exit
+end
+
+fasta_path = ARGV[0]
+qual_path = ARGV[1]
+name = ARGV[2]
+
+
+out_fasta = name+'.fasta'
+out_qual = name+'.fasta.qual'
+
+puts "Opening #{fasta_path}, #{qual_path}"
+
+fqr=FastaQualFile.new(fasta_path,qual_path,true)
+
+out_f=File.new(out_fasta,'w+')
+out_q=File.new(out_qual,'w+')
+
+c=0
+
+linker = 'TCGTATAACTTCGTATAATGTATGCTATACGAAGTTATTACG'
+
+fqr.each do |n,f,q|
+	l_start= 0
+	l_end=f.index(linker)
+
+	if l_end
+		r_start=l_end+linker.length
+		r_end =f.length
+		
+		forward=f[l_start..l_end-1]
+		reverse=f[r_start..r_end]
+		
+		forward_q = q[l_start..l_end-1]
+		reverse_q = q[r_start..r_end]
+		
+		if (forward.length!=forward_q.length) || (reverse.length!=reverse_q.length)
+			puts [forward.length,forward_q.length,reverse.length,reverse_q.length].join(',')
+		end
+		
+		out_f.puts ">#{n}F template=#{n} dir=F library=unadeellas"
+		out_f.puts forward 
+		out_f.puts ">#{n}R template=#{n} dir=R library=unadeellas"
+		out_f.puts reverse
+		
+		out_q.puts ">#{n}F template=#{n} dir=F library=unadeellas"
+		out_q.puts forward_q.join(' ') 
+		out_q.puts ">#{n}R template=#{n} dir=R library=unadeellas"
+		out_q.puts reverse_q.join(' ')
+		
+		 
+	end
+
+	c=c+1
+end
+
+puts c
+
+fqr.close
+
+out_f.close
+out_q.close
+

data/lib/seqtrimnext/actions/action_ab_adapter.rb

@@ -0,0 +1,32 @@
+require "seqtrim_action"
+
+########################################################
+# Author: Almudena Bocinos Rioboo                      
+# 
+# Defines the main methods that are necessary to execute Plugin1                                                     
+# Inherit: Plugin
+########################################################
+
+class ActionAbAdapter < SeqtrimAction
+      
+   def initialize(start_pos,end_pos)
+     super(start_pos,end_pos)    
+     @cut =true
+
+   end
+#    
+# def apply_to(seq)
+#   
+#    # seq.seq_fasta = seq.seq_fasta.slice(start_pos,end_pos)
+#    $LOG.debug " Applying #{self.class} . BEGIN: #{@start_pos}   END: #{@end_pos}  " 
+#   
+# end   
+
+def apply_decoration(char) 
+  return char.magenta.underline
+  
+  #  return char.magenta.negative
+end  
+
+
+end

data/lib/seqtrimnext/actions/action_ab_far_adapter.rb

@@ -0,0 +1,32 @@
+require "seqtrim_action"
+
+########################################################
+# Author: Almudena Bocinos Rioboo                      
+# 
+# Defines the main methods that are necessary to execute Plugin1                                                     
+# Inherit: Plugin
+########################################################
+
+class ActionAbFarAdapter < SeqtrimAction
+      
+   def initialize(start_pos,end_pos)
+     super(start_pos,end_pos)    
+     @cut =true
+
+   end
+#    
+# def apply_to(seq)
+#   
+#    # seq.seq_fasta = seq.seq_fasta.slice(start_pos,end_pos)
+#    $LOG.debug " Applying #{self.class} . BEGIN: #{@start_pos}   END: #{@end_pos}  " 
+#   
+# end   
+
+def apply_decoration(char) 
+  return char.magenta.negative
+  
+  #  return char.magenta.negative
+end  
+
+
+end

data/lib/seqtrimnext/actions/action_ab_left_adapter.rb

@@ -0,0 +1,32 @@
+require "seqtrim_action"
+
+########################################################
+# Author: Almudena Bocinos Rioboo                      
+# 
+# Defines the main methods that are necessary to execute Plugin1                                                     
+# Inherit: Plugin
+########################################################
+
+class ActionAbLeftAdapter < SeqtrimAction
+      
+   def initialize(start_pos,end_pos)
+     super(start_pos,end_pos)    
+     @cut =true
+
+   end
+#    
+# def apply_to(seq)
+#   
+#    # seq.seq_fasta = seq.seq_fasta.slice(start_pos,end_pos)
+#    $LOG.debug " Applying #{self.class} . BEGIN: #{@start_pos}   END: #{@end_pos}  " 
+#   
+# end   
+
+def apply_decoration(char) 
+  return char.magenta.negative
+  
+  #  return char.magenta.negative
+end  
+
+
+end

data/lib/seqtrimnext/actions/action_empty_insert.rb

@@ -0,0 +1,22 @@
+require "seqtrim_action"
+
+########################################################
+# Author: Almudena Bocinos Rioboo                      
+# 
+# Defines the main methods that are necessary to execute ActionShortInserted                                                     
+# Inherit: Plugin
+########################################################
+
+class ActionEmptyInsert < SeqtrimAction
+
+  def initialize(start_pos,end_pos)
+    super(start_pos,end_pos)    
+    @cut =false
+    @informative = true
+  end
+
+def apply_decoration(char)
+  return char
+end             
+              
+end

data/lib/seqtrimnext/actions/action_ignore_repeated.rb

@@ -0,0 +1,24 @@
+require "seqtrim_action" 
+
+
+########################################################
+# Author: Almudena Bocinos Rioboo                      
+# 
+# Defines the main methods that are necessary to execute Plugin1                                                     
+# Inherit: Plugin
+########################################################
+
+class ActionIgnoreRepeated < SeqtrimAction
+   
+  def initialize(start_pos,end_pos)
+    super(start_pos,end_pos)    
+    @cut =false
+  end   
+
+
+def apply_decoration(char) 
+  return char
+  #  return char.magenta.negative
+end  
+  
+end