seqtrimnext 2.0.29

data/History.txt

@@ -0,0 +1,3 @@
+=== 2.0.29 2011-06-06
+
+* Initial public release on rubygems

data/Manifest.txt

@@ -0,0 +1,114 @@
+bin/create_graphs.rb
+bin/extract_seqs.rb
+bin/extract_seqs_from_fasta.rb
+bin/extract_seqs_from_fastq.rb
+bin/fasta2fastq.rb
+bin/fastq2fasta.rb
+bin/gen_qual.rb
+bin/get_seq.rb
+bin/group_by_range.rb
+bin/join_ilumina_paired.rb
+bin/parse_amplicons.rb
+bin/parse_json_results.rb
+bin/parse_params.rb
+bin/resume_clusters.rb
+bin/resume_rejected.sh
+bin/reverse_paired.rb
+bin/seqtrimnext
+bin/split_fastq.rb
+bin/split_ilumina_paired.rb
+bin/split_paired.rb
+lib/seqtrimnext/actions/action_ab_adapter.rb
+lib/seqtrimnext/actions/action_ab_far_adapter.rb
+lib/seqtrimnext/actions/action_ab_left_adapter.rb
+lib/seqtrimnext/actions/action_empty_insert.rb
+lib/seqtrimnext/actions/action_ignore_repeated.rb
+lib/seqtrimnext/actions/action_indetermination.rb
+lib/seqtrimnext/actions/action_induced_low_complexity.rb
+lib/seqtrimnext/actions/action_insert.rb
+lib/seqtrimnext/actions/action_is_contaminated.rb
+lib/seqtrimnext/actions/action_key.rb
+lib/seqtrimnext/actions/action_left_adapter.rb
+lib/seqtrimnext/actions/action_left_primer.rb
+lib/seqtrimnext/actions/action_linker.rb
+lib/seqtrimnext/actions/action_low_complexity.rb
+lib/seqtrimnext/actions/action_low_high_size.rb
+lib/seqtrimnext/actions/action_low_quality.rb
+lib/seqtrimnext/actions/action_mid.rb
+lib/seqtrimnext/actions/action_multiple_linker.rb
+lib/seqtrimnext/actions/action_paired_reads.rb
+lib/seqtrimnext/actions/action_poly_a.rb
+lib/seqtrimnext/actions/action_poly_t.rb
+lib/seqtrimnext/actions/action_rem_adit_artifacts.rb
+lib/seqtrimnext/actions/action_right_adapter.rb
+lib/seqtrimnext/actions/action_right_primer.rb
+lib/seqtrimnext/actions/action_short_insert.rb
+lib/seqtrimnext/actions/action_unexpected_poly_t.rb
+lib/seqtrimnext/actions/action_unexpected_vector.rb
+lib/seqtrimnext/actions/action_vectors.rb
+lib/seqtrimnext/actions/seqtrim_action.rb
+lib/seqtrimnext/classes/action_manager.rb
+lib/seqtrimnext/classes/em_classes/seqtrim_work_manager.rb
+lib/seqtrimnext/classes/em_classes/seqtrim_worker.rb
+lib/seqtrimnext/classes/extract_stats.rb
+lib/seqtrimnext/classes/gnu_plot_graph.rb
+lib/seqtrimnext/classes/graph_stats.rb
+lib/seqtrimnext/classes/install_database.rb
+lib/seqtrimnext/classes/install_requirements.rb
+lib/seqtrimnext/classes/list_db.rb
+lib/seqtrimnext/classes/make_blast_db.rb
+lib/seqtrimnext/classes/one_blast.rb
+lib/seqtrimnext/classes/params.rb
+lib/seqtrimnext/classes/piro.rb
+lib/seqtrimnext/classes/plugin_manager.rb
+lib/seqtrimnext/classes/scan_for_restr_site.rb
+lib/seqtrimnext/classes/scbi_stats.rb
+lib/seqtrimnext/classes/seqtrim.rb
+lib/seqtrimnext/classes/sequence.rb
+lib/seqtrimnext/classes/sequence_group.rb
+lib/seqtrimnext/classes/sequence_with_action.rb
+lib/seqtrimnext/plugins/plugin.rb
+lib/seqtrimnext/plugins/plugin_ab_adapters.rb
+lib/seqtrimnext/plugins/plugin_adapters.rb
+lib/seqtrimnext/plugins/plugin_amplicons.rb
+lib/seqtrimnext/plugins/plugin_contaminants.rb
+lib/seqtrimnext/plugins/plugin_extract_inserts.rb
+lib/seqtrimnext/plugins/plugin_find_poly_at.rb
+lib/seqtrimnext/plugins/plugin_ignore_repeated.rb
+lib/seqtrimnext/plugins/plugin_indeterminations.rb
+lib/seqtrimnext/plugins/plugin_key.rb
+lib/seqtrimnext/plugins/plugin_linker.rb
+lib/seqtrimnext/plugins/plugin_low_complexity.rb
+lib/seqtrimnext/plugins/plugin_low_high_size.rb
+lib/seqtrimnext/plugins/plugin_low_quality.rb
+lib/seqtrimnext/plugins/plugin_mids.rb
+lib/seqtrimnext/plugins/plugin_rem_adit_artifacts.rb
+lib/seqtrimnext/plugins/plugin_short_insert.rb
+lib/seqtrimnext/plugins/plugin_vectors.rb
+lib/seqtrimnext/templates/amplicons.txt
+lib/seqtrimnext/templates/genomics_454.txt
+lib/seqtrimnext/templates/genomics_454_with_paired.txt
+lib/seqtrimnext/templates/low_quality.txt
+lib/seqtrimnext/templates/low_quality_and_low_complexity.txt
+lib/seqtrimnext/templates/transcriptomics_454.txt
+lib/seqtrimnext/templates/transcriptomics_plants.txt
+lib/seqtrimnext/utils/extract_samples.rb
+lib/seqtrimnext/utils/fasta2xml.rb
+lib/seqtrimnext/utils/global_match.rb
+lib/seqtrimnext/utils/hash_stats.rb
+lib/seqtrimnext/utils/json_utils.rb
+lib/seqtrimnext/utils/load_fasta_names_in_hash.rb
+lib/seqtrimnext/utils/load_qual_in_hash.rb
+lib/seqtrimnext/utils/recover_mid.rb
+lib/seqtrimnext/utils/string_utils.rb
+lib/seqtrimnext.rb
+History.txt
+Manifest.txt
+PostInstall.txt
+Rakefile
+README.rdoc
+script/console
+script/destroy
+script/generate
+test/test_helper.rb
+test/test_seqtrimnext.rb

data/PostInstall.txt

@@ -0,0 +1,7 @@
+For more information on seqtrimnext, see http://www.scbi.uma.es
+
+Remember to install the following external prerequisites if you don't have them already installed.
+
+If you need to install core databases, issue the following command:
+
+seqtrimnext -i core

data/README.rdoc

@@ -0,0 +1,159 @@
+= seqtrimnext
+
+* http://www.scbi.uma.es/downloads
+
+== DESCRIPTION:
+
+SeqtrimNEXT is a customizable and distributed pre-processing software for NGS (Next Generation Sequencing) biological data. It makes use of scbi_mapreduce gem to be able to run in parallel and distributed environments. It is specially suited for Roche 454 (normal and paired-end) & Ilumina datasets, although it could be easyly adapted to any other situation.
+ 
+== FEATURES:
+
+* SeqtrimNEXT is very flexible since it's architecture is based on plugins.
+* You can add new plugins if needed.
+* SeqtrimNEXT uses scbi_mapreduce and thus is able to exploit all the benefits of a cluster environment. It also works in multi-core machines big shared-memory servers.
+
+== Default templates for genomics & transcriptomics are provided
+
+<b>genomics_454.txt</b>:: cleans genomics data from Roche 454 sequencer.
+<b>genomics_454_with_paired.txt</b>:: cleans genomic data from a paired-end experiment sequenced with a Roche 454 sequencer.
+<b>low_quality.txt</b>:: trims low quality.
+<b>low_quality_and_low_complexity.txt</b>:: trims low quality and low complexity.
+<b>transcriptomics_454.txt</b>:: cleans transcriptomics data from a Roche 454 sequencer.
+<b>transcriptomics_plants.txt</b>:: cleans transcriptomics data from a Roche 454 sequencer with extra databases for plants.
+<b>amplicons.txt</b>:: filters amplicons.
+  
+== You can define your own templates using a combination of available plugins:
+
+<b>PluginKey</b>:: to remove sequencing keys from 454 input sequences.
+<b>PluginMids</b>:: to remove MIDS (barcodes) from 454 sequences.
+<b>PluginLinker</b>:: splits sequences into two inserts when a valid linker is found (paired-end experiments only)
+<b>PluginAbAdapters</b>:: removes AB adapters from sequences using a predefined DB or one provided by the user.
+<b>PluginFindPolyAt</b>:: removes polyA and polyT from sequences.
+<b>PluginLowComplexity</b>:: filters sequences with low complexity regions
+<b>PluginAdapters</b>:: removes Adapters from sequences using a predefined DB or one provided by the user.
+<b>PluginLowHighSize</b>:: removes sequences too small or too big.
+<b>PluginVectors</b>:: remove vectors from sequences using a predefined database or one provided by the user.
+<b>PluginAmplicons</b>:: filters amplicons using user predefined primers.
+<b>PluginIndeterminations</b>:: removes indeterminations (N) from the sequence.
+<b>PluginLowQuality</b>:: eliminate low quality regions from sequences.
+<b>PluginContaminants</b>:: remove contaminants from sequences or rejects contaminated ones. It uses a core database, but it can be expanded with user provided ones.
+
+
+
+== SYNOPSIS:
+
+Once installed, SeqtrimNEXT is very easy to use:
+  
+To install core databases (it should be done at installation time):
+
+  $> seqtrimnext -i
+
+To perform an analisys using a predefined template with a FASTQ file format using 4 cpus:
+
+  $> seqtrimnext -t genomics_454.txt -Q input_file_in_FASTQ -w 4
+  
+To perform an analisys using a predefined template with a FASTQ file format:
+  
+  $> seqtrimnext -t genomics_454.txt -f input_file_in_FASTA -q input_file_in_QUAL
+
+
+To get additional help and list available templates and databases:
+
+  $> seqtrimnext -h
+  
+
+== REQUIREMENTS:
+
+* Ruby 1.9.2
+* Blast plus 2.24 or greater (prior versions have bugs that produces bad results)
+* [Optional] - GnuPlot version 4.4.2 or greater (prior versions may produce wrong graphs)
+* [Optional] - pdflatex - Optional, to produce a detailed report with results 
+
+== INSTALL:
+
+=== Installing Blast
+
+*Download the latest version of Blast+ from ftp://ftp.ncbi.nlm.nih.gov/blast/executables/release/LATEST/
+*You can also use a precompiled version if you like
+*To install from source, decompress the downloaded file, cd to the decompressed folder, and issue the following commands:
+
+  ./configure
+  make
+  sudo make install
+
+
+=== Installing Ruby 1.9
+
+*You can use RVM to install ruby:
+
+Install RVM:
+
+  $ bash < <(curl -s https://rvm.beginrescueend.com/install/rvm)
+
+Setup environment:  
+  
+  $ echo '[[ -s "$HOME/.rvm/scripts/rvm" ]] && . "$HOME/.rvm/scripts/rvm" # Load RVM function' >> ~/.bash_profile
+  
+Install ruby 1.9.2 (this can take a while):
+  
+  $ rvm install 1.9.2
+  
+Set it as the default:
+
+  $ rvm use 1.9.2 --default
+
+=== Install SeqtrimNEXT
+
+SeqtrimNEXT is very easy to install. It is distributed as a ruby gem:
+
+  gem install seqtrimnext
+  
+This will install seqtrimnext and all the required gems.
+
+=== Install and rebuild SeqtrimNext's core databases
+
+SeqtrimNEXT needs some core databases to work. To install them:
+
+  seqtrimnext -i core
+
+=== Database modifications
+
+Included databases will be usefull for a lot of people, but if you prefer, you can modify them, or add more elements to be search against your sequences. 
+
+You only need to drop new fasta files to each respective directory:
+
+DB/vectors to add more vectors
+DB/contaminants to add more contaminants
+etc...
+
+Once the databases has been modified, you will need to reformat them by issuing the following command:
+
+  seqtrimnext -c
+
+Modified databases will be rebuilt.
+
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2011 Almudena Bocinos & Dario Guerrero
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,38 @@
+require 'rubygems'
+gem 'hoe', '>= 2.1.0'
+require 'hoe'
+require 'fileutils'
+require './lib/seqtrimnext'
+
+Hoe.plugin :newgem
+# Hoe.plugin :website
+# Hoe.plugin :cucumberfeatures
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+$hoe = Hoe.spec 'seqtrimnext' do
+  self.developer 'Dario Guerrero & Almudena Bocinos', 'dariogf@gmail.com & alkoke@gmail.com'
+  self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
+  self.rubyforge_name       = self.name # TODO this is default value
+  # self.extra_deps         = ['narray','gnuplot','term-ansicolor','xml-simple','scbi_blast','scbi_drb','scbi_fasta','scbi_fastq','scbi_plot','scbi_math']
+  
+  self.extra_deps         = []
+  self.extra_deps << ['narray','>=0']
+  self.extra_deps << ['gnuplot','>=0']
+  self.extra_deps << ['term-ansicolor','>=0']
+  self.extra_deps << ['xml-simple','>=0']
+  self.extra_deps << ['scbi_blast','>=0']
+  self.extra_deps << ['scbi_mapreduce','>=0']
+  self.extra_deps << ['scbi_fasta','>=0']
+  self.extra_deps << ['scbi_fastq','>=0']
+  self.extra_deps << ['scbi_plot','>=0']
+  self.extra_deps << ['scbi_math','>=0']
+
+end
+
+require 'newgem/tasks'
+Dir['tasks/**/*.rake'].each { |t| load t }
+
+# TODO - want other tests/tasks run by default? Add them to the list
+# remove_task :default
+# task :default => [:spec, :features, :redocs]

data/bin/create_graphs.rb

@@ -0,0 +1,46 @@
+require 'stringio'
+# require 'test/unit'
+require 'json'
+require 'gnuplot'
+
+ROOT_PATH=File.dirname(File.dirname(__FILE__))
+
+# $: << File.expand_path(File.join(ROOT_PATH,'test'))
+$: << File.expand_path(File.join(ROOT_PATH,'classes'))
+$: << File.expand_path(File.join(ROOT_PATH,'plugins'))
+$: << File.expand_path(File.join(ROOT_PATH,'utils'))
+
+if ARGV.empty?
+puts "Usage: #{$0} stats.json initial_stats.json"
+exit
+end
+
+d=Dir.glob(File.expand_path(File.join(ROOT_PATH,'plugins','*.rb')))
+
+# puts d.entries
+# puts "="*20
+
+require 'plugin'
+
+# require 'params'
+
+d.entries.each do |plugin|
+	require  plugin
+  # puts "Requiring #{plugin}"
+end
+
+require 'graph_stats'
+
+#load stats
+
+r=File.read(ARGV[0])
+stats=JSON::parse(r)
+
+
+r2=File.read(ARGV[1])
+init_stats=JSON::parse(r2)
+
+gs=GraphStats.new(stats,init_stats)
+
+puts "Graphs generated"
+    

data/bin/extract_seqs.rb

@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby
+
+require 'scbi_fastq'
+
+class Array
+ def count
+	self.length
+ end
+
+end
+
+if ARGV.count < 3
+  puts "#{$0} FASTA OUTPUT_NAME SEQ_NAME_FILE [MORE_SEQ_NAMES]"
+  exit
+else
+  
+  fasta = ARGV.shift
+  qual = ARGV.shift
+  output_name = ARGV.shift
+  seqs=ARGV
+  puts seqs.join(';')
+  
+	fqr=FastaQualFile.new(fasta,qual)
+	
+	output_fasta=File.new(output_name+'.fasta','a')
+	output_qual=File.new(output_name+'.fasta.qual','a')
+
+	fqr.each do |seq_name,seq_fasta,seq_qual|
+  	if seqs.index(seq_name)
+  	    output_fasta.puts ">#{seq_name}"
+  			output_fasta.puts seq_fasta
+  	    output_qual.puts ">#{seq_name}"
+  			output_qual.puts seq_qual
+  			seqs.delete(seq_name)
+  			if seqs.empty?
+  			  break
+			  end
+  	end
+  end
+  
+  output_qual.close
+  output_fasta.close
+  fqr.close
+
+end

data/bin/extract_seqs_from_fasta.rb

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+
+require 'scbi_fasta'
+
+# GOOD_QUAL=50
+# BAD_QUAL=10
+# DOWN_CASE=('a'..'z')
+
+
+class Array
+ def count
+	self.length
+ end
+
+end
+
+if ARGV.count < 4
+  puts "#{$0} FASTA QUAL OUTPUT_NAME SEQ_NAMES_FILE"
+  exit
+else
+  
+  fasta = ARGV.shift
+  qual = ARGV.shift
+  output_name = ARGV.shift
+  seqs_file=ARGV.shift
+
+  seqs=[]
+
+  f=File.open(seqs_file).each_line do |line|
+  	seqs.push line.strip.chomp
+  end
+  # puts seqs.join(';')
+  
+	fqr=FastaQualFile.new(fasta,qual)
+	
+	output_fasta=File.new(output_name+'.fasta','a')
+	output_qual=File.new(output_name+'.fasta.qual','a')
+
+	fqr.each do |seq_name,seq_fasta,seq_qual|
+  	if seqs.index(seq_name)
+  	    output_fasta.puts ">#{seq_name}"
+  			output_fasta.puts seq_fasta
+  	    output_qual.puts ">#{seq_name}"
+  			output_qual.puts seq_qual
+  			seqs.delete(seq_name)
+  			if seqs.empty?
+  			  break
+			  end
+  	end
+  end
+  
+  output_qual.close
+  output_fasta.close
+  fqr.close
+
+end

data/bin/extract_seqs_from_fastq.rb

@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby
+
+require 'scbi_fastq'
+
+class Array
+ def count
+	self.length
+ end
+
+end
+
+if ARGV.count != 3
+  puts "#{$0} FASTQ OUTPUT_NAME SEQ_NAMES_FILE"
+  exit
+else
+  
+  fasta = ARGV.shift
+  output_name = ARGV.shift
+  seqs_file=ARGV.shift
+
+  seqs=[]
+
+  f=File.open(seqs_file).each_line do |line|
+  	seqs.push line.strip.chomp
+  end
+  puts seqs.join(';')
+  
+	fqr=FastqFile.new(fasta)
+	
+	output_fastq=FastqFile.new(output_name+'.fastq','w')
+
+	fqr.each do |seq_name,seq_fasta,seq_qual|
+  	if seqs.index(seq_name)
+  	    output_fastq.write_seq(seq_name,seq_fasta,seq_qual)
+  			seqs.delete(seq_name)
+  			if seqs.empty?
+  			  break
+			  end
+  	end
+  end
+  
+  output_fastq.close
+  fqr.close
+
+end

data/bin/fasta2fastq.rb

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+
+require 'scbi_fasta'
+require 'scbi_fastq'
+
+
+if ARGV.count < 3
+  puts "#{$0} FASTA QUAL OUTPUT_NAME"
+  exit
+end
+
+
+  
+fasta = ARGV.shift
+qual = ARGV.shift
+output_name = ARGV.shift
+default_qual = nil
+
+if !File.exists?(qual)
+  fqr=FastaFile.new(fasta)
+  puts "Quality file doesn't exists. Using default qual value = 40"
+  default_qual = [40]
+else
+  fqr=FastaQualFile.new(fasta,qual)
+end
+
+output=FastqFile.new(output_name+'.fastq','w')
+
+fqr.each do |seq_name,seq_fasta,seq_qual|
+    if default_qual
+      seq_qual =   default_qual * seq_fasta.length
+    end
+	  output.write_seq(seq_name,seq_fasta,seq_qual)
+end
+
+output.close
+fqr.close
+

data/bin/fastq2fasta.rb

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+
+require 'scbi_fastq'
+
+
+if ARGV.count < 2
+  puts "#{$0} FASTQ OUTPUT_NAME"
+  exit
+end
+
+
+  
+fastq = ARGV.shift
+output_name = ARGV.shift
+
+
+fasta = File.open(output_name+'.fasta','w')
+qual = File.open(output_name+'.fasta.qual','w')
+
+fqr=FastqFile.new(fastq)
+
+fqr.each do |seq_name,seq_fasta,seq_qual,comments|
+
+  fasta.puts ">#{seq_name} #{comments}"
+  fasta.puts seq_fasta
+  
+  qual.puts ">#{seq_name} #{comments}"
+  qual.puts seq_qual.join(' ')
+  
+end
+
+fasta.close
+qual.close
+fqr.close
+

data/bin/gen_qual.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+
+require 'scbi_fasta'
+
+GOOD_QUAL=50
+BAD_QUAL=10
+DOWN_CASE=('a'..'z')
+
+
+class Array
+ def count
+	self.length
+ end
+
+end
+
+if ARGV.count != 2
+  puts "Programa ENTRADA SALIDA" 
+  exit
+else
+  puts ARGV[0]
+  puts ARGV[1]
+  
+	fqr=FastaQualFile.new(ARGV[0])
+	
+	f = File.new(ARGV[1],'w+')
+
+	fqr.each do |seq_name,seq_fasta,seq_qual|
+    f.puts ">#{seq_name}"
+	res =[]
+	seq_fasta.each_char do |c|
+		if DOWN_CASE.include?(c)
+			res << BAD_QUAL
+		else
+			res << GOOD_QUAL
+		end
+	end
+
+	f.puts res.join(' ')
+#f.puts "50 "*seq_fasta.length
+  end
+  
+  f.close
+  fqr.close
+
+end

data/bin/get_seq.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+
+require 'scbi_fasta'
+
+GOOD_QUAL=50
+BAD_QUAL=10
+DOWN_CASE=('a'..'z')
+
+
+class Array
+ def count
+	self.length
+ end
+
+end
+
+if ARGV.count < 3
+  puts "#{$0} FASTA QUAL SEQ_NAME [f|q|fq]"
+  exit
+else
+  
+	fqr=FastaQualFile.new(ARGV[0],ARGV[1])
+    get_type = 'fq'
+	if ARGV.count == 4
+		get_type=ARGV[3]
+	end
+
+	fqr.each do |seq_name,seq_fasta,seq_qual|
+	if seq_name == ARGV[2]
+		if get_type.index('f')
+	    	puts ">#{seq_name}"
+			puts seq_fasta
+		end
+
+		if get_type.index('q')
+	    	puts ">#{seq_name}"
+			puts seq_qual
+		end
+		break
+	end
+
+  end
+  
+  fqr.close
+
+end