bacterial-annotator 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: feff352efae5fee416e2ed704a1ba20e36c391f3
+  data.tar.gz: 852b8ce7a8c6fa18bae196434ef47a8687584e32
+SHA512:
+  metadata.gz: 794d2ee5ae969f986e0e5a3dbdf65425c6a8b9b39d72df383b8bcb09f1f51ab790355e186c6bf1081bf84f0bd3ee39cdde25e450de5939e14b70114a7a38ac69
+  data.tar.gz: 94c90664667c6d913d4396ec1e0bda356bc3fff86d3e1d4f91b91ba6fb55db16d7310e43f3e44bfc9ba8efd76b0b2e87d2f6b28d1fd08b2f222b798771cf2c3a

data/bin/ba_blat

@@ -0,0 +1,60 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+# author:  	maxime déraspe
+# email:	maxime@deraspe.net
+# review:  	
+# date:    	15-02-24
+# version: 	0.01
+# licence:  	
+
+require 'net/http'
+
+ROOT_path = File.dirname(__FILE__)
+# blat URL = "http://hgdownload.cse.ucsc.edu/admin/exe/linux.x86_64/blat/blat"
+
+# Install blat on the user system
+def installBlat
+
+  begin
+    Net::HTTP.start("hgdownload.cse.ucsc.edu") do |http|
+      resp = http.get("/admin/exe/linux.x86_64/blat/blat")
+      open("#{ROOT_path}/blat.linux", "wb") do |file|
+        file.write(resp.body)
+      end
+    end
+    
+    File.chmod(0755, "#{ROOT_path}/blat.linux")
+  rescue
+    abort "Problem installing Blat, aborting"
+  end
+
+end
+
+
+# Install blat if not already install
+if ! File.exists? "#{ROOT_path}/blat.linux"
+
+  puts "Installing Blat the aligner.."
+  puts ""
+  puts "Please note that the Blat source and executables are freely available for"
+  puts "academic, nonprofit and personal use. Commercial licensing information is"
+  puts "available on the Kent Informatics website (http://www.kentinformatics.com/)."
+  puts "See http://hgdownload.cse.ucsc.edu/admin/exe/linux.x86_64/blat/"
+  puts ""
+  puts "Do you accept the license ? Y/n"
+
+  answer = $stdin.gets.chomp
+
+  if answer.downcase == "y" or answer.downcase == ""
+    puts "OK you accepted the licence let's install the blat binary"
+    installBlat
+    puts "Blat successfully installed : OK"
+    puts ""
+  else
+    puts "Sorry bacterial-annotator rely on blat for the alignment !"
+  end
+
+end
+
+
+# system("#{ROOT_path}/blat.linux")

data/bin/ba_prodigal

@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+# author:  	maxime déraspe
+# email:	maxime@deraspe.net
+# review:  	
+# date:    	15-02-24
+# version: 	0.01
+# licence:  	
+
+require 'open-uri'
+
+ROOT_path = File.dirname(__FILE__)
+# prodigal URL = "https://github.com/hyattpd/Prodigal/releases/download/v2.6.2/prodigal.linux"
+
+# Install prodigal on the user system
+def installProdigal
+
+  begin
+    resp = open("https://github.com/hyattpd/Prodigal/releases/download/v2.6.2/prodigal.linux")
+    open("#{ROOT_path}/prodigal.linux", "wb") do |file|
+      file.write(resp.read)
+    end
+    File.chmod(0755, "#{ROOT_path}/prodigal.linux")
+  rescue
+    abort "Problem installing Prodigal, aborting"
+  end
+
+end
+
+
+# Install prodigal if not already install
+if ! File.exists? "#{ROOT_path}/prodigal.linux"
+
+  puts "Installing Prodigal the ORF finder.."
+  puts "See https://github.com/hyattpd/Prodigal"
+  puts "The Licence is GPLv3"
+  installProdigal
+  puts "Prodigal successfully installed"
+  puts ""
+
+end
+
+# Main
+# Dir.mkdir("Prodigal-Output") if ! File.exists? "Prodigal-Output"
+# system("#{ROOT_path}/prodigal.linux -a Prodigal-Output/Proteins.fa -d Prodigal-Output/Genes.fa -o Prodigal-Output/Genbanks.gbk -i #{ARGV[0]}")

data/bin/bacterial-annotator

@@ -0,0 +1,131 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+# author:  	maxime déraspe
+# email:	maxime@deraspe.net
+# review:  	
+# date:    	15-02-24
+# version: 	0.01
+# licence:  	
+
+
+require 'bacterial-annotator'
+
+
+# Usage message to print to CLI
+def usage
+
+  print <<OEM
+
+bacterial-annotator [OPTIONS]
+
+[OPTIONS]
+
+// IO
+
+  --input/-i		<fasta_file>	Provide the fasta file to annotate
+  --outdir/-o		<outdir>	Output directory [default=BAnnotation]
+  --force/-f		Force to overwrite the output directory
+
+// Dataset
+
+  --refgenome/-g        <GBK_ID> 	Provide a Genbank file or a Gbk Accession ID.
+  --guessref		Will guess the best reference genome to use for the annotation.
+
+  --remotedb		<remote_database> [nr|refseq|swissprot]
+			Complete the annotation of remaining CDS with a remote NCBI BLAST
+			Can be very slow, better to use an external database !
+
+  --externaldb		<proteins fasta_file>
+			Complete or do the annotation of remaining CDS with this database (a protein fasta file).
+			Fasta headers need to look similar to NCBI or EBI fasta headers, ex.:
+			>gi|385721352|gb|AFI72857.1| NDM-1 [Escherichia coli]
+			>sp|C7C422|BLAN1_KLEPN Beta-lactamase NDM-1 OS=Klebsiella pneumoniae..
+
+// Other options
+
+  --pidentity		Minimum percentage identity to incorporate a CDS annotation [default=0.7]
+  --minlength		Minimum contig length for annotation [default=500]
+
+  --meta		Better for metagenome and plasmid annotations because of disparate codon usage [default=off]
+  --gff			Will also generate gff annotation files [off by default]
+
+  --help/-h		Print this !
+OEM
+
+end
+
+# Parse the Options given on the CLI
+def parseOptions
+
+  options = {}
+
+  # default options
+  options[:outdir] = "BAnnotation"
+  options[:pidentity] = 70
+  options[:minlength] = 500
+  options[:meta] = 0
+  
+  while x = ARGV.shift
+
+    case x.downcase
+    when "--input", "-i"
+      options[:input] = ARGV.shift
+    when "--refgenome", "-g"
+      options[:refgenome] = ARGV.shift
+    when "--outdir", "-o"
+      options[:outdir] = ARGV.shift
+    when "--force", "-f"
+      options[:force] = 1
+    when "--gff"
+      options[:gff] = 1
+    when "--minlength"
+      options[:minlength] = ARGV.shift
+    when "--pidentity"
+      options[:pidentity] = ARGV.shift
+    when "--meta"
+      options[:meta] = 1
+    when "--remotedb"
+      options[:remote_db] = ARGV.shift
+    when "--externaldb"
+      options[:external_db] = ARGV.shift
+    when "--help", "-h"
+      usage
+    end
+
+  end
+
+  options
+
+end
+
+
+# Main
+if ARGV.size > 1
+
+  ROOT = File.dirname(__FILE__)
+  options = parseOptions
+
+  # Check for 3rd party dependencies : Prodigal and Blat
+  system("ba_prodigal")
+  system("ba_blat")
+
+  if ! File.exist? ("#{ROOT}/blat.linux")
+    abort "exiting blat is missing"
+  end
+
+  # Check Options 
+  if ! options.has_key? :refgenome and ! options.has_key? :remote_db and ! options.has_key? :external_db
+    puts "You didn't provide a reference genome or a database for the annotation !"
+  elsif ! options.has_key? :input
+    puts "You didn't provide a fasta file to annotate !"
+  elsif
+    puts ""
+  end
+
+  bannot = BacterialAnnotator.new(options, ROOT)
+  bannot.prepare_files_for_annotation
+  bannot.run_annotation
+
+else
+  usage
+end

data/lib/bacterial-annotator.rb

@@ -0,0 +1,406 @@
+# -*- coding: utf-8 -*-
+# author:  	maxime déraspe
+# email:	maxime@deraspe.net
+# review:  	
+# date:    	15-02-24
+# version: 	0.0.1
+# licence:  	
+
+require 'bio'
+require 'fileutils'
+
+require 'bacterial-annotator/genbank-manip'
+require 'bacterial-annotator/fasta-manip'
+require 'bacterial-annotator/synteny-manip'
+require 'bacterial-annotator/remote-ncbi'
+
+class BacterialAnnotator
+
+  # Initialize BacterialAnnotator
+  # options[:input], options[:refgenome], ROOT, options[:outdir], options)
+  def initialize options, root
+
+    @root = root
+    @options = options
+    @outdir = @options[:outdir]
+
+    @minlength = @options[:minlength].to_i
+    @pidentity = @options[:pidentity].to_f
+    @pidentity = @pidentity * 100 if @pidentity < 1.00
+
+    if File.exists? (@outdir)
+      if ! options.has_key? :force
+        abort "Output directory already exist ! Choose another one or use -f to overwrite"
+      else
+        puts "Overwriting output directory #{@outdir}"
+        FileUtils.remove_dir(@outdir, force=true)
+      end
+    end
+    Dir.mkdir(@outdir)
+
+    @fasta = FastaManip.new(@options[:input], @options[:meta])
+
+    @with_refence_genome = false
+    if @options.has_key? :refgenome
+      @with_refence_genome = true
+      @refgenome = GenbankManip.new(@options[:refgenome], @outdir)
+    end
+
+    @prot_synteny = nil
+    @annotation_stats = {by_contigs: {},
+                         annotated_cds: 0,
+                         total_cds: 0,
+                         foreign_contigs: [],
+                         synteny_contigs: [],
+                         short_contigs: []}
+
+    @contig_foreign_cds = {}
+    @contig_annotations = {}
+
+  end                           # end of method
+
+  # Prepare files for the annotation
+  # Will run prodigal on the query and prepare reference genome files
+  def prepare_files_for_annotation
+    puts "\nRunning Prodigal on your genome.."
+    @fasta.run_prodigal @root, @outdir
+    puts "Prodigal done."
+    if @with_refence_genome
+      @refgenome.write_cds_to_file @outdir
+      puts "Successfully loaded #{@refgenome.gbk.definition}"
+    end
+  end                           # end of method
+
+  # run_alignment of reference genome proteins and the query
+  def run_annotation
+
+    # process reference genome synteny
+    if @with_refence_genome        # Annotation with the Reference Genome
+
+      @prot_synteny = SyntenyManip.new(@fasta.prodigal_files[:proteins], @refgenome.cds_file, "Prot-Ref", @pidentity)
+      puts "\nRunning BLAT alignment with Reference Genome.."
+      @prot_synteny.run_blat @root, @outdir
+      @prot_synteny.extract_hits :refgenome
+
+      @fasta.prodigal_files[:contigs].each_with_index do |contig, contig_index|
+
+        # Skip short contigs
+        if @fasta.prodigal_files[:contigs_length][contig_index] < @minlength
+          @annotation_stats[:short_contigs] << contig
+          next
+        end
+
+        contig_prots = @fasta.prodigal_files[:prot_ids_by_contig][contig]
+
+        # contig_prot_annotations = @prot_synteny.get_annotation_for_contig contig_prots, @refgenome.coding_seq
+        @contig_annotations[contig] = @prot_synteny.get_annotation_for_contig contig_prots, @refgenome.coding_seq
+
+        remaining_cds = cumulate_annotation_stats_reference contig, @contig_annotations[contig]
+
+        if ! remaining_cds.empty?
+          @contig_foreign_cds[contig] = remaining_cds
+        end
+
+      end
+
+      # dump foreign proteins to file
+      foreign_cds_file = dump_cds
+
+    else
+
+      # no reference genome .. will process all the CDS
+      foreign_cds_file = @fasta.prodigal_files[:proteins]
+
+    end
+
+    # Finishing annotation for foreign proteins
+    finish_annotation foreign_cds_file
+
+    # Parse annotations to genbank files
+    parsing_genbank_files
+
+    puts "\nPrinting Statistics.."
+    print_stats "#{@outdir}/Annotation-Stats.txt"
+
+  end                           # end of method
+
+
+  # Finishing the annotation of the remaining CDS
+  def finish_annotation remaining_cds_file
+
+    if @options.has_key? :external_db	# from an external DB
+
+      db_file = @options[:external_db]
+      ref_cds = extract_externaldb_prot_info db_file
+
+      externaldb_synteny = SyntenyManip.new(remaining_cds_file, db_file, "Prot-ExternalDB", @pidentity)
+      puts "\nRunning BLAT alignment with External Database.."
+      externaldb_synteny.run_blat @root, @outdir
+      externaldb_synteny.extract_hits :externaldb
+
+      externaldb_synteny.aln_hits.each do |k,v|
+        contig_of_protein = k.split("_")[0..-2].join("_")
+
+        if ! @contig_annotations.has_key? contig_of_protein
+          @contig_annotations[contig_of_protein] = {}
+        end
+
+        hit_gi = v[:hits][0]
+
+        note = "correspond to gi:#{hit_gi}"
+
+        # p v
+        # p ref_cds[hit_gi]
+
+        if ref_cds[hit_gi][:org] != ""
+          note +=  " from #{ref_cds[hit_gi][:org]}"
+        end
+        @contig_annotations[contig_of_protein][k] = {product: ref_cds[hit_gi][:product],
+                                                     gene: nil,
+                                                     locustag: nil,
+                                                     note: note}
+
+      end
+
+
+    elsif @options.has_key? :remote_db	# from a remote DB
+
+      # do it by chunk to avoid NCBI CPU exceeding limit
+      cds_files = split_remaining_cds_file remaining_cds_file
+      @remotedb = @options[:remote_db]
+
+      puts "\n# NCBI Blast on #{@remotedb}"
+
+      cds_files.each do |cds_file|
+
+        # remotedb = @options[:remote_db]
+        valid = true
+        begin
+          # puts "\nNCBI blast on #{@remotedb} for #{cds_file}"
+          ncbiblast = RemoteNCBI.new(@remotedb,
+                                     cds_file,
+                                     "#{cds_file}.#{@remotedb}.xml",
+                                     @pidentity)
+        rescue
+          valid = false
+        end
+
+        # ncbi blast didn't worked out
+        if !valid
+          puts "Problem NCBI blast for foreign proteins"
+        else
+          ncbiblast.extract_blast_results
+          if ! ncbiblast.aln_hits
+            puts "Didn't produce the annotation for #{cds_file}"
+            next
+          end
+          ncbiblast.aln_hits.each do |k,v|
+            contig_of_protein = k.split("_")[0..-2].join("_")
+            # @contig_annotations[contig_of_protein][k][:product] = v[:hits][0][:product]
+            if ! @contig_annotations.has_key? contig_of_protein
+              @contig_annotations[contig_of_protein] = {}
+            end
+            note = "correspond to gi:#{v[:hits][0][:gi]}"
+            if v[:hits][0][:org] != ""
+              note +=  " from #{v[:hits][0][:org]}"
+            end
+            @contig_annotations[contig_of_protein][k] = {product: v[:hits][0][:product],
+                                                         gene: nil,
+                                                         locustag: nil,
+                                                         note: note}
+          end
+
+        end
+
+      end
+
+    end
+
+  end                           # end of method
+
+
+  # parse all genbank files
+  def parsing_genbank_files
+
+    puts "\nParsing annotation into genbank files.."
+    @contig_annotations.each do |contig, contig_prot_annotations|
+      gbk_path = @fasta.prodigal_files[:gbk_path]
+      gbk_to_annotate = GenbankManip.new("#{gbk_path}/#{contig}.gbk", "#{gbk_path}")
+      gbk_to_annotate.add_annotation contig_prot_annotations, gbk_path, 0
+    end
+
+  end                           # end of method
+
+
+  # cumulate the stats for the synteny
+  # return : unannotated cds array
+  def cumulate_annotation_stats_reference contig, contig_prots_ann
+
+    remaining_cds = []
+    contig_prots = @fasta.prodigal_files[:prot_ids_by_contig][contig]
+
+    @annotation_stats[:total_cds] += contig_prots.length if contig_prots
+    contig_prots_ann.each do |k,v|
+      if v != nil
+        @annotation_stats[:annotated_cds] += 1
+      else
+        remaining_cds << k
+      end
+    end
+
+    # Annotated Contigs
+    if contig_prots_ann.keys.length < 1
+      @annotation_stats[:foreign_contigs] << contig
+    else
+      @annotation_stats[:synteny_contigs] << contig
+    end
+
+    remaining_cds
+  end                           # end of method
+
+
+  # print statistics to file
+  def print_stats file
+
+    total_nb_contigs = @annotation_stats[:foreign_contigs].length +
+                       @annotation_stats[:synteny_contigs].length +
+                       @annotation_stats[:short_contigs].length
+    p_contigs_annotated = @annotation_stats[:synteny_contigs].length.to_f/total_nb_contigs.to_f
+    p_cds_annotated = @annotation_stats[:annotated_cds].to_f/@annotation_stats[:total_cds].to_f
+
+    File.open(file, "w") do |fopen|
+      fopen.write("#Contigs annotation based on reference genomes\n")
+      fopen.write("Short Contigs (< #{@minlength}) :\t\t" + @annotation_stats[:short_contigs].length.to_s + "\n")
+      fopen.write("Foreign Contigs :\t\t" + @annotation_stats[:foreign_contigs].length.to_s + "\n")
+      fopen.write("Annotated Contigs :\t\t" + @annotation_stats[:synteny_contigs].length.to_s + "\n")
+      fopen.write("Total Contigs :\t\t\t" + total_nb_contigs.to_s + "\n")
+      fopen.write("% Contigs annotated :\t\t" + (p_contigs_annotated*100).round(2).to_s + "\n")
+      fopen.write("\n")
+
+      fopen.write("#CDS annotations based on reference genomes\n")
+      fopen.write("Annotated CDS :\t\t\t" + @annotation_stats[:annotated_cds].to_s + "\n")
+      fopen.write("Total CDS :\t\t\t" + @annotation_stats[:total_cds].to_s + "\n")
+      fopen.write("% CDS annotated :\t\t" + (p_cds_annotated*100).round(2).to_s + "\n")
+      fopen.write("\n")
+
+    end
+
+  end                           # end of method
+
+
+  # dump cds to file for blast
+  def dump_cds
+
+    cds_outfile = File.open("#{@outdir}/Proteins-foreign.fa","w")
+    foreign_cds = []
+    @contig_foreign_cds.each_value do |v|
+      foreign_cds.push(*v)
+    end
+    inprot = false
+    File.open(@fasta.prodigal_files[:proteins]) do |fprot|
+      while l=fprot.gets
+        if l[0] == ">"
+          inprot = false
+          prot_id = l.chomp.split(" ")[0].delete(">")
+          if foreign_cds.include? prot_id
+            inprot = true
+            cds_outfile.write(l)
+          end
+        elsif inprot
+          cds_outfile.write(l)
+        end
+      end
+    end
+    cds_outfile.close
+    return "#{@outdir}/Proteins-foreign.fa"
+
+  end                           # end of method
+
+
+  # extract the information on protein from an externaldb
+  def extract_externaldb_prot_info db
+
+    # NCBI
+    # >gi|103485499|ref|YP_615060.1| chromosomal replication initiation protein [Sphingopyxis alaskensis RB2256]
+    # Swissprot
+    # >sp|C7C422|BLAN1_KLEPN Beta-lactamase NDM-1 OS=Klebsiella pneumoniae GN=blaNDM-1 PE=1 SV=1
+    # TrEMBL
+    # >tr|E5KIY2|E5KIY2_ECOLX Beta-lactamase NDM-1 OS=Escherichia coli GN=blaNDM-1 PE=1 SV=1
+
+    ref_cds = {}
+
+    File.open(db, "r") do |dbfile|
+      while l=dbfile.gets
+
+        if l[0] == ">"
+
+          lA = l.chomp.split("|")
+          key_gi = lA[1]
+          product_long = lA[-1]
+
+          organism = ""
+          product = ""
+
+          if product_long.include? " [" and product_long.include? "]" # NCBI
+            organism = product_long[/\[.*?\]/]
+            product = product_long.split(" [")[0].strip
+          elsif product_long.include? "OS="
+            product_tmp = product.split("OS=")
+            organism = product_tmp[1].split(/[A-Z][A-Z]=/)[0].strip
+            product = product_tmp[0].strip
+          elsif product_long.include? "[A-Z][A-Z]="
+            product = product_long.split(/[A-Z][A-Z]=/)[0].strip
+          end
+          org = organism.gsub("[","").gsub("]","")
+          product.lstrip!
+          ref_cds[key_gi] = {product: product, org: org}
+
+        end
+
+      end
+
+    end                         # end of file reading
+
+    ref_cds
+
+  end                           # end of method
+
+
+  # split fasta file to multiple fasta
+  def split_remaining_cds_file file
+
+    cds_files = []
+    outdir = "#{@outdir}/Protein-foreign.split"
+
+    Dir.mkdir(outdir) if ! Dir.exists? outdir
+
+    iter = 0
+    file_nb = 0
+    fout = File.open("#{outdir}/ProtForeign.#{file_nb}.fa", "w")
+    cds_files << "#{outdir}/ProtForeign.#{file_nb}.fa"
+
+    File.open(file, "r") do |fopen|
+      while l=fopen.gets
+        if l[0] == ">"
+          if iter > 19
+            fout.close
+            iter = 0
+            file_nb += 1
+            fout = File.open("#{outdir}/ProtForeign.#{file_nb}.fa", "w")
+            cds_files << "#{outdir}/ProtForeign.#{file_nb}.fa"
+          end
+          iter += 1
+        end
+        fout.write(l)
+      end
+    end
+
+    fout.close
+
+    cds_files
+
+  end                           # end of method
+
+  private :dump_cds, :split_remaining_cds_file
+
+end                             # end of class