miga-base 0.3.0.0 → 0.3.0.1

data/utils/enveomics/Scripts/Table.df2dist.R

@@ -0,0 +1,30 @@
+#!/usr/bin/env Rscript
+#
+# @author  Luis M. Rodriguez-R
+# @update  Jan-04-2016
+# @license artistic license 2.0
+#
+
+#= Load stuff
+args <- commandArgs(trailingOnly = F)  
+enveomics_R <- file.path(dirname(
+   sub("^--file=", "", args[grep("^--file=", args)])),
+   "lib", "enveomics.R")
+source(file.path(enveomics_R, "R", "cliopts.R"))
+source(file.path(enveomics_R, "R", "df2dist.R"))
+
+#= Generate interface
+opt <- enve.cliopts(enve.df2dist,
+   file.path(enveomics_R, "man", "enve.df2dist.Rd"),
+   positional_arguments=1,
+   usage="usage: %prog [options] output.mat",
+   mandatory=c("x"),
+   number=c("default.d", "max.sim"),
+   o_desc=list(x="A tab-delimited table with the distances."),
+   p_desc="Transform a tab-delimited list of distances into a squared matrix.")
+
+#= Run it!
+opt$options[['x']] <- read.table(opt$options[['x']],
+   header=TRUE, sep="\t", as.is=TRUE)
+dist <- do.call("enve.df2dist", opt$options)
+write.table(as.matrix(dist), opt$args[1], quote=FALSE, sep="\t", col.names=NA)

data/utils/enveomics/Scripts/Table.filter.pl

@@ -0,0 +1,61 @@
+#!/usr/bin/env perl
+#
+# @author: Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update: Mar-23-2015
+# @license: artistic license 2.0
+#
+
+use warnings;
+use strict;
+use Getopt::Std;
+
+my %o;
+getopts('k:s:ihn', \%o);
+my($list, $table) = @ARGV;
+
+($list and $table) or die "
+.Description:
+   Extracts (and re-orders) a subset of rows from a raw table.
+
+.Usage: $0 [options] list.txt table.txt > subset.txt
+   
+   Options:
+      -k <int>	Column of the table to use as key to filter.  By default, 1.
+      -s <str>	String to use as separation between rows.  By default, tabulation.
+      -i	If set, reports the inverse of the list (i.e., reports only rows
+      		absent in the list).  Implies -n.
+      -h	Keep first row of the table (header) untouched.
+      -n	No re-order.  The output has the same order of the table.  By
+      		default, it prints in the order of the list.
+
+   list.txt	List of IDs to extract.
+   table.txt	Table file containing the superset.
+   subset.txt	Table file to be created.
+
+";
+
+$o{k} ||= 1;
+$o{s} ||= "\t";
+$o{n}=1 if $o{i};
+my $HEADER = "";
+
+my $tbl2 = $o{n} ? $list : $table;
+open TBL, "<", $tbl2 or die "Cannot read file: $tbl2: $!\n";
+$HEADER = <TBL> if $o{h} and not $o{n};
+my %tbl2 = map { my $l=$_; chomp $l; my @r=split $o{s}, $l; $r[ $o{n} ? 0 : $o{k}-1] => $l } <TBL>;
+close TBL;
+
+my $tbl1 = $o{n} ? $table : $list;
+open TBL, "<", $tbl1 or die "Cannot read file: $tbl1: $!\n";
+$HEADER = <TBL> if $o{h} and $o{n};
+print $HEADER;
+while(my $ln = <TBL>){
+   chomp $ln;
+   next unless $ln;
+   my @ln = split $o{s}, $ln;
+   my $good = exists $tbl2{ $ln[$o{n} ? $o{k}-1 : 0] };
+   $good = not $good if $o{i};
+   print "".($o{n} ? $ln : $tbl2{$ln[0]})."\n" if $good;
+}
+close TBL;
+

data/utils/enveomics/Scripts/Table.merge.pl

@@ -0,0 +1,77 @@
+#!/usr/bin/env perl
+#
+# @author: Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update: Sep-20-2015
+# @license: artistic license 2.0
+#
+
+use warnings;
+use strict;
+use Getopt::Std;
+
+my %o;
+getopts('si:o:ne:h:H:r:', \%o);
+my @files = @ARGV;
+
+$#files>0 or die "
+.Description:
+   Merges multiple (two-column) lists into one table.
+
+.Usage:
+   $0 [options] files... > output.txt
+   
+   Options:
+      -s	Values are read as Strings.  By default, values are read as numbers.
+      -i <str>	Input field-delimiter.  By default: tabulation (\"\\t\").
+      -o <str>	Output field-delimiter.  By default: tabulation (\"\\t\").
+      -n	No-header.  By default, the header is determined by the file names.
+      -e <str>	Default string when no value is found.  By default, the \"empty\" value
+      		is 0 if values are numeric (i.e., unless -s is set) or an empty string
+		otherwise.
+      -h <str>	Header of the first column, containing the IDs.  By default: \"Tag\".
+      -H <str>	Format of filenames capturing the column header in the first capturing
+		parenthesis.  Non-capturing paretheses can be defined as (?:...).  By
+		default: \"(?:.*/)?([^\\.]+)\", which captures the part of the basename
+		of the file before the first dot (if any).
+      -r <int>	Number of leading rows to ignore in the input files. Zero by default.
+
+";
+$o{i} ||= "\t";
+$o{o} ||= "\t";
+$o{e} ||= ($o{s} ? "" : 0);
+$o{h} ||= "Tag";
+$o{H} ||= "(?:.*/)?([^\\.]+)";
+$o{r} += 0;
+
+my $notes = {};
+
+print $o{h} unless $o{n};
+my $i = 0;
+for my $file (@files){
+   unless($o{n}){
+      $file =~ m/$o{H}/ or die "Filename '$file' doesn't match format '$o{H}'.";
+      my $tag=$1;
+      print $o{o}.$tag;
+   }
+   open IN, "<", $file or die "Cannot read file: $file: $!\n";
+   while(<IN>){
+      next if $. <= $o{r};
+      chomp;
+      my @l = split $o{i};
+      $l[1]+=0 unless $o{s};
+      $notes->{$l[0]} ||= [];
+      $notes->{$l[0]}->[$i] = $l[1];
+   }
+   close IN;
+   $i++;
+}
+print "\n" unless $o{n};
+
+for my $id (keys %$notes){
+   print $id;
+   for my $i (0 .. $#files){
+      print $o{o}.(( defined $notes->{$id}->[$i] ? $notes->{$id}->[$i] : $o{e} ));
+   }
+   print "\n";
+}
+

data/utils/enveomics/Scripts/Table.replace.rb

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+
+#
+# @author  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update  Feb 01 2016
+# @license artistic license 2.0
+#
+
+require "optparse"
+
+o = {delimiter: "\t", key: 1, default: ""}
+ARGV << "-h" if ARGV.size==0
+OptionParser.new do |opts|
+   opts.banner = "\nReplaces a field in a table using a mapping file."
+   opts.on("-m", "--map FILE",
+      "Mapping file with two columns (key and replacement)."){ |v| o[:map] = v }
+   opts.on("-i", "--in FILE", "Input table."){ |v| o[:in] = v }
+   opts.on("-o", "--out FILE", "Output table."){ |v| o[:out] = v }
+   opts.on("-k", "--key INT",
+      "Column to replace in --in. By deafult: 1."){ |v| o[:key] = v.to_i }
+   opts.on("-u", "--unknown STR",
+      "String to use whenever the key is not found in --map."
+      ){ |v| o[:default] = v }
+   opts.on("-d", "--delimiter STR",
+      "String delimiting columns. By default, tabulation."
+      ){ |v| o[:delimiter] = v }
+   opts.on("-h", "--help", "Display this screen") do
+      puts opts
+      exit
+   end
+   opts.separator ""
+end.parse!
+abort "-m is mandatory" if o[:map].nil?
+abort "-i is mandatory" if o[:in].nil?
+abort "-o is mandatory" if o[:out].nil?
+
+class String
+   def is_number?
+      true if Float(self) rescue false
+   end
+end
+
+begin
+   # Read mapping file
+   ifh = File.open(o[:map], "r")
+   map = {}
+   while(ln = ifh.gets)
+      row = ln.chomp.split(o[:delimiter])
+      map[ row[0] ] = row[1]
+   end
+   ifh.close
+   # Process table
+   ifh = File.open(o[:in], "r")
+   ofh = File.open(o[:out], "w")
+   while(ln = ifh.gets)
+      row = ln.chomp.split(o[:delimiter])
+      k = row[ o[:key]-1 ]
+      v = map[ k ]
+      v = o[:default] if v.nil?
+      row[ o[:key]-1 ] = v
+      ofh.puts(row.join(o[:delimiter]))
+   end
+   ifh.close
+   ofh.close
+rescue => err
+   $stderr.puts "Exception: #{err}\n\n"
+   err.backtrace.each { |l| $stderr.puts " - " + l + "\n" }
+   err
+end

data/utils/enveomics/Scripts/Table.round.rb

@@ -0,0 +1,63 @@
+#!/usr/bin/env ruby
+
+#
+# @author: Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update: Feb 04 2015
+# @license: artistic license 2.0
+#
+
+require 'optparse'
+
+o = {:ndigits=>0, :action=>:round, :delimiter=>"\t"}
+ARGV << '-h' if ARGV.size==0
+OptionParser.new do |opts|
+   opts.banner = "\nRounds numbers in a table."
+   opts.on("-i", "--in FILE", "Input table."){ |v| o[:in] = v}
+   opts.on("-o", "--out FILE", "Output table."){ |v| o[:out] = v }
+   opts.on("-n", "--ndigits INT", "Number of decimal digits. By default: #{o[:ndigits]}"){ |v| o[:ndigits] = v.to_i }
+   opts.on("-f", "--floor", "Floors the values instead of rounding them. Ignores -n."){ o[:action] = :floor }
+   opts.on("-c", "--ceil", "Ceils the values instead of rounding them. Ignores -n."){ o[:action] = :ceil }
+   opts.on("-d", "--delimiter STR", "String delimiting columns. By default, tabulation."){ |v| o[:delimiter] = v }
+   opts.on("-h", "--help", "Display this screen") do
+      puts opts
+      exit
+   end
+   opts.separator ""
+end.parse!
+abort "-i is mandatory" if o[:in].nil?
+abort "-o is mandatory" if o[:out].nil?
+
+class String
+   def is_number?
+      true if Float(self) rescue false
+   end
+end
+
+begin
+   ifh = File.open(o[:in], "r")
+   ofh = File.open(o[:out], "w")
+   while(ln = ifh.gets)
+      ln.chomp!
+      row = []
+      ln.split(o[:delimiter]).each do |value|
+	 if value.is_number?
+	    case o[:action]
+	    when :round
+	       value = value.to_f.round(o[:ndigits])
+	    when :floor
+	       value = value.to_f.floor
+	    when :ceil
+	       value = value.to_f.ceil
+	    end
+	 end
+	 row.push value.to_s
+      end
+      ofh.puts(row.join(o[:delimiter]))
+   end
+   ifh.close
+   ofh.close
+rescue => err
+   $stderr.puts "Exception: #{err}\n\n"
+   err.backtrace.each { |l| $stderr.puts " - " + l + "\n" }
+   err
+end

data/utils/enveomics/Scripts/Table.split.pl

@@ -0,0 +1,57 @@
+#!/usr/bin/env perl
+#
+# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update Feb-01-2016
+# @license artistic license 2.0
+#
+
+use warnings;
+use strict;
+use Getopt::Std;
+use Symbol;
+
+my %o;
+getopts('i:o:d:e:h', \%o);
+my $file = shift @ARGV;
+
+($file and not $o{h}) or die "
+.Description:
+   Split a file with multiple columns into multiple two-columns lists.
+
+.Usage:
+   $0 [options] file
+   
+   Options:
+      -i <str>	Input field-delimiter.  By default: tabulation (\"\\t\").
+      -o <str>	Prefix of the output files.  By default: no prefix (\"\").
+      -d <str>	Output directory. By default: current directory (\"\").
+
+";
+$o{i} ||= "\t";
+$o{o} ||= "";
+$o{o} = $o{d}."/".$o{o} if $o{d};
+
+my $open=0;
+my @fhs=();
+open IN, "<", $file or die "Cannot read file: $file: $!\n";
+while(<IN>){
+   chomp;
+   my @row = split $o{i};
+   my $h = shift @row;
+   if($open){
+      for my $i (0 .. $#row){
+         print { qualify_to_ref $fhs[$i] } $h.$o{i}.$row[$i]."\n" if $row[$i];
+      }
+   }else{
+      $open++;
+      for my $l (@row){
+         $l =~ s/[\.\/:]/_/g;
+	 my $gs = gensym;
+	 open($gs, '>', $o{o}.$l.".txt") or die "Cannot create file: $o{o}$l.txt: $!\n";
+	 push @fhs, $gs;
+      }
+   }
+}
+close IN;
+close $_ for @fhs;
+

data/utils/enveomics/Scripts/Taxonomy.silva2ncbi.rb

@@ -0,0 +1,227 @@
+#!/usr/bin/env ruby
+
+#
+# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update: Feb-06-2015
+# @license artistic license 2.0
+#
+
+require 'optparse'
+
+$opts = {:warns=>false}
+ARGV << '-h' if ARGV.size==0
+OptionParser.new do |opt|
+   opt.separator "Re-formats Silva taxonomy into NCBI-like taxonomy dump files."
+   opt.separator ""
+   opt.separator "Mandatory arguments"
+   opt.on("-k", "--silvaranks FILE", "Input Silva ranks file (e.g., tax_ranks_ssu_115.txt)."){ |v| $opts[:silvaranks]=v }
+   opt.on("-f", "--silvaref FILE", "Input Silva ref alignment file (e.g., SSURef_NR99_115_tax_silva_full_align_trunc.fasta)."){ |v| $opts[:silvaref]=v }
+   opt.separator ""
+   opt.separator "Additional options"
+   opt.on("-p", "--patch FILE", "If passed, it replaces the paths specified in the patch."){ |v| $opts[:patch]=v }
+   opt.on("-s", "--seqinfo FILE", "If passed, it creates a CSV seq-info file compatible with taxtastic."){ |v| $opts[:seqinfo]=v }
+   opt.on("-t", "--taxfile FILE", "If passed, it creates a simple TSV taxonomy file."){ |v| $opts[:taxfile]=v }
+   opt.on("-n", "--ncbi FILE", "If passed, output folder for the NCBI dump files (e.g., taxdmp)."){ |v| $opts[:ncbi]=v }
+   opt.on("-w", "--warns", "Verbously display warnings."){ $opts[:warns]=true }
+   opt.on("-h", "--help","Display this screen") do
+      puts opt
+      exit
+   end
+   opt.separator ""
+end.parse!
+abort "-k/--silvaranks is mandatory." if $opts[:silvaranks].nil?
+abort "-k/--silvaranks must exist." unless File.exists? $opts[:silvaranks]
+abort "-f/--silvaref is mandatory." if $opts[:silvaref].nil?
+abort "-f/--silvaref must exist." unless File.exists? $opts[:silvaref]
+
+class Node
+   attr_accessor :id, :tax, :leaf, :name_type
+   attr_reader :name, :rank, :parent, :children
+   def initialize(name, rank=nil)
+      @name = name
+      @rank = rank.nil? ? "no rank" : rank
+      @children = []
+      @leaf = false
+      @name_type = "scientific name";
+   end
+   def parent=(node)
+      @parent=node
+      node.add_child(self)
+   end
+   def add_child(node)
+      @children << node
+   end
+   def ncbirank
+      ncbirank =
+	 self.rank == "superkingdom" ? "no rank" :
+	 self.rank == "domain" ? "superkingdom" :
+	 self.rank == "major_clade" ? "no rank" : self.rank
+      return ncbirank
+   end
+   def path
+      if self.parent.nil?
+         self.name
+      else
+         "#{self.parent.path};#{self.name}"
+      end
+   end
+   def each_desc internals, leaves, &blk
+      blk[self] if (leaves and self.leaf) or (internals and not self.leaf)
+      self.children.each {|child| child.each_desc internals, leaves, &blk}
+   end
+   def to_s
+      "#{self.name} (#{self.rank})"
+   end
+end
+
+class Taxonomy
+   attr_reader :root, :next_id
+   def initialize
+      @root = Node.new('root')
+      @root.id = 1
+      @next_id = 2
+   end
+   def register(node)
+      node.id = self.next_id
+      node.parent = self.root if node.parent.nil?
+      @next_id += 1
+   end
+   def node(path)
+      node = self.root
+      path.each do |level|
+         node.children.each do |child|
+	    if child.name == level
+	       node = child
+	       break
+	    end
+	 end
+	 unless node.name == level
+	    $stderr.puts "Warning: Impossible to find #{level} at #{node.to_s}, making it up." if $opts[:warns]
+	    child = Node.new(level)
+	    child.parent = node
+	    self.register(child)
+	    node = child
+	 end
+      end
+      node
+   end
+   def each_node &blk
+      self.root.each_desc true, true, &blk
+   end
+   def each_leaf &blk
+      self.root.each_desc false, true, &blk
+   end
+   def each_internal &blk
+      self.root.each_desc true, false, &blk
+   end
+end
+
+begin
+   taxo = Taxonomy.new()
+   
+   ## Read patch
+   patch = {}
+   unless $opts[:patch].nil?
+      $stderr.puts "Reading patch: #{$opts[:patch]}"
+      f = File.open($opts[:patch], "r")
+      while(ln = f.gets)
+	 m = ln.chomp.split(/\t/)
+	 patch[ m[0] ] = m[1]
+      end
+   end
+   
+   ## Read the Silva ranks
+   $stderr.puts "Reading Silva ranks: #{$opts[:silvaranks]}"
+   f = File.open($opts[:silvaranks], "r")
+   f.gets # header
+   while(ln = f.gets)
+      m = ln.chomp.split(/\t/)
+      m[0] = patch[ m[0] ] unless patch[ m[0] ].nil?
+      p = m[0].split(/;/)
+      raise "Inconsistent path and node name at line #{$.}: #{ln}." unless m[1] == p.pop
+      if m[3] != "w"
+	 node = Node.new(m[1], m[2])
+	 node.name_type = "common name" if m[3] == "a"
+	 node.parent = taxo.node(p)
+	 taxo.register(node)
+      end
+   end
+   f.close
+
+   $stderr.puts "  Top taxa:"
+   taxo.root.children.each do |top|
+      $stderr.puts "    o #{top.to_s} has #{top.children.length} children."
+   end
+
+   ## Read the Silva ref alignment
+   $stderr.puts "Reading Silva ref alignment: #{$opts[:silvaref]}"
+   i = 0
+   f = File.open($opts[:silvaref], "r")
+   while(ln = f.gets)
+      m = />([^\s]+)\s(.*)/.match(ln)
+      next unless m
+      # Patch
+      pm = /(.+);([^;]+)/.match(m[2])
+      path = "#{patch[ pm[1] ].nil? ? pm[1] : patch[ pm[1] ]};#{pm[2]}".split(/;/)
+      # Register
+      node = taxo.node(path)
+      taxo.register(node)
+      refseq = Node.new(m[1], 'refseq')
+      refseq.parent = node
+      refseq.leaf = true
+      taxo.register(refseq)
+      i += 1
+   end
+   f.close
+   $stderr.puts "  Saved #{i} leaves."
+
+   ### NCBI
+   unless $opts[:ncbi].nil?
+      ## Create taxonomy .dmp files
+      $stderr.puts "Creating NCBI-like files: #{$opts[:ncbi]}"
+      Dir.mkdir($opts[:ncbi]) unless Dir.exists?($opts[:ncbi]);
+      # merged.dmp
+      $stderr.puts "  o Creating merged.dmp"
+      File.open(File.join($opts[:ncbi], 'merged.dmp'), 'w'){}
+      # names.dmp
+      $stderr.puts "  o Creating names.dmp"
+      f = File.open(File.join($opts[:ncbi], 'names.dmp'), 'w')
+      taxo.each_internal do |n|
+	 f.puts [n.id, n.name, "", n.name_type].join("\t|\t")+"\t|"
+      end
+      f.close
+      # nodes.dmp
+      $stderr.puts "  o Creating nodes.dmp"
+      f = File.open(File.join($opts[:ncbi], 'nodes.dmp'), 'w')
+      taxo.each_internal do |n|
+	 f.puts ([n.id, n.parent.nil? ? n.id : n.parent.id, n.ncbirank, ""] << Array.new(8,0) << "").join("\t|\t")+"\t|"
+      end
+      f.close
+   end
+
+   ## Taxtastic
+   unless $opts[:seqinfo].nil?
+      $stderr.puts "Creating seq-info file: #{$opts[:seqinfo]}"
+      f = File.open($opts[:seqinfo], 'w')
+      f.puts "\"seqname\",\"tax_id\",\"group_name\""
+      taxo.each_leaf { |n| f.puts "\"#{n.name}\",\"#{n.parent.id}\",\"#{n.parent.name}\"" }
+      f.close
+   end
+   
+   ## Misc
+   unless $opts[:taxfile].nil?
+      $stderr.puts "Creating taxonomy file: #{$opts[:taxfile]}"
+      f = File.open($opts[:taxfile], 'w')
+      f.puts "tax_id\tparent_id\trank\ttax_name"
+      taxo.each_internal do |n|
+	 f.puts [n.id, n.parent.nil? ? n.id : n.parent.id, n.rank, n.name].join("\t")
+      end
+      f.close
+   end
+rescue => err
+   $stderr.puts "Exception: #{err}\n\n"
+   err.backtrace.each { |l| $stderr.puts l + "\n" }
+   err
+end
+
+