miga-base 0.2.1.4 → 0.2.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1dd8e19d8780562edc9d8bf94bb84e5f60487e54
-  data.tar.gz: dd86f3de40811fb2862adef1ef5eddb128a9d4d0
+  metadata.gz: 5145851e5906de7cafb98f5c701c69caca6404cc
+  data.tar.gz: 745cb4dcc63e62c3d054cae99a37a22a6dbdb80e
 SHA512:
-  metadata.gz: f93f2e32564193007777e83fb26583a96081eb607881167ba179ba25187ed4ee466897a188d5a81157d6287bcfff73d0690cb834842ebae182533049cc3d89f3
-  data.tar.gz: a4fcfb6ef19264145afedf204ff9b412db4f9d32549a060d4dc8fab21eca6a0e2c13f8646307c8fff56ef902404246a226668a3045f128c2a494edfd57e46d36
+  metadata.gz: 71308be2992a9f78776f618afb2d8e8f3f4055d77d2cada696772130b2570468f7031fc3cee04371ebe721c8fd3e598d302cd20dda85fe4d43ddb850d1bdc213
+  data.tar.gz: 0ff19642a0dcd5a679adb9562988a731d631c811551400c02884b8a47ba39072f23efc1f5a1a651f87cdfd01d2c962464e2821965dc0a06b81202df2ee2f9d5b

data/actions/create_dataset.rb

@@ -16,6 +16,10 @@ OptionParser.new do |opt|
     "Owner of the dataset."){ |v| o[:user]=v }
   opt.on("-c", "--comments STRING",
     "Comments on the dataset."){ |v| o[:comments]=v }
+  opt.on("-m", "--metadata STRING",
+    "Metadata as key-value pairs separated by = and delimited by comma.",
+    "Values are saved as strings except for booleans (true / false) or nil."
+    ){ |v| o[:metadata]=v }
   opt.on("--update",
     "Updates the dataset if it already exists."){ o[:update]=true }
   opt_common(opt, o)
@@ -34,6 +38,20 @@ $stderr.puts "Loading dataset." unless o[:q]
 d = o[:update] ? p.dataset(o[:dataset]) :
   MiGA::Dataset.new(p, o[:dataset], o[:ref], {})
 raise "Dataset does not exist." if d.nil?
+unless o[:metadata].nil?
+  o[:metadata].split(",").each do |pair|
+    (k,v) = pair.split("=")
+    case v
+    when "true"
+      v = true
+    when "false"
+      v = false
+    when "nil"
+      v = nil
+    end
+    d.metadata[k] = v
+  end
+end
 [:type, :description, :user, :comments].each do |k|
   d.metadata[k]=o[k] unless o[k].nil?
 end

data/lib/miga/dataset.rb

@@ -191,7 +191,9 @@ class MiGA::Dataset < MiGA::MiGA
   # execution order). This typically corresponds to the result used as the
   # initial input. Passes +save+ to #add_result.
   def first_preprocessing(save=false)
-    @@PREPROCESSING_TASKS.find{ |t| not add_result(t, save).nil? }
+    @@PREPROCESSING_TASKS.find do |t|
+      not ignore_task?(t) and not add_result(t, save).nil?
+    end
   end
   
   ##
@@ -212,6 +214,7 @@ class MiGA::Dataset < MiGA::MiGA
   ##
   # Should I ignore +task+ for this dataset?
   def ignore_task?(task)
+    return !metadata["run_#{task}"] unless metadata["run_#{task}"].nil?
     ( (@@EXCLUDE_NOREF_TASKS.include?(task) and not is_ref?) or
       (@@ONLY_MULTI_TASKS.include?(task) and not is_multi?) or
       (@@ONLY_NONMULTI_TASKS.include?(task) and not is_nonmulti?))

data/lib/miga/metadata.rb

@@ -107,8 +107,8 @@ class MiGA::Metadata < MiGA::MiGA
     v=v.miga_name if k==:name
     # Symbolize the special field :type
     v=v.to_sym if k==:type
-    # Register and return
-    @data[k]=v
+    # Delete if nil, register, and return
+    v.nil? ? @data.delete(k) : (@data[k]=v)
   end
 
   ##

data/lib/miga/remote_dataset.rb

@@ -130,7 +130,8 @@ class MiGA::RemoteDataset < MiGA::MiGA
       else
         download("#{base}.LargeContigs.fna")
       end
-      File.symlink("#{base}.LargeContigs.fna", "#{base}.AllContigs.fna")
+      File.symlink(
+        File.basename("#{base}.LargeContigs.fna"), "#{base}.AllContigs.fna")
       File.open("#{base}.done", "w") { |ofh| ofh.puts Time.now.to_s }
     else
       raise "Unexpected error: Unsupported result for database #{db}."

data/lib/miga/tax_dist.rb

@@ -53,9 +53,9 @@ module MiGA::TaxDist
     out = {}
     meaning.each do |phrase, thresholds|
       lwr, upr = thresholds
-      min = pv.values.select{ |v| v >= lwr }.min
+      min = pv.values.select{ |v| v < upr }.max
       return out if min.nil?
-      if min < upr
+      if min >= lwr
         v = pv.select{ |_,v| v==min }
         out[phrase] = (test==:intax ? v.reverse_each : v).first
       end

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.2, 1, 4]
+  VERSION = [0.2, 1, 5]
   
   ##
   # Nickname for the current major.minor version.

data/scripts/_distances_functions.bash

@@ -62,6 +62,19 @@ function haai {
   fi
 }
 
+function haai_or_aai {
+  local FH1=$1
+  local FH2=$2
+  local DBH=$3
+  local  F1=$4
+  local  F2=$5
+  local  DB=$6
+  local  TH=$7
+  AAI=$(haai $FH1 $FH2 $TH $DBH $DB)
+  [[ "${AAI%.*}" -le 0 ]] && AAI=$(aai $F1 $F2 $TH $DB)
+  echo $AAI
+}
+
 function val_from_db {
   local N1=$1
   local N2=$2

data/scripts/_distances_noref_nomulti.bash

@@ -24,91 +24,73 @@ function checkpoint_n {
   fi
 }
 
+function noref_haai_or_aai {
+  local Q=$1
+  local S=$2
+  haai_or_aai $ESS/$Q.ess.faa $ESS/$S.ess.faa $TMPDIR/$Q.haai.db \
+    ../06.cds/$Q.faa ../06.cds/$S.faa $TMPDIR/$Q.aai.db $CORES
+}
+
+function noref_ani {
+  local Q=$1
+  local S=$2
+  ani ../05.assembly/$Q.LargeContigs.fna ../05.assembly/$S.LargeContigs.fna \
+    $CORES $TMPDIR/$Q.ani.db
+}
+
 ESS="../07.annotation/01.function/01.essential"
 if [[ $(miga project_info -P "$PROJECT" -m type) != "clade" ]] ; then
   # Classify aai-clade (if project type is not clade)
   CLADES="../10.clades/01.find"
-  CLASSIF="."
-  [[ -e "$DATASET.aai-medoids.tsv" ]] && rm "$DATASET.aai-medoids.tsv"
-  while [[ -e "$CLADES/$CLASSIF/miga-project.medoids" ]] ; do
-    MAX_AAI=0
-    AAI_MED=""
-    AAI_CLS=""
-    i_n=0
-    for i in $(cat "$CLADES/$CLASSIF/miga-project.medoids") ; do
-      let i_n=$i_n+1
-      AAI=$(haai $ESS/$DATASET.ess.faa $ESS/$i.ess.faa $CORES \
-        $TMPDIR/$DATASET.haai.db $TMPDIR/$DATASET.aai.db)
-      [[ "${AAI%.*}" -le 0 ]] \
-        && AAI=$(aai ../06.cds/$DATASET.faa ../06.cds/$i.faa $CORES \
-        $TMPDIR/$DATASET.aai.db)
-      checkpoint_n
-      if [[ $(perl -e "print 1 if '$AAI' >= '$MAX_AAI'") == "1" ]] ; then
-        MAX_AAI=$AAI
-        AAI_MED=$i
-        AAI_CLS=$i_n
-	echo "[$CLASSIF] New max: $AAI_MED ($AAI_CLS): $MAX_AAI"
-      fi
-    done
-    CLASSIF="$CLASSIF/miga-project.sc-$AAI_CLS"
-    echo "$AAI_CLS	$AAI_MED	$MAX_AAI	$CLASSIF" \
-      >> "$DATASET.aai-medoids.tsv"
-  done
-
-  # Calculate all the AAIs/ANIs against the lowest subclade (if classified)
-  if [[ "$CLASSIF" != "." ]] ; then
-    PAR=$(dirname "$CLADES/$CLASSIF")/miga-project.classif
-    if [[ -s "$PAR" ]] ; then
-      for i in $(cat "$PAR" | awk "\$2==$AAI_CLS{print \$1}") ; do
-        AAI=$(aai ../06.cds/$DATASET.faa ../06.cds/$i.faa $CORES \
-          $TMPDIR/$DATASET.aai.db)
-        if [[ $(perl -e "print 1 if '$AAI' >= 90") == "1" ]] ; then
-          ani ../05.assembly/$DATASET.LargeContigs.fna \
-            ../05.assembly/$i.LargeContigs.fna \
-            $TMPDIR/$DATASET.ani.db >/dev/null
-        fi
-        checkpoint_n
-      done
-    fi
-  fi
+  METRIC="aai"
 else
   # Classify ani-clade (if project type is clade)
   CLADES="../10.clades/02.ani"
-  CLASSIF="."
-  [[ -e "$DATASET.ani-medoids.tsv" ]] && rm "$DATASET.ani-medoids.tsv"
-  while [[ -e "$CLADES/$CLASSIF/miga-project.medoids" ]] ; do
-    MAX_ANI=0
-    ANI_MED=""
-    ANI_CLS=""
-    i_n=0
-    for i in $(cat "$CLADES/$CLASSIF/miga-project.medoids") ; do
-      let i_n=$i_n+1
-      ANI=$(ani ../05.assembly/$DATASET.LargeContigs.fna \
-        ../05.assembly/$i.LargeContigs.fna $CORES $TMPDIR/$DATASET.ani.db)
-      checkpoint_n
-      if [[ $(perl -e "print 1 if '$ANI' >= '$MAX_ANI'") == "1" ]] ; then
-        MAX_ANI=$ANI
-        ANI_MED=$i
-        ANI_CLS=$i_n
-	echo "[$CLASSIF] New max: $ANI_MED ($ANI_CLS): $MAX_ANI"
-      fi
-    done
-    CLASSIF="$CLASSIF/miga-project.sc-$ANI_CLS"
-    echo "$ANI_CLS	$ANI_MED	$MAX_ANI	$CLASSIF" \
-      >> "$DATASET.ani-medoids.tsv"
-  done
+  METRIC="ani"
+fi
 
-  # Calculate all the ANIs against the lowest subclade (if classified in-clade)
-  if [[ "$CLASSIF" != "." ]] ; then
-    PAR=$(dirname "$CLADES/$CLASSIF")/miga-project.classif
-    if [[ -s "$CLADES/$CLASSIF/miga-project.all" ]] ; then
-      for i in $(cat "$PAR" | awk "\$2==$ANI_CLS{print \$1}") ; do
-        ani ../05.assembly/$DATASET.LargeContigs.fna \
-          ../05.assembly/$i.LargeContigs.fna $CORES \
-          $TMPDIR/$DATASET.ani.db > /dev/null
-        checkpoint_n
-      done
+CLASSIF="."
+[[ -e "$DATASET.$METRIC-medoids.tsv" ]] && rm "$DATASET.$METRIC-medoids.tsv"
+while [[ -e "$CLADES/$CLASSIF/miga-project.medoids" ]] ; do
+  MAX_VAL=0
+  VAL_MED=""
+  VAL_CLS=""
+  i_n=0
+  for i in $(cat "$CLADES/$CLASSIF/miga-project.medoids") ; do
+    let i_n=$i_n+1
+    if [[ $METRIC == "aai" ]] ; then
+      VAL=$(noref_haai_or_aai $DATASET $i)
+    else
+      VAL=$(noref_ani $DATASET $i)
+    fi
+    checkpoint_n
+    if [[ $(perl -e "print 1 if '$VAL' >= '$MAX_VAL'") == "1" ]] ; then
+      MAX_VAL=$VAL
+      VAL_MED=$i
+      VAL_CLS=$i_n
+      echo "[$CLASSIF] New max: $VAL_MED ($VAL_CLS): $MAX_VAL"
     fi
+  done
+  CLASSIF="$CLASSIF/miga-project.sc-$VAL_CLS"
+  echo "$VAL_CLS	$VAL_MED	$MAX_VAL	$CLASSIF" \
+    >> "$DATASET.$METRIC-medoids.tsv"
+done
+
+# Calculate all the AAIs/ANIs against the lowest subclade (if classified)
+if [[ "$CLASSIF" != "." ]] ; then
+  PAR=$(dirname "$CLADES/$CLASSIF")/miga-project.classif
+  if [[ -s "$PAR" ]] ; then
+    for i in $(cat "$PAR" | awk "\$2==$VAL_CLS{print \$1}") ; do
+      if [[ $METRIC == "aai" ]] ; then
+        AAI=$(noref_haai_or_aai $DATASET $i)
+      else
+        AAI=100
+      fi
+      if [[ $(perl -e "print 1 if '$AAI' >= 90") == "1" ]] ; then
+        noref_ani $DATASET $i
+      fi
+      checkpoint_n
+    done
   fi
 fi
 

data/scripts/haai_distances.bash

@@ -21,15 +21,21 @@ for i in $DS ; do
 done
 
 # R-ify
-echo "
-haai <- read.table('miga-project.txt', sep='\\t', h=T);
-save(haai, file='miga-project.Rdata');
-h <- hist(haai[,'value'], breaks=100, plot=FALSE);
-write.table(
-   cbind(h[['breaks']][-length(h[['breaks']])],h[['breaks']][-1],h[['counts']]),
-   file='miga-project.hist', quote=FALSE, sep='\\t',
-   col.names=FALSE, row.names=FALSE);
-" | R --vanilla
+if true ; then
+  echo "
+  haai <- read.table('miga-project.txt', sep='\\t', h=T);
+  save(haai, file='miga-project.Rdata');"
+  if [[ $(cat miga-project.txt | wc -l) -gt 1 ]] ; then
+    echo "
+    h <- hist(haai[,'value'], breaks=100, plot=FALSE);
+    write.table(
+      cbind(h[['breaks']][-length(h[['breaks']])],
+        h[['breaks']][-1],h[['counts']]),
+      file='miga-project.hist', quote=FALSE, sep='\\t',
+      col.names=FALSE, row.names=FALSE);
+    "
+  fi
+fi | R --vanilla
 
 # Gzip
 gzip -9 -f miga-project.txt

data/scripts/ogs.bash

@@ -31,6 +31,9 @@ done
 ogs.mcl.rb -o miga-project.ogs -d miga-project.rbm -t $CORES
 ogs.stats.rb -o miga-project.ogs -j miga-project.stats
 
+# Clean RBMs
+rm -rf miga-project.rbm
+
 # Finalize
 date "+%Y-%m-%d %H:%M:%S %z" > "miga-project.done"
 miga add_result -P "$PROJECT" -r ogs

data/utils/subclades.R

@@ -41,15 +41,16 @@ subclades <- function(ani_file, out_base, thr=1, ani=c()) {
   cl <- makeCluster(thr)
   s <- parSapply(cl, k, function(x) {
       library(cluster)
-      pam(ani.d, x, do.swap=FALSE, pamonce=1)$silinfo$avg.width
+      s <- pam(ani.d, x, do.swap=FALSE, pamonce=1)$silinfo
+      c(s$avg.width, -sum(ifelse(s$widths[,3]>0,0,s$widths[,3])))
     })
   stopCluster(cl)
-  ds <- (s[-c(1,length(s))]-pmax(s[-length(s)+c(0,1)],s[-c(1,2)]))
-  top.n <- head(k[order(c(-Inf,ds,-Inf), decreasing=T)], n=1)
+  ds <- s[1,]/s[2,]
+  top.n <- k[which.max(ds)]
   
   # Classify genomes
   say("Classify")
-  ani.cl <- pam(ani.d, top.n)
+  ani.cl <- pam(ani.d, top.n, pamonce=1)
   ani.types <- ani.cl$clustering
   ani.medoids <- ani.cl$medoids
   
@@ -58,7 +59,7 @@ subclades <- function(ani_file, out_base, thr=1, ani=c()) {
   pdf(paste(out_base, ".pdf", sep=""), 7, 12)
   layout(1:4)
   plot_distances(ani.d)
-  plot_silhouette(k, s, ds, top.n)
+  plot_silhouette(k, s[1,], s[2,], top.n)
   plot_clustering(ani.cl, ani.d, ani.types)
   plot_tree(ani.ph, ani.types, ani.medoids)
   dev.off()
@@ -67,6 +68,7 @@ subclades <- function(ani_file, out_base, thr=1, ani=c()) {
   say("Text report")
   write.table(ani.medoids, paste(out_base, "medoids", sep="."),
     quote=FALSE, col.names=FALSE, row.names=FALSE)
+  save(ani.d, file=paste(out_base, "dist.rdata", sep="."))
   classif <- cbind(names(ani.types), ani.types, ani.medoids[ ani.types ], NA)
   for(j in 1:nrow(classif)){
     classif[j,4] <- 100 - as.matrix(ani.d)[classif[j,1], classif[j,3]]
@@ -75,6 +77,7 @@ subclades <- function(ani_file, out_base, thr=1, ani=c()) {
     quote=FALSE, col.names=FALSE, row.names=FALSE, sep="\t")
 
   # Recursive search
+  say("Recursive search")
   for(i in 1:top.n){
     medoid <- ani.medoids[i]
     ds_f <- names(ani.types)[ ani.types==i ]
@@ -109,13 +112,13 @@ plot_silhouette <- function(k, s, ds, top.n) {
     ylim=range(s), bty="n", xaxs="i", yaxt="n")
   polygon(c(k[1], k, k[length(k)]), c(0,s,0), border=NA, col="grey80")
   axis(2, fg="grey60", col.axis="grey60")
-  mtext("Average silhouette", side=2, line=3, col="grey60")
+  mtext("Mean silhouette", side=2, line=3, col="grey60")
   par(new=TRUE)
   plot(1, t="n", xlab="", xaxt="n", ylab="", yaxt="n", xlim=range(c(0,k)),
     ylim=range(ds), bty="n", xaxs="i")
-  points(k[-c(1,length(k))], ds, type="o", pch=16, col=rgb(1/2,0,0,3/4))
+  points(k, ds, type="o", pch=16, col=rgb(1/2,0,0,3/4))
   axis(4, fg="darkred", col.axis="darkred")
-  mtext("Silhouette gain", side=4, line=3, col="darkred")
+  mtext("Negative silhouette area", side=4, line=3, col="darkred")
   abline(v=top.n, lty=2)
 }
 
@@ -129,7 +132,11 @@ plot_clustering <- function(cl, dist, types) {
   top.n <- length(cl$medoids)
   col <- ggplotColours(top.n)
   plot(silhouette(cl), col=col)
-  clusplot(cl, dist=dist, main="", col.p=col[types], lines=0)
+  if(length(labels(dist))<=15){
+    plot(1, type="n", axes=FALSE, xlab="", ylab="", bty="n")
+  }else{
+    clusplot(cl, dist=dist, main="", col.p=col[types], lines=0)
+  }
 }
 
 plot_tree <- function(phy, types, medoids){

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.2.1.4
+  version: 0.2.1.5
 platform: ruby
 authors:
 - Luis M. Rodriguez-R