miga-base 0.2.0.9 → 0.2.1.0

data/scripts/_distances_functions.bash

@@ -0,0 +1,82 @@
+#!/bin/bash
+# Available variables: $PROJECT, $DATASET, $RUNTYPE, $MIGA, $CORES, $TMPDIR,
+#                      $NOMULTI, $REF
+
+set -e
+
+if [[ ! -n $MIGA_AAI_SAVE_RBM ]] ; then
+  MIGA_AAI_SAVE_RBM="save-rbm"
+  if [[ -n $PROJECT ]] ; then
+    if [[ $(miga project_info -P "$PROJECT" -m type) != "clade" ]] ; then
+      MIGA_AAI_SAVE_RBM="no-save-rbm"
+    fi
+  fi
+fi
+
+function make_empty_aai_db {
+  local DB=$1
+  echo "create table if not exists aai(seq1 varchar(256), seq2 varchar(256)," \
+    " aai float, sd float, n int, omega int);" | sqlite3 $DB
+}
+
+function ds_name {
+  basename $1 | perl -pe "s/[^A-Za-z0-9_].*//"
+}
+
+function aai {
+  local F1=$1
+  local F2=$2
+  local TH=$3
+  local DB=$4
+  local N1=$(ds_name $F1)
+  local N2=$(ds_name $F2)
+  aai.rb -1 $F1 -2 $F2 -t $TH -a --lookup-first -S $DB --name1 $N1 --name2 $N2 \
+    --$MIGA_AAI_SAVE_RBM || echo "0"
+}
+
+function ani {
+  local F1=$1
+  local F2=$2
+  local TH=$3
+  local DB=$4
+  local N1=$(ds_name $F1)
+  local N2=$(ds_name $F2)
+  ani.rb -1 $F1 -2 $F2 -t $TH -a --no-save-regions --no-save-rbm \
+    --lookup-first -S $DB --name1 $N1 --name2 $N2 || echo "0"
+}
+
+function haai {
+  local F1=$1
+  local F2=$2
+  local TH=$3
+  local DB=$4
+  local AAI_DB=$5
+  local N1=$(ds_name $F1)
+  local N2=$(ds_name $F2)
+  local HAAI=$(MIGA_AAI_SAVE_RBM="no-save-rbm" aai $F1 $F2 $TH $DB)
+  if [[ "$HAAI" != "" && $(perl -e "print 1 if '$HAAI' <= 90") == "1" ]] ; then
+    local AAI=$(perl -e "print (100-exp(2.435076 + 0.4275193*log(100-$HAAI)))")
+    [[ ! -s $AAI_DB ]] && make_empty_aai_db $AAI_DB
+    echo "insert into aai values('$N1','$N2','$AAI',0,0,0);" | sqlite3 $AAI_DB
+    echo $AAI
+  fi
+}
+
+function val_from_db {
+  local N1=$1
+  local N2=$2
+  local DB=$3
+  local MT=$4
+  if [[ -s $DB ]] ; then
+    echo "select $MT from $MT where seq1='$N1' and seq2='$N2';" \
+      | sqlite3 $DB || echo 0
+  fi
+}
+
+function aai_from_db {
+  val_from_db $1 $2 $3 aai
+}
+
+function ani_from_db {
+  val_from_db $1 $2 $3 ani
+}

data/scripts/_distances_noref_nomulti.bash

@@ -1,88 +1,117 @@
 #!/bin/bash
 # Available variables: $PROJECT, $DATASET, $RUNTYPE, $MIGA, $CORES, $TMPDIR,
-# 			$NOMULTI, $REF
+#                      $NOMULTI, $REF
 
 set -e
 
 # Deal with previous runs (if any)
+exists $DATASET.haai.db && cp $DATASET.haai.db $TMPDIR
 exists $DATASET.a[an]i.db && cp $DATASET.a[an]i.db $TMPDIR
 exists $DATASET.a[an]i.9[05] && rm $DATASET.a[an]i.9[05]
 N=0
 function checkpoint_n {
-   let N=$N+1
-   if [[ $N -ge 10 ]] ; then
-      for metric in aai ani ; do
-	 if [[ -s $TMPDIR/$DATASET.$metric.db ]] ; then
-	    echo "select count(*) from $metric;" \
-	       | sqlite3 $TMPDIR/$DATASET.$metric.db \
-	       || exit 1
-	    cp $TMPDIR/$DATASET.$metric.db .
-	 fi
-      done
-      N=0
-   fi
+  let N=$N+1
+  if [[ $N -ge 10 ]] ; then
+    for metric in haai aai ani ; do
+      if [[ -s $TMPDIR/$DATASET.$metric.db ]] ; then
+        echo "select count(*) from ${metric#h};" \
+          | sqlite3 $TMPDIR/$DATASET.$metric.db \
+          >/dev/null || exit 1
+        cp $TMPDIR/$DATASET.$metric.db .
+      fi
+    done
+    N=0
+  fi
 }
 
-# Find 95%ANI clade(s) with AAI <= 90% / ANI <= 95%
-REFGENOMES=$(cat ../10.clades/01.find/miga-project.ani95-clades \
-   | tail -n +2 | cut -d , -f 1)
-for i in $REFGENOMES ; do
-   AAI=$(aai.rb -1 ../06.cds/$DATASET.faa \
-      -2 ../06.cds/$i.faa -t $CORES -a --lookup-first \
-      -S $TMPDIR/$DATASET.aai.db --name1 $DATASET --name2 $i || echo "")
-   checkpoint_n
-   if [[ $(perl -MPOSIX -e "print ceil $AAI") -ge 90 ]] ; then
-      echo $i >> $DATASET.aai90
-      [[ -e "../05.assembly/$DATASET.LargeContigs.fna" ]] || continue
-      [[ -e "../05.assembly/$i.LargeContigs.fna" ]] || continue
-      ANI=$(ani.rb -1 ../05.assembly/$DATASET.LargeContigs.fna \
-	 -2 ../05.assembly/$i.LargeContigs.fna -t $CORES -a \
-	 --no-save-regions --no-save-rbm --lookup-first \
-	 -S $TMPDIR/$DATASET.ani.db --name1 $DATASET --name2 $i || echo "")
+ESS="../07.annotation/01.function/01.essential"
+if [[ $(miga project_info -P "$PROJECT" -m type) != "clade" ]] ; then
+  # Classify aai-clade (if project type is not clade)
+  CLADES="../10.clades/01.find"
+  CLASSIF="."
+  [[ -e "$DATASET.aai-medoids.tsv" ]] && rm "$DATASET.aai-medoids.tsv"
+  while [[ -e "$CLADES/$CLASSIF/miga-project.medoids" ]] ; do
+    MAX_AAI=0
+    AAI_MED=""
+    AAI_CLS=""
+    i_n=0
+    for i in $(cat "$CLADES/$CLASSIF/miga-project.medoids") ; do
+      let i_n=$i_n+1
+      AAI=$(haai $ESS/$DATASET.ess.faa $ESS/$i.ess.faa $CORES \
+        $TMPDIR/$DATASET.haai.db $TMPDIR/$DATASET.aai.db)
+      [[ "${AAI%.*}" -le 0 ]] \
+        && AAI=$(aai ../06.cds/$DATASET.faa ../06.cds/$i.faa $CORES \
+        $TMPDIR/$DATASET.aai.db)
       checkpoint_n
-      if [[ $(perl -MPOSIX -e "print ceil $ANI") -ge 95 ]] ; then
-	 echo $i >> $DATASET.ani95
+      if [[ $(perl -e "print 1 if '$AAI' >= '$MAX_AAI'") == "1" ]] ; then
+        MAX_AAI=$AAI
+        AAI_MED=$i
+        AAI_CLS=$i_n
+	echo "[$CLASSIF] New max: $AAI_MED ($AAI_CLS): $MAX_AAI"
       fi
-   fi
-done
+    done
+    CLASSIF="$CLASSIF/miga-project.sc-$AAI_CLS"
+    echo "$AAI_CLS	$AAI_MED	$MAX_AAI	$CLASSIF" \
+      >> "$DATASET.aai-medoids.tsv"
+  done
 
-# Classify in-clade (if project type is clade)
-CLADES="../10.clades/02.ani"
-CLASSIF="."
-MAX_ANI=0
-ANI_MED=""
-[[ -e "$DATASET.medoids" ]] && rm "$DATASET.medoids"
-while [[ -e "$CLADES/$CLASSIF/miga-project.1.medoids" ]] ; do
-   for i in $(cat "$CLADES/$CLASSIF/miga-project.1.medoids") ; do
-      ANI=$(ani.rb -1 ../05.assembly/$DATASET.LargeContigs.fna \
-	 -2 ../05.assembly/$i.LargeContigs.fna -t $CORES -a \
-	 --no-save-regions --no-save-rbm --lookup-first \
-	 -S $TMPDIR/$DATASET.ani.db --name1 $DATASET --name2 $i || echo "")
+  # Calculate all the AAIs/ANIs against the lowest subclade (if classified)
+  if [[ "$CLASSIF" != "." ]] ; then
+    PAR=$(dirname "$CLADES/$CLASSIF")/miga-project.classif
+    if [[ -s "$PAR" ]] ; then
+      for i in $(cat "$PAR" | awk "\$2==$AAI_CLS{print \$1}") ; do
+        AAI=$(aai ../06.cds/$DATASET.faa ../06.cds/$i.faa $CORES \
+          $TMPDIR/$DATASET.aai.db)
+        if [[ $(perl -e "print 1 if '$AAI' >= 90") == "1" ]] ; then
+          ani ../05.assembly/$DATASET.LargeContigs.fna \
+            ../05.assembly/$i.LargeContigs.fna \
+            $TMPDIR/$DATASET.ani.db >/dev/null
+        fi
+        checkpoint_n
+      done
+    fi
+  fi
+else
+  # Classify ani-clade (if project type is clade)
+  CLADES="../10.clades/02.ani"
+  CLASSIF="."
+  [[ -e "$DATASET.ani-medoids.tsv" ]] && rm "$DATASET.ani-medoids.tsv"
+  while [[ -e "$CLADES/$CLASSIF/miga-project.medoids" ]] ; do
+    MAX_ANI=0
+    ANI_MED=""
+    ANI_CLS=""
+    i_n=0
+    for i in $(cat "$CLADES/$CLASSIF/miga-project.medoids") ; do
+      let i_n=$i_n+1
+      ANI=$(ani ../05.assembly/$DATASET.LargeContigs.fna \
+        ../05.assembly/$i.LargeContigs.fna $CORES $TMPDIR/$DATASET.ani.db)
       checkpoint_n
-      if [[ $(perl -e "print 1 if $ANI > $MAX_ANI") == "1" ]] ; then
-         MAX_ANI=$ANI
-	 ANI_MED=$i
+      if [[ $(perl -e "print 1 if '$ANI' >= '$MAX_ANI'") == "1" ]] ; then
+        MAX_ANI=$ANI
+        ANI_MED=$i
+        ANI_CLS=$i_n
+	echo "[$CLASSIF] New max: $ANI_MED ($ANI_CLS): $MAX_ANI"
       fi
-   done
-   echo $i >> "$DATASET.medoids"
-   CLASSIF="$CLASSIF/miga-project.1.subcl-$i"
-done
-echo $CLASSIF > "$DATASET.class"
+    done
+    CLASSIF="$CLASSIF/miga-project.sc-$ANI_CLS"
+    echo "$ANI_CLS	$ANI_MED	$MAX_ANI	$CLASSIF" \
+      >> "$DATASET.ani-medoids.tsv"
+  done
 
-# Calculate all the ANIs against the lowest subclade (if classified in-clade)
-if [[ "$CLASSIF" != "." ]] ; then
-   if [[ -s "$CLADES/$CLASSIF/miga-project.all" ]] ; then
-      for i in $(cat "$CLADES/$CLASSIF/miga-project.all") ; do
-	 ANI=$(ani.rb -1 ../05.assembly/$DATASET.LargeContigs.fna \
-	    -2 ../05.assembly/$i.LargeContigs.fna -t $CORES -a \
-	    --no-save-regions --no-save-rbm --lookup-first \
-	    -S $TMPDIR/$DATASET.ani.db --name1 $DATASET --name2 $i || echo "")
-	 checkpoint_n
+  # Calculate all the ANIs against the lowest subclade (if classified in-clade)
+  if [[ "$CLASSIF" != "." ]] ; then
+    PAR=$(dirname "$CLADES/$CLASSIF")/miga-project.classif
+    if [[ -s "$CLADES/$CLASSIF/miga-project.all" ]] ; then
+      for i in $(cat "$PAR" | awk "\$2==$ANI_CLS{print \$1}") ; do
+        ani ../05.assembly/$DATASET.LargeContigs.fna \
+          ../05.assembly/$i.LargeContigs.fna $CORES \
+          $TMPDIR/$DATASET.ani.db > /dev/null
+        checkpoint_n
       done
-   fi
+    fi
+  fi
 fi
 
-# Finalize
-mv $TMPDIR/$DATASET.aai.db 02.aai/$DATASET.db
-mv $TMPDIR/$DATASET.ani.db 03.ani/$DATASET.db
-
+#Finalize
+N=11
+checkpoint_n

data/scripts/_distances_ref_nomulti.bash

@@ -5,102 +5,71 @@
 set -e
 
 function checkpoint_n {
-   if [[ $N -eq 10 ]] ; then
-      for t in 01.haai 02.aai 03.ani ; do
-         if [[ -s $TMPDIR/$t.db ]] ; then
-	    tab="aai"
-	    [[ "$t" == "03.ani" ]] && tab="ani"
-	    echo "select count(*) from $tab;" \
-	       | sqlite3 $TMPDIR/$t.db\
-	       || exit 1
-	    cp $TMPDIR/$t.db $t/$DATASET.db
-	 fi
-      done
-      N=0
-   fi
-   let N=$N+1
+  if [[ $N -eq 10 ]] ; then
+    for t in 01.haai 02.aai 03.ani ; do
+      if [[ -s $TMPDIR/$t.db ]] ; then
+        tab="aai"
+        [[ "$t" == "03.ani" ]] && tab="ani"
+        echo "select count(*) from $tab;" \
+          | sqlite3 $TMPDIR/$t.db \
+          >/dev/null || exit 1
+        cp $TMPDIR/$t.db $t/$DATASET.db
+      fi
+    done
+    N=0
+  fi
+  let N=$N+1
 }
 
 ESS="../07.annotation/01.function/01.essential"
 
 # Initialize temporals
 for t in 01.haai 02.aai 03.ani ; do
-   [[ -s $t/$DATASET.db ]] && cp $t/$DATASET.db $TMPDIR/$t.db
+  [[ -s $t/$DATASET.db ]] && cp $t/$DATASET.db $TMPDIR/$t.db
 done
-echo "create table if not exists aai(seq1 varchar(256), seq2 varchar(256)," \
-   "aai float, sd float, n int, omega int);" | sqlite3 $TMPDIR/02.aai.db
 N=1
 
 # Traverse "nearly-half" of the ref-datasets using first-come-first-served
 for i in $(miga list_datasets -P "$PROJECT" --ref --no-multi) ; do
-   echo "=[ $i ]"
-   date "+%Y-%m-%d %H:%M:%S %z"
-   HAAI=""; AAI=""; ANI="";
-   # Check if the i-th dataset is ready
-   [[ -s $ESS/$i.done && -s $ESS/$i.json ]] || continue
-   # Check if this is done (e.g., in a previous failed iteration)
-   AAI=$(echo "select aai from aai where seq1='$DATASET' and seq2='$i';" \
-      | sqlite3 $TMPDIR/02.aai.db || echo "")
-   # Try the other direction
-   if [[ "$AAI" == "" && -s 02.aai/$i.db ]] ; then
-      cp "02.aai/$i.db" "$TMPDIR/$i.db"
-      AAI=$(echo "select aai from aai where seq2='$DATASET' and seq1='$i';" \
-	 | sqlite3 "$TMPDIR/$i.db" || echo "")
-      rm "$TMPDIR/$i.db"
-   fi
-   # Try with hAAI
-   if [[ "$AAI" == "" ]] ; then
-      [[ -e "$TMPDIR/$DATASET.ess.faa" ]] \
-	 || cp $ESS/$DATASET.ess.faa $TMPDIR/$DATASET.ess.faa
-      HAAI=$(aai.rb -1 $TMPDIR/$DATASET.ess.faa -2 $ESS/$i.ess.faa \
-	 -t $CORES -a -n 10 -S $TMPDIR/01.haai.db --name1 $DATASET \
-	 --name2 $i --lookup-first --no-save-rbm || echo "")
-      if [[ "$HAAI" != "" \
-	    && $(perl -MPOSIX -e "print floor $HAAI") -lt 90 ]] ; then
-	 AAI=$(perl -e \
-	    "printf '%f', 100-exp(2.435076 + 0.4275193*log(100-$HAAI))")
-	 echo "insert into aai values('$DATASET','$i','$AAI',0,0,0);" \
-	    | sqlite3 $TMPDIR/02.aai.db
-      fi
-   fi
-   # Try with complete AAI
-   if [[ "$AAI" == "" ]] ; then
-      [[ -e "$TMPDIR/$DATASET.faa" ]] \
-	 || cp ../06.cds/$DATASET.faa $TMPDIR/$DATASET.faa
-      AAI=$(aai.rb -1 $TMPDIR/$DATASET.faa -2 ../06.cds/$i.faa -t $CORES -a \
-	 -S $TMPDIR/02.aai.db --name1 $DATASET --name2 $i --lookup-first \
-	 || echo "")
-   fi
-   date "+%Y-%m-%d %H:%M:%S %z"
-   # Check if ANI is meaningful
-   if [[ -e "../05.assembly/$DATASET.LargeContigs.fna" \
-	 && -e "../05.assembly/$i.LargeContigs.fna" \
-	 && $(perl -MPOSIX -e "print ceil $AAI") -gt 90 ]] ; then
-      # Check if this is done (e.g., in a previous failed iteration)
-      ANI=$(echo "select ani from ani where seq1='$DATASET' and seq2='$i';" \
-	 | sqlite3 $TMPDIR/03.ani.db || echo "")
-      # Try the other direction
-      if [[ "$ANI" == "" && -s 03.ani/$i.db ]] ; then
-	 cp "03.ani/$i.db" "$TMPDIR/$i.db"
-	 ANI=$(echo "select ani from ani" \
-	    "where seq2='$DATASET' and seq1='$i';" \
-	    | sqlite3 "$TMPDIR/$i.db" || echo "")
-	 rm "$TMPDIR/$i.db"
-      fi
-      # Calculate it
-      if [[ "$ANI" == "" ]] ; then
-	 [[ -e "$TMPDIR/$DATASET.LargeContigs.fna" ]] \
-	    || cp ../05.assembly/$DATASET.LargeContigs.fna \
-	       $TMPDIR/$DATASET.LargeContigs.fna
-	 ANI=$(ani.rb -1 $TMPDIR/$DATASET.LargeContigs.fna \
-	    -2 ../05.assembly/$i.LargeContigs.fna -t $CORES \
-	    -S $TMPDIR/03.ani.db -a --name1 $DATASET --name2 $i \
-	    --no-save-regions --no-save-rbm --lookup-first \
-	    || echo "")
-      fi
-   fi
-   echo "$AAI;$ANI"
-   checkpoint_n
+  echo "[ $(date "+%Y-%m-%d %H:%M:%S %z") ] $i"
+  AAI=""; ANI="";
+  # Check if the i-th dataset is ready
+  [[ -s $ESS/$i.done && -s $ESS/$i.json ]] || continue
+  # Check if this is done (e.g., in a previous failed iteration)
+  AAI=$(aai_from_db $DATASET $i $TMPDIR/02.aai.db)
+  # Try the other direction
+  [[ "${AAI%.*}" -le 0 ]] && AAI=$(aai_from_db $i $DATASET 02.aai/$i.db)
+  # Try with hAAI
+  if [[ "${AAI%.*}" -le 0 ]] ; then
+    [[ -e "$TMPDIR/$DATASET.ess.faa" ]] \
+      || cp $ESS/$DATASET.ess.faa $TMPDIR/$DATASET.ess.faa
+    AAI=$(haai $TMPDIR/$DATASET.ess.faa $ESS/$i.ess.faa \
+      $CORES $TMPDIR/01.haai.db $TMPDIR/02.aai.db)
+  fi
+  # Try with complete AAI
+  if [[ "${AAI%.*}" -le 0 ]] ; then
+    [[ -e "$TMPDIR/$DATASET.faa" ]] \
+      || cp ../06.cds/$DATASET.faa $TMPDIR/$DATASET.faa
+    AAI=$(aai $TMPDIR/$DATASET.faa ../06.cds/$i.faa $CORES $TMPDIR/02.aai.db)
+  fi
+  # Check if ANI is meaningful
+  if [[ -e "../05.assembly/$DATASET.LargeContigs.fna" \
+      && -e "../05.assembly/$i.LargeContigs.fna" \
+      && $(perl -e "print 1 if '$AAI' >= 90") == "1" ]] ; then
+    # Check if this is done (e.g., in a previous failed iteration)
+    ANI=$(ani_from_db $DATASET $i $TMPDIR/03.ani.db)
+    # Try the other direction
+    [[ "${ANI%.*}" -le 0 ]] && ANI=$(ani_from_db $i $DATASET 03.ani/$i.db)
+    # Calculate it
+    if [[ "${ANI%.*}" -le 0 ]] ; then
+      [[ -e "$TMPDIR/$DATASET.LargeContigs.fna" ]] \
+        || cp ../05.assembly/$DATASET.LargeContigs.fna \
+          $TMPDIR/$DATASET.LargeContigs.fna
+      ANI=$(ani $TMPDIR/$DATASET.LargeContigs.fna \
+        ../05.assembly/$i.LargeContigs.fna $CORES $TMPDIR/03.ani.db)
+    fi
+  fi
+  checkpoint_n
 done
 N=10
 checkpoint_n

data/scripts/assembly.bash

@@ -11,12 +11,26 @@ b=$DATASET
 # Initialize
 date "+%Y-%m-%d %H:%M:%S %z" > "$DATASET.start"
 
+# Interpose (if needed)
+TF="../04.trimmed_fasta"
+if [[ -s $TF/$DATASET.1.fasta \
+      && -s $TF/$DATASET.2.fasta \
+      && ! -s $TF/$DATASET.CoupledReads.fa ]] ; then
+  FastA.interpose.pl $TF/$DATASET.CoupledReads.fa $TF/$DATASET.[12].fasta
+  gzip -9 -f $TF/$DATASET.1.fasta
+  gzip -9 -f $TF/$DATASET.2.fasta
+  miga add_result -P "$PROJECT" -D "$DATASET" -r trimmed_fasta
+fi
+
 # Assemble
-FA="../04.trimmed_fasta/$DATASET.CoupledReads.fa"
+FA="$TF/$DATASET.CoupledReads.fa"
 [[ -e $FA ]] || FA="$FA.gz"
 [[ -e $FA ]] || FA="../04.trimmed_fasta/$DATASET.SingleReads.fa"
 [[ -e $FA ]] || FA="$FA.gz"
-idba_ud --pre_correction -r "$FA" -o "$DATASET" --num_threads "$CORES"
+RD="r"
+[[ $FA == *.SingleReads.fa* ]] && RD="l"
+idba_ud --pre_correction -$RD "$FA" -o "$DATASET" --num_threads "$CORES" || true
+[[ -s $DATASET/contig.fa ]] || exit 1
 
 # Clean
 cd $DATASET
@@ -36,4 +50,3 @@ FastA.length.pl $DATASET.AllContigs.fna | awk '$2>=1000{print $1}' \
 # Finalize
 date "+%Y-%m-%d %H:%M:%S %z" > "$DATASET.done"
 miga add_result -P "$PROJECT" -D "$DATASET" -r assembly
-