miga-base 0.7.26.0 → 1.0.0.1

data/lib/miga/metadata.rb

@@ -105,7 +105,11 @@ class MiGA::Metadata < MiGA::MiGA
   ##
   # Return the value of +k+ in #data
   def [](k)
-    data[k.to_sym]
+    if k.to_s =~ /(.+):(.+)/
+      data[$1.to_sym]&.fetch($2)
+    else
+      data[k.to_sym]
+    end
   end
 
   ##

data/lib/miga/parallel.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+##
+# Parallel execution in MiGA.
+class MiGA::Parallel < MiGA::MiGA
+  class << self
+    ##
+    # Executes the passed block with the thread number as argument (0-numbered)
+    # in +threads+ processes
+    def process(threads)
+      threads.times do |i|
+        Process.fork { yield(i) }
+      end
+      Process.waitall
+    end
+
+    ##
+    # Distributes +enum+ across +threads+ and calls the passed block with args:
+    # 1. Unitary object from +enum+
+    # 2. Index of the unitary object
+    # 3. Index of the acting thread
+    def distribute(enum, threads, &blk)
+      process(threads) { |thr| thread_enum(enum, threads, thr, &blk) }
+    end
+
+    ##
+    # Enum through +enum+ executing the passed block only for thread with index
+    # +thr+, one of +threads+ threads. The passed block has the same arguments
+    # as the one in +#distribute+
+    def thread_enum(enum, threads, thr)
+      enum.each_with_index do |obj, idx|
+        yield(obj, idx, thr) if idx % threads == thr
+      end
+    end
+  end
+end

data/lib/miga/project.rb

@@ -42,18 +42,18 @@ class MiGA::Project < MiGA::MiGA
   # Create an empty project
   def create
     unless MiGA::MiGA.initialized?
-      raise 'Impossible to create project in uninitialized MiGA.'
+      warn 'Projects cannot be processed yet, first run: miga init'
     end
 
-    dirs = [path] + @@FOLDERS.map { |d| "#{path}/#{d}" } +
-           @@DATA_FOLDERS.map { |d| "#{path}/data/#{d}" }
-    dirs.each { |d| Dir.mkdir(d) unless Dir.exist? d }
+    dirs = @@FOLDERS.map { |d| File.join(path, d) }
+    dirs += @@DATA_FOLDERS.map { |d| File.join(path, 'data', d) }
+    dirs.each { |d| FileUtils.mkdir_p(d) }
     @metadata = MiGA::Metadata.new(
-      File.expand_path('miga.project.json', path),
-      { datasets: [], name: File.basename(path) }
+      File.join(path, 'miga.project.json'),
+      datasets: [], name: File.basename(path)
     )
-    d_path = File.expand_path('daemon/daemon.json', path)
-    File.open(d_path, 'w') { |fh| fh.puts '{}' } unless File.exist? d_path
+    d_path = File.join(path, 'daemon', 'daemon.json')
+    File.open(d_path, 'w') { |fh| fh.puts '{}' } unless File.exist?(d_path)
     pull_hook :on_create
     self.load
   end

data/lib/miga/project/base.rb

@@ -131,15 +131,15 @@ module MiGA::Project::Base
     },
     haai_p: {
       desc: 'Value of aai.rb -p on hAAI', type: String,
-      default: proc { |project| project.clade? ? 'no' : 'blast+' },
-      in: %w[fastaai blast+ blast blat diamond no]
+      default: proc { |project| project.clade? ? 'no' : 'fastaai' },
+      in: %w[blast+ blast blat diamond fastaai no]
     },
     aai_p: {
-      desc: 'Value of aai.rb -p on AAI', default: 'blast+', type: String,
+      desc: 'Value of aai.rb -p on AAI', default: 'diamond', type: String,
       in: %w[blast+ blast blat diamond]
     },
     ani_p: {
-      desc: 'Value of ani.rb -p on ANI', default: 'blast+', type: String,
+      desc: 'Value of ani.rb -p on ANI', default: 'fastani', type: String,
       in: %w[blast+ blast blat fastani]
     },
     max_try: {

data/lib/miga/project/result.rb

@@ -55,12 +55,12 @@ module MiGA::Project::Result
   ##
   # Add result of any type +:*_distances+ at +base+ (no +_opts+ supported).
   def add_result_distances(base, _opts)
-    return nil unless result_files_exist?(base, %w[.Rdata .log .txt])
+    return nil unless result_files_exist?(base, %w[.Rdata .txt])
 
     r = MiGA::Result.new("#{base}.json")
     r.add_file(:rdata,  'miga-project.Rdata')
     r.add_file(:matrix, 'miga-project.txt')
-    r.add_file(:log,    'miga-project.log')
+    r.add_file(:log,    'miga-project.log') # Legacy file
     r.add_file(:hist,   'miga-project.hist')
     r
   end

data/lib/miga/sqlite.rb

@@ -37,12 +37,20 @@ class MiGA::SQLite < MiGA::MiGA
   # Executes +cmd+ and returns the result
   def run(*cmd)
     busy_attempts ||= 0
-    conn = SQLite3::Database.new(path)
-    conn.execute(*cmd)
+    io_attempts ||= 0
+    y = nil
+    SQLite3::Database.new(path) { |conn| y = conn.execute(*cmd) }
+    y
   rescue SQLite3::BusyException => e
     busy_attempts += 1
     raise "Database busy #{path}: #{e.message}" if busy_attempts >= 3
 
+    sleep(1)
+    retry
+  rescue SQLite3::IOException => e
+    io_attempts += 1
+    raise "Database I/O error #{path}: #{e.message}" if io_attempts >= 3
+
     sleep(1)
     retry
   end

data/lib/miga/version.rb

@@ -9,23 +9,33 @@ module MiGA
   # Current version of MiGA. An Array with three values:
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
-  # - Integer representing minor changes that require new version number.
-  VERSION = [0.7, 26, 0].freeze
+  # - String indicating release status:
+  #   - rc* release candidate, not released as gem
+  #   - [0-9]+ stable release, released as gem
+  VERSION = [1.0, 0, 1].freeze
 
   ##
   # Nickname for the current major.minor version.
-  VERSION_NAME = 'lithograph'
+  VERSION_NAME = 'prima'
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2021, 3, 1)
+  VERSION_DATE = Date.new(2021, 4, 13)
 
   ##
-  # Reference of MiGA.
-  CITATION = 'Rodriguez-R et al (2018). ' \
-    'The Microbial Genomes Atlas (MiGA) webserver: taxonomic and gene ' \
-    'diversity analysis of Archaea and Bacteria at the whole genome level. ' \
-    'Nucleic Acids Research 46(W1):W282-W288. doi:10.1093/nar/gky467.'
+  # References of MiGA
+  CITATION = []
+  CITATION << <<~REF
+    Rodriguez-R et al (2018). The Microbial Genomes Atlas (MiGA) webserver:
+      taxonomic and gene diversity analysis of Archaea and Bacteria at the whole
+      genome level. Nucleic Acids Research 46(W1):W282-W288.
+      doi:10.1093/nar/gky467.
+  REF
+  CITATION << <<~REF
+    Rodriguez-R et al (2020). Classifying prokaryotic genomes using the
+      Microbial Genomes Atlas (MiGA) webserver. Bergey's Manual of Systematics
+      of Archaea and Bacteria.
+  REF
 end
 
 class MiGA::MiGA
@@ -58,6 +68,10 @@ class MiGA::MiGA
   ##
   # Reference of MiGA
   def self.CITATION
+    CITATION.map { |i| "- #{i}" }.join
+  end
+
+  def self.CITATION_ARRAY
     CITATION
   end
 end

data/scripts/aai_distances.bash

@@ -9,34 +9,32 @@ DIR="$PROJECT/data/09.distances/02.aai"
 # Initialize
 miga_start_project_step "$DIR"
 
-echo -n "" > miga-project.log
-DS=$(miga ls -P "$PROJECT" --ref --no-multi --active)
-
 # Extract values
 rm -f miga-project.txt
+SQL="SELECT seq1, seq2, aai, sd, n, omega from aai;"
+DS=$(miga ls -P "$PROJECT" --ref --no-multi --active)
 (
-  echo "metric a b value sd n omega" | tr " " "\\t"
+  echo "a b value sd n omega" | tr " " "\\t"
   for i in $DS ; do
-    echo "SELECT CASE WHEN omega!=0 THEN 'AAI' ELSE 'hAAI_AAI' END," \
-      " seq1, seq2, aai, sd, n, omega from aai;" \
-      | sqlite3 "$DIR/$i.db" | tr "\\|" "\\t"
-    echo "$i" >> miga-project.log
+    echo "$SQL" | sqlite3 "$DIR/$i.db" | tr "\\|" "\\t"
   done
 ) | gzip -9c > miga-project.txt.gz
 
 # R-ify
-echo "
-aai <- read.table(gzfile('miga-project.txt.gz'), sep='\\t', h=T, as.is=TRUE);
-save(aai, file='miga-project.Rdata');
-if(sum(aai[,'a'] != aai[,'b']) > 0){
-  h <- hist(aai[aai[,'a'] != aai[,'b'], 'value'], breaks=100, plot=FALSE);
+cat <<R | R --vanilla
+file <- gzfile('miga-project.txt.gz')
+aai <- read.table(file, sep = '\t', header = TRUE, as.is = TRUE)
+save(aai, file = 'miga-project.Rdata')
+if(sum(aai[, 'a'] != aai[, 'b']) > 0) {
+  h <- hist(aai[aai[, 'a'] != aai[, 'b'], 'value'], breaks = 100, plot = FALSE)
+  len <- length(h[['breaks']])
   write.table(
-    cbind(h[['breaks']][-length(h[['breaks']])],
-      h[['breaks']][-1], h[['counts']]),
-    file='miga-project.hist', quote=FALSE, sep='\\t',
-    col.names=FALSE, row.names=FALSE);
+    cbind(h[['breaks']][-len], h[['breaks']][-1], h[['counts']]),
+    file = 'miga-project.hist', quote = FALSE, sep = '\t',
+    col.names = FALSE, row.names = FALSE
+  )
 }
-" | R --vanilla
+R
 
 # Finalize
 miga_end_project_step "$DIR"

data/scripts/ani_distances.bash

@@ -9,33 +9,32 @@ DIR="$PROJECT/data/09.distances/03.ani"
 # Initialize
 miga_start_project_step "$DIR"
 
-echo -n "" > miga-project.log
-DS=$(miga ls -P "$PROJECT" --ref --no-multi --active)
-
 # Extract values
 rm -f miga-project.txt
+SQL="SELECT seq1, seq2, ani, sd, n, omega from ani;"
+DS=$(miga ls -P "$PROJECT" --ref --no-multi --active)
 (
-  echo "metric a b value sd n omega" | tr " " "\\t"
+  echo "a b value sd n omega" | tr " " "\\t"
   for i in $DS ; do
-    echo "SELECT 'ANI', seq1, seq2, ani, sd, n, omega from ani ;" \
-      | sqlite3 "$DIR/$i.db" | tr "\\|" "\\t"
-    echo "$i" >> miga-project.log
+    echo "$SQL" | sqlite3 "$DIR/$i.db" | tr "\\|" "\\t"
   done
 ) | gzip -9c > miga-project.txt.gz
 
 # R-ify
-echo "
-ani <- read.table(gzfile('miga-project.txt.gz'), sep='\\t', h=T, as.is=TRUE);
-save(ani, file='miga-project.Rdata');
-if(sum(ani[,'a'] != ani[,'b']) > 0){
-  h <- hist(ani[ani[,'a'] != ani[,'b'], 'value'], breaks=100, plot=FALSE);
+cat <<R | R --vanilla
+file <- gzfile('miga-project.txt.gz')
+ani <- read.table(file, sep = '\t', header = TRUE, as.is = TRUE)
+save(ani, file = 'miga-project.Rdata')
+if(sum(ani[, 'a'] != ani[, 'b']) > 0) {
+  h <- hist(ani[ani[, 'a'] != ani[, 'b'], 'value'], breaks = 100, plot = FALSE)
+  len <- length(h[['breaks']])
   write.table(
-    cbind(h[['breaks']][-length(h[['breaks']])],
-      h[['breaks']][-1], h[['counts']]),
-    file='miga-project.hist', quote=FALSE, sep='\\t',
-    col.names=FALSE, row.names=FALSE);
+    cbind(h[['breaks']][-len], h[['breaks']][-1], h[['counts']]),
+    file = 'miga-project.hist', quote = FALSE, sep = '\t',
+    col.names = FALSE, row.names = FALSE
+  )
 }
-" | R --vanilla
+R
 
 # Finalize
 miga_end_project_step "$DIR"

data/scripts/assembly.bash

@@ -11,30 +11,44 @@ miga date > "$DATASET.start"
 
 # Interpose (if needed)
 TF="../04.trimmed_fasta"
-if [[ -s "$TF/$DATASET.1.fasta" \
-      && -s "$TF/$DATASET.2.fasta" \
-      && ! -s "$TF/$DATASET.CoupledReads.fa" ]] ; then
-  FastA.interpose.pl "$TF/$DATASET.CoupledReads.fa" "$TF/$DATASET".[12].fasta
-  gzip -9 -f "$TF/$DATASET.1.fasta"
-  gzip -9 -f "$TF/$DATASET.2.fasta"
-  miga add_result -P "$PROJECT" -D "$DATASET" -r trimmed_fasta -f
+b=$DATASET
+if [[ -s "$TF/${b}.2.fasta" || -s "$TF/${b}.2.fasta.gz" ]] ; then
+  cr="$TF/${b}.CoupledReads.fa"
+  if [[ ! -s "$cr" && ! -s "${cr}.gz" ]] ; then
+    for s in 1 2 ; do
+      if [[ -s "$TF/${b}.${s}.fasta" ]] ; then
+        ln -s "$TF/${b}.${s}.fasta" "${b}.${s}.tmp"
+      else
+        gzip -cd "$TF/${b}.${s}.fasta.gz" > "${b}.${s}.tmp"
+      fi
+    done
+    FastA.interpose.pl "$cr" "$b".[12].tmp
+    rm "$b".[12].tmp
+    miga add_result -P "$PROJECT" -D "$DATASET" -r trimmed_fasta -f
+  fi
 fi
 
+# Gzip (if needed)
+for i in SingleReads CoupledReads ; do
+  base="$TF/${DATASET}.${i}.fa"
+  if [[ -e "$base" && ! -s "${base}.gz" ]] ; then
+    gzip -9f "$base"
+    miga add_result -P "$PROJECT" -D "$DATASET" -r trimmed_fasta -f
+  fi
+done
+
 # Assemble
-FA="$TF/$DATASET.CoupledReads.fa"
-[[ -e "$FA" ]] || FA="$FA.gz"
-[[ -e "$FA" ]] || FA="../04.trimmed_fasta/$DATASET.SingleReads.fa"
-[[ -e "$FA" ]] || FA="$FA.gz"
+FA="$TF/${DATASET}.CoupledReads.fa.gz"
+[[ -e "$FA" ]] || FA="$TF/${DATASET}.SingleReads.fa.gz"
 RD="r"
 [[ $FA == *.SingleReads.fa* ]] && RD="l"
-idba_ud --pre_correction -$RD "$FA" -o "$DATASET" --num_threads "$CORES" || true
+gzip -cd "$FA" \
+  | idba_ud --pre_correction -$RD /dev/stdin \
+    -o "$DATASET" --num_threads "$CORES" || true
 [[ -s "$DATASET/contig.fa" ]] || exit 1
 
 # Clean
-(
-  cd "$DATASET"
-  rm kmer graph-*.fa align-* local-contig-*.fa contig-*.fa
-)
+( cd "$DATASET" && rm kmer graph-*.fa align-* local-contig-*.fa contig-*.fa )
 
 # Extract
 if [[ -s "$DATASET/scaffold.fa" ]] ; then
@@ -49,3 +63,4 @@ FastA.length.pl "$DATASET.AllContigs.fna" | awk '$2>=1000{print $1}' \
 # Finalize
 miga date > "$DATASET.done"
 miga add_result -P "$PROJECT" -D "$DATASET" -r "$SCRIPT" -f
+

data/scripts/haai_distances.bash

@@ -12,34 +12,10 @@ miga_start_project_step "$DIR"
 # Cleanup databases
 ruby -I "$MIGA/lib" "$MIGA/utils/cleanup-databases.rb" "$PROJECT" "$CORES"
 
-# Run hAAI
+# No real need for hAAI distributions at all
 echo -n "" > miga-project.log
-DS=$(miga ls -P "$PROJECT" --ref --no-multi --active)
-
-# Extract values
-rm -f miga-project.txt
-(
-  echo "metric a b value sd n omega" | tr " " "\\t"
-  for i in $DS ; do
-    echo "SELECT 'hAAI', seq1, seq2, aai, sd, n, omega from aai ;" \
-      | sqlite3 "$DIR/$i.db" | tr "\\|" "\\t"
-    echo "$i" >> miga-project.log
-  done
-) | gzip -9c > miga-project.txt.gz
-
-# R-ify
-echo "
-haai <- read.table(gzfile('miga-project.txt.gz'), sep='\\t', h=T, as.is=TRUE);
-save(haai, file='miga-project.Rdata');
-if(sum(haai[,'a'] != haai[,'b']) > 0){
-  h <- hist(haai[haai[,'a'] != haai[,'b'], 'value'], breaks=100, plot=FALSE);
-  write.table(
-    cbind(h[['breaks']][-length(h[['breaks']])],
-      h[['breaks']][-1], h[['counts']]),
-    file='miga-project.hist', quote=FALSE, sep='\\t',
-    col.names=FALSE, row.names=FALSE);
-}
-" | R --vanilla
+echo -n "" > miga-project.txt
+echo "aai <- NULL; save(aai, file = 'miga-project.Rdata')" | R --vanilla
 
 # Finalize
 miga_end_project_step "$DIR"

data/scripts/miga.bash

@@ -7,9 +7,11 @@ SCRIPT=${SCRIPT:-$(basename "$0" .bash)}
 # shellcheck source=/dev/null
 . "$MIGA_HOME/.miga_rc"
 
-# Ensure submodules are first in PATH
-export PATH="$MIGA/bin:$MIGA/utils/enveomics/Scripts:$PATH"
-export PATH="$MIGA/utils/FastAAI/FastAAI:$PATH"
+# Ensure MiGA & submodules are first in PATH
+export PATH="$MIGA/bin:$PATH"
+for util in enveomics/Scripts FastAAI/FastAAI multitrim ; do
+  export PATH="$MIGA/utils/$util:$PATH"
+done
 
 # Ancillary functions
 function exists { [[ -e "$1" ]] ; }
@@ -38,7 +40,7 @@ if [[ "$SCRIPT" != "d" && "$SCRIPT" != "p" ]] ; then
   echo ""
   echo "######[ $SCRIPT ]######"
   echo "# Date: $(miga date)"
-  echo "# Host: $(hostname)"
+  echo "# Host: $(hostname) [$CORES]"
   echo "# MiGA: $MIGA"
   echo "# Project: $PROJECT"
   if [[ -n $DATASET ]] ; then

data/scripts/p.bash

@@ -12,7 +12,7 @@ while true ; do
   if [[ "$res" == "$last_res" ]] ; then
     let k=$k+1
     if [[ $k -gt 10 ]] ; then
-      miga new --update -P "$PROJECT" \
+      miga edit -P "$PROJECT" \
         -m "run_$res=false,warn=Too many failed attempts to run $res"
     fi
   else

data/scripts/read_quality.bash

@@ -6,28 +6,19 @@ SCRIPT="read_quality"
 . "$MIGA/scripts/miga.bash" || exit 1
 cd "$PROJECT/data/03.read_quality"
 
-b=$DATASET
-
 # Initialize
 miga date > "$DATASET.start"
 
-# FastQC
-[[ -d "$b.fastqc" ]] || mkdir "$b.fastqc"
-fastqc "../02.trimmed_reads/$b".[12].clipped.fastq -o "$b.fastqc"
-
-# SolexaQA++
-[[ -d "$b.solexaqa" ]] || mkdir "$b.solexaqa"
-exists "../02.trimmed_reads/$b".[12].*.pdf \
-  && mv "../02.trimmed_reads/$b".[12].*.pdf "$b.solexaqa/"
-
-# Clean 02.trimmed_reads
-rm -f "../02.trimmed_reads/$b".[12].fastq_trimmed.segments
-rm -f "../02.trimmed_reads/$b".[12].fastq.trimmed.paired
-rm -f "../02.trimmed_reads/$b".[12].fastq.trimmed.single
-rm -f "../02.trimmed_reads/$b".[12].fastq.trimmed
-rm -f "../02.trimmed_reads/$b".[12].fastq
-miga add_result -P "$PROJECT" -D "$DATASET" -r trimmed_reads -f
+# Gzip (if necessary)
+for s in 1 2 ; do
+  in="../02.trimmed_reads/${DATASET}.${s}.clipped.fastq"
+  if [[ -s "$in" ]] ; then
+    gzip -9f "$in"
+    miga add_result -P "$PROJECT" -D "$DATASET" -r trimmed_reads -f
+  fi
+done
 
 # Finalize
 miga date > "$DATASET.done"
 miga add_result -P "$PROJECT" -D "$DATASET" -r "$SCRIPT" -f
+