miga-base 0.7.26.1 → 1.0.2.0

data/scripts/trimmed_fasta.bash

@@ -11,43 +11,27 @@ b=$DATASET
 # Initialize
 miga date > "$DATASET.start"
 
-# Gunzip (if necessary)
-for sis in 1 2 ; do
-  for ext in clipped clipped.single ; do
-    [[ -e "../02.trimmed_reads/$b.$sis.${ext}.fastq.gz" \
-      && ! -e "../02.trimmed_reads/$b.$sis.${ext}.fastq" ]] \
-        && gzip -d "../02.trimmed_reads/$b.$sis.${ext}.fastq.gz"
-  done
+# FastQ -> FastA
+for s in 1 2 ; do
+  in="../02.trimmed_reads/${b}.${s}.clipped.fastq.gz"
+  [[ -s "$in" ]] \
+    && FastQ.maskQual.rb -i "$in" -o "${b}.1.fasta" --fasta --qual 18
 done
-miga add_result -P "$PROJECT" -D "$DATASET" -r trimmed_reads -f
 
-# FastQ -> FastA
-FQ2A="$MIGA/utils/enveomics/Scripts/FastQ.toFastA.awk"
-awk -f "$FQ2A" < "../02.trimmed_reads/$b.1.clipped.fastq" > "$b.1.fasta"
-if [[ -e "../02.trimmed_reads/$b.2.clipped.fastq" ]] ; then
-  awk -f "$FQ2A" < "../02.trimmed_reads/$b.2.clipped.fastq" > "$b.2.fasta"
-  FastA.interpose.pl "$b.CoupledReads.fa" "$b".[12].fasta
-  gzip -9 -f "$b.2.fasta"
-  gzip -9 -f "$b.1.fasta"
-  awk -f "$FQ2A" < "../02.trimmed_reads/$b".[12].clipped.single.fastq \
-    > "$b.SingleReads.fa"
-  gzip -9 -f "$b.SingleReads.fa"
+# Interpose
+if [[ -e "${b}.2.fasta" ]] ; then
+  FastA.interpose.pl "${b}.CoupledReads.fa" "$b".[12].fasta
 else
-  mv "$b.1.fasta" "$b.SingleReads.fa"
+  mv "${b}.1.fasta" "${b}.SingleReads.fa"
 fi
 
-# Compress input at 01.raw_reads and 02.trimmed_reads
-for sis in 1 2 ; do
-  [[ -e "../01.raw_reads/$b.$sis.fastq" ]] \
-    && gzip -9 -f "../01.raw_reads/$b.$sis.fastq"
-  [[ -e "../02.trimmed_reads/$b.$sis.clipped.fastq" ]] \
-    && gzip -9 -f "../02.trimmed_reads/$b.$sis.clipped.fastq"
-  [[ -e "../02.trimmed_reads/$b.$sis.clipped.single.fastq" ]] \
-    && gzip -9 -f "../02.trimmed_reads/$b.$sis.clipped.single.fastq"
+# Gzip
+for x in 1.fasta 2.fasta SingleReads.fa CoupledReads.fa ; do
+  in="${b}.${x}"
+  [[ -e "$in" ]] && gzip -9f "$in"
 done
-miga add_result -P "$PROJECT" -D "$DATASET" -r raw_reads -f
-miga add_result -P "$PROJECT" -D "$DATASET" -r trimmed_reads -f
 
 # Finalize
 miga date > "$DATASET.done"
 miga add_result -P "$PROJECT" -D "$DATASET" -r "$SCRIPT" -f
+

data/scripts/trimmed_reads.bash

@@ -11,49 +11,49 @@ b=$DATASET
 # Initialize
 miga date > "$DATASET.start"
 
-# Unzip (if necessary)
-[[ -e "../01.raw_reads/$b.1.fastq.gz" && ! -e "../01.raw_reads/$b.1.fastq" ]] \
-  && gunzip "../01.raw_reads/$b.1.fastq.gz"
-[[ -e "../01.raw_reads/$b.2.fastq.gz" && ! -e "../01.raw_reads/$b.2.fastq" ]] \
-  && gunzip "../01.raw_reads/$b.2.fastq.gz"
-miga add_result -P "$PROJECT" -D "$DATASET" -r raw_reads -f
-
 # Clean existing files
 exists "$b".[12].* && rm "$b".[12].*
 
+# Gzip (if necessary)
+for s in 1 2 ; do
+  in="../01.raw_reads/${b}.${s}.fastq"
+  if [[ -s "$in" ]] ; then
+    gzip -9f "$in"
+    miga add_result -P "$PROJECT" -D "$DATASET" -r raw_reads -f
+  fi
+done
+
 # Tag
-FastQ.tag.rb -i "../01.raw_reads/$b.1.fastq" -p "$b-" -s "/1" -o "$b.1.fastq"
-[[ -e "../01.raw_reads/$b.2.fastq" ]] \
-  && FastQ.tag.rb -i "../01.raw_reads/$b.2.fastq" -p "$b-" -s "/2" \
-      -o "$b.2.fastq"
-
-# Trim
-SolexaQA++ dynamictrim "$b".[12].fastq -h 20 -d .
-SolexaQA++ lengthsort  "$b".[12].fastq.trimmed -l 50 -d .
-
-# Clean adapters
-if [[ -e "$b.2.fastq.trimmed.paired" ]] ; then
-  scythe -a "$MIGA/utils/adapters.fa" "$b.1.fastq.trimmed.paired" \
-    > "$b.1.clipped.all.fastq"
-  scythe -a "$MIGA/utils/adapters.fa" "$b.2.fastq.trimmed.paired" \
-    > "$b.2.clipped.all.fastq"
-  SolexaQA++ lengthsort "$b".[12].clipped.all.fastq -l 50 -d .
-  rm "$b".[12].clipped.all.fastq
-  [[ -e "$b".1.clipped.all.fastq.single ]] \
-    && mv "$b.1.clipped.all.fastq.single" "$b.1.clipped.single.fastq"
-  [[ -e "$b".2.clipped.all.fastq.single ]] \
-    && mv "$b.2.clipped.all.fastq.single" "$b.2.clipped.single.fastq"
-  mv "$b.1.clipped.all.fastq.paired" "$b.1.clipped.fastq"
-  mv "$b.2.clipped.all.fastq.paired" "$b.2.clipped.fastq"
-  rm -f "$b.1.clipped.all.fastq.summary.txt"
+in1="../01.raw_reads/$b.1.fastq.gz"
+in2="../01.raw_reads/$b.2.fastq.gz"
+FastQ.tag.rb -i "$in1" -p "$b-" -s "/1" -o "$b.1.fastq.gz"
+[[ -e "$in2" ]] && FastQ.tag.rb -i "$in2" -p "$b-" -s "/2" -o "$b.2.fastq.gz"
+
+# Multitrim
+CMD="multitrim.py --zip gzip --level 9 --threads $CORES -o $b"
+if [[ -s "$b.2.fastq.gz" ]] ; then
+  # Paired
+  $CMD -1 "$b.1.fastq.gz" -2 "$b.2.fastq.gz"
+  for s in 1 2 ; do
+    mv "$b/${s}.post_trim_${b}.${s}.fq.gz" "${b}.${s}.clipped.fastq.gz"
+    mv "$b/${s}.pre_trim_QC_${b}.${s}.html" "../03.read_quality/${b}.pre.${s}.html"
+    mv "$b/${s}.post_trim_QC_${b}.${s}.html" "../03.read_quality/${b}.post.${s}.html"
+  done
 else
-  scythe -a "$MIGA/utils/adapters.fa" "$b.1.fastq.trimmed.single" \
-    > "$b.1.clipped.all.fastq"
-  SolexaQA++ lengthsort "$b.1.clipped.all.fastq" -l 50 -d .
-  mv "$b.1.clipped.all.fastq.single" "$b.1.clipped.fastq"
+  # Unpaired
+  $CMD -u "$b.1.fastq.gz"
+  mv "$b/unpaired.post_trim_${b}.1.fq.gz" "${b}.1.clipped.fastq.gz"
+  mv "$b/unpaired.pre_trim_QC_${b}.1.html" "../03.read_quality/${b}.pre.1.html"
+  mv "$b/unpaired.post_trim_QC_${b}.1.html" "../03.read_quality/${b}.post.1.html"
 fi
-rm -f "$b".[12].*.discard
+mv "$b/Subsample_Adapter_Detection.stats.txt" \
+  "../03.read_quality/$b.adapters.txt"
+
+# Cleanup
+rm -r "$b"
+rm -f "$b".[12].fastq.gz
 
 # Finalize
 miga date > "$DATASET.done"
 miga add_result -P "$PROJECT" -D "$DATASET" -r "$SCRIPT" -f
+

data/test/parallel_test.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class ParallelTest < Test::Unit::TestCase
+  include TestHelper
+
+  def test_distribute
+    declare_forks
+
+    base = tmpfile('base')
+    assert(!File.exist?("#{base}-3"))
+    MiGA::Parallel.distribute((0..3), 2) do |o, _k, t|
+      File.open("#{base}-#{o}", 'w') { |fh| fh.puts t }
+    end
+    assert(File.exist?("#{base}-3"))
+    assert(!File.exist?("#{base}-4"))
+    t = (0..3).map { |i| File.read("#{base}-#{i}").chomp.to_i }
+    assert_equal([0, 0, 1, 1], t.sort)
+  end
+
+  def test_thread_enum
+    MiGA::Parallel.thread_enum(%w[a b c d], 3, 1) do |o, _k, _t|
+      assert_equal('b', o)
+    end
+
+    n = 0
+    MiGA::Parallel.thread_enum(0..19, 4, 0) { n += 1 }
+    assert_equal(5, n)
+  end
+end

data/test/project_test.rb

@@ -27,10 +27,11 @@ class ProjectTest < Test::Unit::TestCase
   def test_create
     assert_equal(tmpfile('create'), project('create').path)
     assert_path_exist(tmpfile('create'))
-    assert_raise do
+    err = capture_stderr do
       ENV['MIGA_HOME'] = tmpfile('chez-moi')
       project('cuckoo')
     end
+    assert_match(/Projects cannot be processed yet/, err.string)
   end
 
   def test_load

data/test/remote_dataset_test.rb

@@ -47,7 +47,7 @@ class RemoteDatasetTest < Test::Unit::TestCase
   end
 
   def test_net_ftp
-    cjac = 'ftp://ftp.ebi.ac.uk/pub/databases/ena/tsa/public/ga/GAPJ01.fasta.gz'
+    cjac = 'ftp://ftp.ebi.ac.uk/pub/databases/ena/tsa/public/gap/GAPJ01.fasta.gz'
     n = 'Cjac_L14'
     rd = MiGA::RemoteDataset.new(cjac, :assembly_gz, :web)
     assert_equal([cjac], rd.ids)

data/utils/distance/commands.rb

@@ -169,6 +169,7 @@ module MiGA::DistanceRunner::Commands
         aai_data[out[1]] = [out[6].to_f, 0, 0, 0] if out[6] !~ /^>/
       end
     end
+    puts "Results: #{haai_data.size} | Inferences: #{aai_data.size}"
     batch_data_to_db(:haai, haai_data)
     batch_data_to_db(:aai, aai_data)
 

data/utils/distance/database.rb

@@ -126,7 +126,6 @@ module MiGA::DistanceRunner::Database
   def batch_data_to_db(metric, data)
     db = tmp_dbs[metric]
     table = metric == :haai ? :aai : metric
-    `cp #{db} ~/here.db`
     SQLite3::Database.new(db) do |conn|
       data.each do |k, v|
         sql = <<~SQL

data/utils/distance/runner.rb

@@ -18,10 +18,8 @@ class MiGA::DistanceRunner
       @ref_project = MiGA::Project.load(ref_path)
       raise "Cannot load reference project: #{ref_path}" if @ref_project.nil?
     elsif !opts[:run_taxonomy] && dataset.option(:db_project)
-      ref_path = dataset.option(:db_project)
-      if project.option(:db_proj_dir)
-        ref_path = File.expand_path(ref_path, project.option(:db_proj_dir))
-      end
+      ref_location = project.option(:db_proj_dir) || File.dirname(project.path)
+      ref_path = File.expand_path(dataset.option(:db_project), ref_location)
       @ref_project = MiGA::Project.load(ref_path)
       raise "Cannot load reference project: #{ref_path}" if @ref_project.nil?
     else

data/utils/enveomics/Manifest/Tasks/fasta.json

@@ -4,7 +4,8 @@
       "task": "FastA.N50.pl",
       "description": ["Calculates the N50 value of a set of sequences.",
         "Alternatively, it can calculate other N** values. It also calculates",
-        "the total number of sequences and the total added length."],
+        "the total number of sequences, the total added length, and the",
+        "longest sequence length."],
       "help_arg": "",
       "see_also": ["FastA.length.pl"],
       "options": [
@@ -354,14 +355,14 @@
           "opt": "--in",
           "arg": "in_file",
           "mandatory": true,
-          "description": "Input FastA file."
+          "description": "Input FastA file (supports .gz compression)."
         },
         {
           "name": "Output file",
           "opt": "--out",
           "arg": "out_file",
           "mandatory": true,
-          "description": "Output FastA file."
+          "description": "Output FastA file (supports .gz compression)."
         },
         {
           "opt": "--fraction",
@@ -733,6 +734,41 @@
         }
       ]
     },
+    {
+      "task": "FastA.toFastQ.rb",
+      "description": "Creates a FastQ-compliant file from a FastA file.",
+      "see_also": "FastQ.toFastA.awk",
+      "help_arg": "--help",
+      "options": [
+        {
+          "name": "Input FastA",
+          "opt": "--in",
+          "arg": "in_file",
+          "mandatory": true,
+          "description": "Input FastA file (supports .gz compression)."
+        },
+        {
+          "name": "Output FastQ",
+          "opt": "--out",
+          "arg": "out_file",
+          "mandatory": true,
+          "description": "Output FastQ file (supports .gz compression)."
+        },
+        {
+          "opt": "--quality",
+          "arg": "integer",
+          "default": 31,
+          "description": ["PHRED quality score to use (fixed), in the range",
+            "[-5, 41]."]
+        },
+        {
+          "opt": "--encoding",
+          "arg": "integer",
+          "default": 33,
+          "description": "Base encoding (33 or 64)."
+        }
+      ]
+    },
     {
       "task": "FastA.wrap.rb",
       "description": "Wraps sequences in a FastA to a given line length.",

data/utils/enveomics/Manifest/Tasks/fastq.json

@@ -81,6 +81,47 @@
         }
       ]
     },
+    {
+      "task": "FastQ.maskQual.rb",
+      "description": "Masks low-quality bases in a FastQ file.",
+      "help_arg": "--help",
+      "options": [
+        {
+          "opt": "--input",
+          "arg": "in_file",
+          "mandatory": true,
+          "description": ["Path to the FastQ file containing the sequences.",
+            "Supports compression with .gz extension."]
+        },
+        {
+          "opt": "--output",
+          "arg": "out_file",
+          "mandatory": true,
+          "description": ["Path to the output FastQ file.",
+            "Supports compression with .gz extension."]
+        },
+        {
+          "opt": "--qual",
+          "arg": "integer",
+          "default": 15,
+          "description": "Minimum quality score to allow a base."
+        },
+        {
+          "opt": "--offset",
+          "arg": "integer",
+          "default": 33,
+          "description": "Q-score offset."
+        },
+        {
+          "opt": "--fasta",
+          "description": "Output sequences in FastA format."
+        },
+        {
+          "opt": "--quiet",
+          "description": "Run quietly."
+        }
+      ]
+    },
     {
       "task": "FastQ.offset.pl",
       "description": ["There are several FastQ formats. This script takes a",
@@ -160,14 +201,20 @@
           "opt": "--in",
           "arg": "in_file",
           "mandatory": true,
-          "description": "FastQ file containing the sequences."
+          "description": [
+            "FastQ file containing the sequences.",
+            "Supports compression with .gz extension."
+          ]
         },
         {
           "name": "Output file",
           "opt": "--out",
           "arg": "out_file",
           "mandatory": true,
-          "description": "FastQ to create."
+          "description": [
+            "FastQ to create.",
+            "Supports compression with .gz extension."
+          ]
         },
         {
           "opt": "--prefix",
@@ -188,6 +235,7 @@
     {
       "task": "FastQ.toFastA.awk",
       "description": "Translates FastQ files into FastA.",
+      "see_also": "FastA.toFastQ.rb",
       "help_arg": "'' --help",
       "options": [
         "<",

data/utils/enveomics/Manifest/Tasks/mapping.json

@@ -62,6 +62,76 @@
           "description": "Window size, in base pairs."
         }
       ]
+    },
+    {
+      "task": "sam.filter.rb",
+      "description": ["Filters a SAM or BAM file by target sequences and/or",
+        "identity."],
+      "see_also": ["anir.rb"],
+      "help_arg": "--help",
+      "options": [
+        {
+          "opt": "--genome",
+          "arg": "in_file",
+          "mandatory": true,
+          "description": ["Genome assembly.",
+            "Supports compression with .gz extension."]
+        },
+        {
+          "opt": "--mapping",
+          "arg": "in_file",
+          "mandatory": true,
+          "description": ["Mapping file.",
+            "Supports compression with .gz extension."]
+        },
+        {
+          "opt": "--out-sam",
+          "arg": "out_file",
+          "mandatory": true,
+          "description": ["Output filtered file in SAM format.",
+            "Supports compression with .gz extension."]
+        },
+        {
+          "opt": "--g-format",
+          "arg": "select",
+          "values": ["fasta", "list"],
+          "default": "fasta",
+          "description": ["Genome assembly format."]
+        },
+        {
+          "opt": "--m-format",
+          "arg": "select",
+          "values": ["sam", "bam"],
+          "default": "sam",
+          "description": ["Mapping file format. SAM supports compression with",
+            ".gz file extension."]
+        },
+        {
+          "opt": "--identity",
+          "arg": "float",
+          "description": "Set a fixed threshold of percent identity.",
+          "default": 95.0
+        },
+        {
+          "opt": "--no-header",
+          "description": "Do not include the headers."
+        },
+        {
+          "opt": "--threads",
+          "arg": "integer",
+          "description": "Threads to use.",
+          "default": 2
+        },
+        {
+          "opt": "--log",
+          "arg": "out_file",
+          "description": "Log file to save output."
+        },
+        {
+          "opt": "--quiet",
+          "description": "Run quietly."
+        }
+      ]
     }
   ]
 }