miga-base 0.7.8.0 → 0.7.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5cce1edf29f5f41ad7a53225978513597f2234abd781c5f179c9f45f9d6f1ec2
-  data.tar.gz: 846a570b68efe9dba8acbc1434c55752373d9864df2cea2ff467330bf7e8b1ac
+  metadata.gz: f99f8fd530489d42672cdd96123f321725b9437ee4a81e822a07854ec924ad53
+  data.tar.gz: c4d6607a4b6062b45cc94985b8bc920bb25307232851b20436ee4b9cd8a8986b
 SHA512:
-  metadata.gz: 123117d43c4ef4200ebe4a51cb14d19e48b89d249e66fa27b6e0834cf1c3b738535c3f514856e373259428c42543a25886744b4a8a6775f75b587d0a557e4d48
-  data.tar.gz: fa178d1e899da25212983bf7640cc391f3922e560d5d5ee4ff072781b6301013d928212868c9db1590dfcd1f41449e07b5019765dc866d02295226c9ed8a3e2d
+  metadata.gz: '03478c56a40e948ad9eb4cb09fbedc72bac331072ccf46ce5468f1f78a08e891260684771e455b92841d8af454fcab997d7d51a68360b8337ffa13c8c2ec88a4'
+  data.tar.gz: e992d10e5de206a85ac425e15c7594629f86e8088ec6f59ba820c5ecaf6c8901af0142455501a1d8af03902a3ca559d0374692e4df9f41acdceaf45d5d750b1f

data/README.md

@@ -41,6 +41,7 @@ Developed and maintained by [Luis M. Rodriguez-R][lrr]. MiGA is the result of a
 collaboration between [Kostas Lab][kostas] at the Georgia Institute of
 Technology and [RDP][rdp] at Michigan State University.
 
+See also the [complete list of contributors](manual/part1/contributors.md).
 
 # License
 

data/lib/miga/cli/action/browse.rb

@@ -0,0 +1,213 @@
+# frozen_string_literal: true
+
+require 'miga/cli/action'
+
+# Action: miga browse
+class MiGA::Cli::Action::Browse < MiGA::Cli::Action
+  def parse_cli
+    cli.parse do |opt|
+      cli.defaults = { open: true }
+      cli.opt_object(opt, [:project])
+    end
+  end
+
+  def perform
+    p = cli.load_project
+    create_empty_page(p)
+    generate_project_page(p)
+    say 'Creating dataset pages'
+    cli.load_project.each_dataset do |d|
+      generate_dataset_page(p, d)
+    end
+    generate_datasets_index(p)
+    say "Open in your browser: #{File.join(p.path, 'index.html')}"
+  end
+
+  private
+
+  ##
+  # Create an empty page with necessary assets for project +p+
+  def create_empty_page(p)
+    say 'Creating project page'
+    FileUtils.mkdir_p(browse_file(p, '.'))
+    %w[favicon-32.png style.css].each do |i|
+      FileUtils.cp(template_file(i), browse_file(p, i))
+    end
+    write_file(p, 'about.html') do
+      build_from_template('about.html', citation: MiGA::MiGA.CITATION)
+    end
+  end
+
+  ##
+  # Create landing page for project +p+
+  def generate_project_page(p)
+    # Redirect page
+    write_file(p, '../index.html') { build_from_template('redirect.html') }
+
+    # Summaries
+    summaries = Dir["#{p.path}/*.tsv"].map do |i|
+      "<li><a href='file://#{i}'>#{File.basename(i)}</a></li>"
+    end.join('')
+
+    # Project index page
+    data = {
+      project_active: 'active',
+      information: format_metadata(p),
+      summaries: summaries.empty? ? 'None' : "<ul>#{summaries}</ul>",
+      results: format_results(p)
+    }
+    write_file(p, 'index.html') { build_from_template('index.html', data) }
+  end
+
+  ##
+  # Create page for dataset +d+ within project +p+
+  def generate_dataset_page(p, d)
+    data = {
+      unmiga_name: d.name.unmiga_name,
+      information: format_metadata(d),
+      results: format_results(d)
+    }
+    write_file(p, "d_#{d.name}.html") do
+      build_from_template('dataset.html', data)
+    end
+  end
+
+  ##
+  # Create pages for reference and query dataset indexes
+  def generate_datasets_index(p)
+    say 'Creating index pages'
+    data = format_dataset_index(p)
+    data.each do |k, v|
+      write_file(p, "#{k}_datasets.html") do
+        v[:list] = 'None' if v[:list] == ''
+        build_from_template(
+          'datasets.html',
+          v.merge(:"#{k}_datasets_active" => 'active')
+        )
+      end
+    end
+  end
+
+  def format_dataset_index(p)
+    data = {
+      ref: { type_name: 'Reference', list: '' },
+      qry: { type_name: 'Query', list: '' }
+    }
+    p.each_dataset do |d|
+      data[d.ref? ? :ref : :qry][:list] +=
+        "<li><a href='d_#{d.name}.html'>#{d.name.unmiga_name}</a></li>"
+    end
+    data
+  end
+
+  ##
+  # Format +obj+ metadata as a table
+  def format_metadata(obj)
+    '<table class="table table-sm table-responsive">' +
+      obj.metadata.data.map do |k, v|
+        case k
+        when /^run_/, :plugins, :user
+          next
+        when :web_assembly_gz
+          v = "<a href='#{v}'>#{v[0..50]}...</a>"
+        when :datasets
+          v = v.size
+        end
+        "<tr><td class='text-right pr-4'><b>#{format_name(k)}</b></td>" \
+             "<td>#{v}</td></tr>"
+      end.compact.join('') +
+      '</table>'
+  end
+
+  ##
+  # Format +obj+ results as cards
+  def format_results(obj)
+    o = ''
+    obj.each_result do |key, res|
+      links = format_result_links(res)
+      stats = format_result_stats(res)
+      next unless links || stats
+      name = format_name(key)
+      url_doc =
+        'http://manual.microbial-genomes.org/part5/workflow#' +
+        key.to_s.tr('_', '-')
+      o += <<~CARD
+        <div class="col-md-6 mb-4">
+          <h3>#{name}</h3>
+          <div class='border-left p-3'>
+            #{stats}
+            #{links}
+          </div>
+          <div class='border-top p-2 bg-light'>
+            <a target=_blank href="#{url_doc}" class='p-2'>Learn more</a>
+          </div>
+        </div>
+      CARD
+    end
+    "<div class='row'>#{o}</div>"
+  end
+
+  def format_name(str)
+    str
+      .to_s.unmiga_name
+      .sub(/^./, &:upcase)
+      .gsub(/(Aai|Ani|Ogs|Cds|Ssu| db$| ssu )/, &:upcase)
+      .sub(/Haai/, 'hAAI')
+      .sub(/Mytaxa/, 'MyTaxa')
+      .sub(/ pvalue$/, ' p-value')
+      .sub(/contigs$/, 'Contigs')
+  end
+
+  def format_result_links(res)
+    links = []
+    res.each_file do |key, _|
+      name = format_name(key)
+      links << "<a href='file://#{res.file_path(key)}'>#{name}</a><br/>"
+    end
+    links.empty? ? nil : links.join('')
+  end
+
+  def format_result_stats(res)
+    res.stats.map do |k, v|
+      v = [v, ''] unless v.is_a? Array
+      v[0] = ('%.3g' % v[0]) if v[0].is_a? Float
+      "<b>#{format_name(k)}:</b> #{v[0]}#{v[1]}<br/>"
+    end.join('') + '<br/>' unless res.stats.empty?
+  end
+
+  ##
+  # Write +file+ within the browse folder of project +p+ using the passed
+  # block output as content
+  def write_file(p, file)
+    File.open(browse_file(p, file), 'w') { |fh| fh.print yield }
+  end
+
+  ##
+  # Use a +template+ file to generate content with a hash of +data+ over the
+  # layout page if +layout+ is true
+  def build_from_template(template, data = {}, layout = true)
+    cont = File.read(template_file(template)).miga_variables(data)
+    return cont unless layout
+
+    build_from_template(
+      'layout.html',
+      data.merge(content: cont, project_name: cli.load_project.name),
+      false
+    )
+  end
+
+  ##
+  # Path to the template browse file
+  def template_file(file)
+    File.join(
+      MiGA::MiGA.root_path,
+      'lib', 'miga', 'cli', 'action', 'browse', file
+    )
+  end
+
+  ##
+  # Path to the browse file in the project
+  def browse_file(p, file)
+    File.join(p.path, 'browse', file)
+  end
+end

data/lib/miga/cli/action/classify_wf.rb

@@ -42,7 +42,9 @@ class MiGA::Cli::Action::ClassifyWf < MiGA::Cli::Action
         '--no-summaries',
         'Do not generate intermediate step summaries'
       ) { |v| cli[:summaries] = v }
-      opts_for_wf(opt, 'Input genome assemblies (nucleotides, FastA)')
+      opts_for_wf(
+        opt, 'Input genome assemblies (nucleotides, FastA)', qual: false
+      )
     end
   end
 

data/lib/miga/cli/action/derep_wf.rb

@@ -19,6 +19,10 @@ class MiGA::Cli::Action::DerepWf < MiGA::Cli::Action
         'Use Average Amino Acid Identity (AAI) as genome similarity metric',
         'By default: Use Average Nucleotide Identity (ANI)'
       ) { cli[:metric] = :aai }
+      opt.on(
+        '--ani',
+        'Use Average Nucleotide Identity (ANI) as similarity metric (default)'
+      ) { cli[:metric] = :ani }
       opt.on(
         '--threshold FLOAT', Float,
         "Metric threshold (%) to dereplicate. By default: #{cli[:threshold]}"

data/lib/miga/cli/action/edit.rb

@@ -17,18 +17,21 @@ class MiGA::Cli::Action::Edit < MiGA::Cli::Action
         'Activate dataset; requires -D'
       ) { |v| cli[:activate] = v }
       opt.on(
-        '--inactivate',
-        'Inactivate dataset; requires -D'
-      ) { |v| cli[:activate] = !v }
+        '--inactivate [reason]',
+        'Inactivate dataset; requires -D',
+        'The argument is optional: reason to inactivate dataset'
+      ) { |v| cli[:activate] = false ; cli[:reason] = v }
     end
   end
 
   def perform
     obj = cli.load_project_or_dataset
     unless cli[:activate].nil?
-      cli.ensure_par({ dataset: '-D' },
-                     '%<name>s is mandatory with --[in-]activate: please provide %<flag>s')
-      cli[:activate] ? obj.activate! : obj.inactivate!
+      cli.ensure_par(
+        { dataset: '-D' },
+        '%<name>s is mandatory with --[in-]activate: please provide %<flag>s'
+      )
+      cli[:activate] ? obj.activate! : obj.inactivate!(cli[:reason])
     end
     cli.add_metadata(obj)
     obj.save

data/lib/miga/cli/action/quality_wf.rb

@@ -15,7 +15,10 @@ class MiGA::Cli::Action::QualityWf < MiGA::Cli::Action
         '-m', '--mytaxa-scan',
         'Perform MyTaxa scan analysis'
       ) { |v| cli[:mytaxa] = v }
-      opts_for_wf(opt, 'Input genome assemblies (nucleotides, FastA)')
+      opts_for_wf(
+        opt, 'Input genome assemblies (nucleotides, FastA)',
+        qual: false
+      )
     end
   end
 

data/lib/miga/cli/action/stats.rb

@@ -14,7 +14,7 @@ class MiGA::Cli::Action::Stats < MiGA::Cli::Action
       ) { |v| cli[:key] = v }
       opt.on(
         '--compute-and-save',
-        'Compute and saves the statistics'
+        'Compute and save the statistics'
       ) { |v| cli[:compute] = v }
       opt.on(
         '--try-load',
@@ -38,7 +38,7 @@ class MiGA::Cli::Action::Stats < MiGA::Cli::Action
     end
     if cli[:key].nil?
       r[:stats].each do |k, v|
-        k_n = k == :g_c_content ? 'G+C content' : k.to_s.unmiga_name.capitalize
+        k_n = k.to_s.unmiga_name.sub(/^./, &:upcase)
         cli.puts "#{k_n}: #{v.is_a?(Array) ? v.join(' ') : v}"
       end
     else

data/lib/miga/cli/action/summary.rb

@@ -26,6 +26,10 @@ class MiGA::Cli::Action::Summary < MiGA::Cli::Action
         '--with-units',
         'Include units in each cell'
       ) { |v| cli[:units] = v }
+      opt.on(
+        '--compute-and-save',
+        'Compute and save the statistics if not yet available'
+      ) { |v| cli[:compute] = v }
     end
   end
 
@@ -34,7 +38,8 @@ class MiGA::Cli::Action::Summary < MiGA::Cli::Action
     ds = cli.load_and_filter_datasets
     cli.say 'Loading results'
     stats = ds.map do |d|
-      r = d.add_result(cli[:result].to_sym, false)
+      r = d.result(cli[:result])
+      r.compute_stats if cli[:compute] && !r.nil? && r[:stats].empty?
       s = r.nil? ? {} : r[:stats]
       s.tap { |i| i[:dataset] = d.name }
     end

data/lib/miga/cli/action/wf.rb

@@ -15,7 +15,7 @@ module MiGA::Cli::Action::Wf
 
   def opts_for_wf(opt, files_desc, params = {})
     {
-      multi: false, cleanup: true, project_type: false, ncbi: true
+      multi: false, cleanup: true, project_type: false, ncbi: true, qual: true
     }.each { |k, v| params[k] = v if params[k].nil? }
     opt.on(
       '-o', '--out_dir PATH',
@@ -40,6 +40,13 @@ module MiGA::Cli::Action::Wf
         'Only download complete genomes, not drafts'
       ) { |v| cli[:ncbi_draft] = v }
     end
+    if params[:qual]
+      opt.on(
+        '--min-qual FLOAT', Float,
+        'Minimum genome quality to include in analysis',
+        'By default: 50.0'
+      ) { |v| cli[:min_qual] = v }
+    end
     if params[:cleanup]
       opt.on(
         '-c', '--clean',
@@ -125,7 +132,7 @@ module MiGA::Cli::Action::Wf
              ]) unless MiGA::Project.exist? cli[:outdir]
     # Define project metadata
     p = cli.load_project(:outdir, '-o')
-    [:haai_p, :aai_p, :ani_p, :ess_coll].each { |i| p_metadata[i] = cli[i] }
+    %i[haai_p aai_p ani_p ess_coll min_qual].each { |i| p_metadata[i] = cli[i] }
     p_metadata[:type] = cli[:project_type]
     transfer_metadata(p, p_metadata)
     # Download datasets
@@ -159,9 +166,10 @@ module MiGA::Cli::Action::Wf
                  '-P', cli[:outdir],
                  '-r', r,
                  '-o', File.expand_path("#{r}.tsv", cli[:outdir]),
-                 '--tab'
+                 '--tab', '--ref', '--active'
                ])
     end
+    call_cli(['browse', '-P', cli[:outdir]])
   end
 
   def cleanup

data/lib/miga/cli/base.rb

@@ -11,39 +11,40 @@ module MiGA::Cli::Base
     preproc_wf: 'Preprocess input genomes or metagenomes',
     index_wf: 'Generate distance indexing of input genomes',
     # Projects
-    new: 'Creates an empty MiGA project',
-    about: 'Displays information about a MiGA project',
-    doctor: 'Performs consistency checks on a MiGA project',
-    get_db: 'Downloads a pre-indexed database',
+    new: 'Create an empty MiGA project',
+    about: 'Display information about a MiGA project',
+    doctor: 'Perform consistency checks on a MiGA project',
+    get_db: 'Download a pre-indexed database',
+    browse: 'Explore a project locally using a web browser',
     # Datasets
-    add: 'Creates a dataset in a MiGA project',
-    get: 'Downloads a dataset from public databases into a MiGA project',
-    ncbi_get: 'Downloads all genomes in a taxon from NCBI into a MiGA project',
-    rm: 'Removes a dataset from an MiGA project',
-    find: 'Finds unregistered datasets based on result files',
+    add: 'Create a dataset in a MiGA project',
+    get: 'Download a dataset from public databases into a MiGA project',
+    ncbi_get: 'Download all genomes in a taxon from NCBI into a MiGA project',
+    rm: 'Remove a dataset from an MiGA project',
+    find: 'Find unregistered datasets based on result files',
     ln: 'Link datasets (including results) from one project to another',
-    ls: 'Lists all registered datasets in an MiGA project',
-    archive: 'Generates a tar-ball with all files from select datasets',
+    ls: 'List all registered datasets in an MiGA project',
+    archive: 'Generate a tar-ball with all files from select datasets',
     # Results
-    add_result: 'Registers a result',
-    stats: 'Extracts statistics for the given result',
-    files: 'Lists registered files from the results of a dataset or project',
-    run: 'Executes locally one step analysis producing the given result',
-    summary: 'Generates a summary table for the statistics of all datasets',
-    next_step: 'Returns the next task to run in a dataset or project',
+    add_result: 'Register a result',
+    stats: 'Extract statistics for the given result',
+    files: 'List registered files from the results of a dataset or project',
+    run: 'Execute locally one step analysis producing the given result',
+    summary: 'Generate a summary table for the statistics of all datasets',
+    next_step: 'Return the next task to run in a dataset or project',
     # Objects (Datasets or Projects)
-    edit: 'Edits the metadata of a dataset or project',
+    edit: 'Edit the metadata of a dataset or project',
     # System
     init: 'Initialize MiGA to process new projects',
-    daemon: 'Controls the daemon of a MiGA project',
-    lair: 'Controls groups of daemons for several MiGA projects',
-    date: 'Returns the current date in standard MiGA format',
-    console: 'Opens an IRB console with MiGA',
+    daemon: 'Control the daemon of a MiGA project',
+    lair: 'Control groups of daemons for several MiGA projects',
+    date: 'Return the current date in standard MiGA format',
+    console: 'Open an IRB console with MiGA',
     # Taxonomy
-    tax_set: 'Registers taxonomic information for datasets',
-    tax_test: 'Returns test of taxonomic distributions for query datasets',
-    tax_index: 'Creates a taxonomy-indexed list of the datasets',
-    tax_dist: 'Estimates distributions of distance by taxonomy',
+    tax_set: 'Register taxonomic information for datasets',
+    tax_test: 'Return test of taxonomic distributions for query datasets',
+    tax_index: 'Create a taxonomy-indexed list of the datasets',
+    tax_dist: 'Estimate distributions of distance by taxonomy',
   }
 
   @@TASK_ALIAS = {

data/lib/miga/common/format.rb

@@ -68,15 +68,20 @@ module MiGA::Common::Format
   # a FastA or FastQ file (supports gzipped files). The +format+ must be a
   # Symbol, one of +:fasta+ or +:fastq+. Additional estimations can be
   # controlled via the +opts+ Hash. Supported options include:
-  # - +:n50+: If true, it also returns the N50 and the median (in bp)
-  # - +:gc+: If true, it also returns the G+C content (in %)
-  # - +:x+: If true, it also returns the undetermined bases content (in %)
+  # - +:n50+: Include the N50 and the median (in bp)
+  # - +:gc+: Include the G+C content (in %)
+  # - +:x+: Include the undetermined bases content (in %)
+  # - +:skew+: Include G-C and A-T sequence skew (in %; forces gc: true).
+  #   See definition used here in DOI:10.1177/117693430700300006
   def seqs_length(file, format, opts = {})
+    opts[:gc] = true if opts[:skew]
     fh = file =~ /\.gz/ ? Zlib::GzipReader.open(file) : File.open(file, 'r')
     l = []
     gc = 0
     xn = 0
-    i = 0 # <- Zlib::GzipReader doesn't set `$.`
+    t  = 0
+    c  = 0
+    i  = 0 # <- Zlib::GzipReader doesn't set `$.`
     fh.each_line do |ln|
       i += 1
       if (format == :fasta and ln =~ /^>/) or
@@ -86,16 +91,27 @@ module MiGA::Common::Format
         l[l.size - 1] += ln.chomp.size
         gc += ln.scan(/[GCgc]/).count if opts[:gc]
         xn += ln.scan(/[XNxn]/).count if opts[:x]
+        if opts[:skew]
+          t += ln.scan(/[Tt]/).count
+          c += ln.scan(/[Cc]/).count
+        end
       end
     end
     fh.close
 
-    o = { n: l.size, tot: l.inject(:+), max: l.max }
+    o = { n: l.size, tot: l.inject(0, :+), max: l.max }
+    return o if o[:tot].zero?
     o[:avg] = o[:tot].to_f / l.size
     o[:var] = l.map { |a| a**2 }.inject(:+).to_f / l.size - o[:avg]**2
     o[:sd]  = Math.sqrt o[:var]
     o[:gc]  = 100.0 * gc / o[:tot] if opts[:gc]
     o[:x]   = 100.0 * xn / o[:tot] if opts[:x]
+    if opts[:skew]
+      at = o[:tot] - gc
+      o[:at_skew] = 100.0 * (2 * t - at) / at
+      o[:gc_skew] = 100.0 * (2 * c - gc) / gc
+    end
+
     if opts[:n50]
       l.sort!
       thr = o[:tot] / 2
@@ -106,7 +122,8 @@ module MiGA::Common::Format
         break if pos >= thr
       end
       o[:med] = o[:n].even? ?
-            0.5 * l[o[:n] / 2 - 1, 2].inject(:+) : l[(o[:n] - 1) / 2]
+            0.5 * l[o[:n] / 2 - 1, 2].inject(:+) :
+            l[(o[:n] - 1) / 2]
     end
     o
   end
@@ -130,9 +147,14 @@ class String
   end
 
   ##
-  # Replace underscores by spaces or dots (depending on context).
+  # Replace underscores by spaces or other symbols depending on context
   def unmiga_name
-    gsub(/_(str|sp|subsp|pv)__/, '_\\1._').tr('_', ' ')
+    gsub(/_(str|sp|subsp|pv)__/, '_\\1._')
+      .gsub(/g_c_(content)/, 'G+C \\1')
+      .gsub(/g_c_(skew)/, 'G-C \\1')
+      .gsub(/a_t_(skew)/, 'A-T \\1')
+      .gsub(/x_content/, &:capitalize)
+      .tr('_', ' ')
   end
 
   ##

data/lib/miga/daemon.rb

@@ -72,6 +72,7 @@ class MiGA::Daemon < MiGA::MiGA
     say '-----------------------------------'
     say 'MiGA:%s launched' % project.name
     say '-----------------------------------'
+    recalculate_status!
     load_status
     say 'Configuration options:'
     say @runopts.to_s
@@ -99,6 +100,7 @@ class MiGA::Daemon < MiGA::MiGA
   end
 
   def recalculate_status!
+    say 'Recalculating status for all datasets'
     project.each_dataset(&:recalculate_status)
   end
 
@@ -158,8 +160,8 @@ class MiGA::Daemon < MiGA::MiGA
   end
 
   ##
-  # Traverse datasets, and returns boolean indicating if at any datasets
-  # are incomplete
+  # Traverse datasets, and returns boolean indicating if at any reference
+  # datasets are incomplete
   def check_datasets
     l_say(2, 'Checking datasets')
     o = false
@@ -167,7 +169,7 @@ class MiGA::Daemon < MiGA::MiGA
       next unless ds.status == :incomplete
       next if ds.next_preprocessing(false).nil?
 
-      o = true
+      o = true if ds.ref?
       queue_job(:d, ds)
     end
     o
@@ -183,7 +185,7 @@ class MiGA::Daemon < MiGA::MiGA
     return if project.dataset_names.empty?
 
     # Double-check if all datasets are ready
-    return unless project.done_preprocessing?(false)
+    return unless project.done_preprocessing?
 
     # Queue project-level job
     to_run = project.next_task(nil, false)

data/lib/miga/dataset.rb

@@ -97,7 +97,10 @@ class MiGA::Dataset < MiGA::MiGA
 
   ##
   # Inactivate a dataset. This halts automated processing by the daemon
-  def inactivate!
+  # 
+  # If given, the +reason+ string is saved as a metadata +:warn+ entry
+  def inactivate!(reason = nil)
+    metadata[:warn] = "Inactive: #{reason}" unless reason.nil?
     metadata[:inactive] = true
     metadata.save
     pull_hook :on_inactivate
@@ -107,6 +110,7 @@ class MiGA::Dataset < MiGA::MiGA
   # Activate a dataset. This removes the +:inactive+ flag
   def activate!
     metadata[:inactive] = nil
+    metadata[:warn] = nil if metadata[:warn] && metadata[:warn] =~ /^Inactive: /
     metadata.save
     pull_hook :on_activate
   end

data/lib/miga/dataset/base.rb

@@ -35,8 +35,8 @@ module MiGA::Dataset::Base
     mytaxa: '07.annotation/02.taxonomy/01.mytaxa',
     mytaxa_scan: '07.annotation/03.qa/02.mytaxa_scan',
     # Distances (for single-species datasets)
-    distances: '09.distances',
     taxonomy: '09.distances/05.taxonomy',
+    distances: '09.distances',
     # General statistics
     stats: '90.stats'
   }
@@ -66,7 +66,7 @@ module MiGA::Dataset::Base
   @@PREPROCESSING_TASKS = [
     :raw_reads, :trimmed_reads, :read_quality, :trimmed_fasta,
     :assembly, :cds, :essential_genes, :ssu, :mytaxa, :mytaxa_scan,
-    :distances, :taxonomy, :stats
+    :taxonomy, :distances, :stats
   ]
 
   ##
@@ -77,7 +77,7 @@ module MiGA::Dataset::Base
   ##
   # Tasks to be executed only in datasets that are not multi-organism. These
   # tasks are ignored for multi-organism datasets or for unknown types.
-  @@ONLY_NONMULTI_TASKS = [:mytaxa_scan, :distances, :taxonomy]
+  @@ONLY_NONMULTI_TASKS = [:mytaxa_scan, :taxonomy, :distances]
   @@_ONLY_NONMULTI_TASKS_H = Hash[@@ONLY_NONMULTI_TASKS.map { |i| [i, true] }]
 
   ##

data/lib/miga/dataset/hooks.rb

@@ -52,15 +52,15 @@ module MiGA::Dataset::Hooks
   end
 
   ##
-  # Run +cmd+ in the command-line with {{variables}}: dataset, project, miga,
-  # object (as defined for the event, if any)
+  # Run +cmd+ in the command-line with {{variables}}:
+  # dataset, project, project_name, miga, object (if defined for the event)
   # - +hook_args+: +[cmd]+
   # - +event_args+: +[object (optional)]+
   def hook_run_cmd(hook_args, event_args)
     Process.wait(
       spawn hook_args.first.miga_variables(
-        dataset: name, project: project.path, miga: MiGA::MiGA.root_path,
-        object: event_args.first
+        dataset: name, project: project.path, project_name: project.name,
+        miga: MiGA::MiGA.root_path, object: event_args.first
       )
     )
   end

data/lib/miga/dataset/result.rb

@@ -26,15 +26,24 @@ module MiGA::Dataset::Result
   # The values are symbols:
   # - empty: the dataset has no data
   # - inactive: the dataset is inactive
+  # - upstream: the task is upstream from dataset's input
   # - force: forced to ignore by metadata
   # - project: incompatible project
   # - noref: incompatible dataset, only for reference
   # - multi: incompatible dataset, only for multi
   # - nonmulti: incompatible dataset, only for nonmulti
+  # - complete: the task is already complete
   # - execute: do not ignore, execute the task
   def why_ignore(task)
-    if !active?
+    if !get_result(task).nil?
+      :complete
+    elsif !active?
       :inactive
+    elsif first_preprocessing.nil?
+      :empty
+    elsif @@PREPROCESSING_TASKS.index(task) <
+          @@PREPROCESSING_TASKS.index(first_preprocessing)
+      :upstream
     elsif !metadata["run_#{task}"].nil?
       metadata["run_#{task}"] ? :execute : :force
     elsif task == :taxonomy && project.metadata[:ref_project].nil?
@@ -56,7 +65,7 @@ module MiGA::Dataset::Result
   # initial input. Passes +save+ to #add_result.
   def first_preprocessing(save = false)
     @first_processing ||= @@PREPROCESSING_TASKS.find do |t|
-      !ignore_task?(t) && !add_result(t, save).nil?
+      !add_result(t, save).nil?
     end
   end
 
@@ -70,7 +79,7 @@ module MiGA::Dataset::Result
         false
       elsif add_result(t, save).nil?
         if (metadata["_try_#{t}"] || 0) > (project.metadata[:max_try] || 10)
-          inactivate!
+          inactivate! "Too many errors in step #{t}"
           false
         else
           true
@@ -121,17 +130,12 @@ module MiGA::Dataset::Result
   # - complete: a task with registered results
   # - pending: a task queued to be performed
   def result_status(task)
-    if first_preprocessing.nil?
-      :ignore_empty
-    elsif !get_result(task).nil?
-      :complete
-    elsif @@PREPROCESSING_TASKS.index(task) <
-          @@PREPROCESSING_TASKS.index(first_preprocessing)
-      :-
-    elsif ignore_task?(task)
-      :"ignore_#{why_ignore task}"
-    else
-      :pending
+    reason = why_ignore(task)
+    case reason
+    when :upstream; :-
+    when :execute; :pending
+    when :complete; :complete
+    else; :"ignore_#{reason}"
     end
   end
 

data/lib/miga/lair.rb

@@ -111,7 +111,7 @@ class MiGA::Lair < MiGA::MiGA
 
         yield(project)
       elsif Dir.exist? f
-        each_project(f) { |project| yield(project) }
+        each_project(f) { |p| yield(p) }
       end
     end
   end

data/lib/miga/project/dataset.rb

@@ -134,12 +134,10 @@ module MiGA::Project::Dataset
   ##
   # Are all the datasets in the project preprocessed? Save intermediate results
   # if +save+ (until the first incomplete dataset is reached).
-  def done_preprocessing?(save = true)
-    dataset_names.each do |dn|
-      ds = dataset(dn)
-      return false if ds.is_ref? and not ds.done_preprocessing?(save)
+  def done_preprocessing?(save = false)
+    !each_dataset.any? do |d|
+      d.ref? && d.active? && !d.done_preprocessing?(save)
     end
-    true
   end
 
   ##

data/lib/miga/project/hooks.rb

@@ -26,14 +26,15 @@ module MiGA::Project::Hooks
   end
 
   ##
-  # Run +cmd+ in the command-line with {{variables}}: project, miga,
-  # object (as defined by the event, if any)
+  # Run +cmd+ in the command-line with {{variables}}:
+  # project, project_name, miga, object (if defined by the event)
   # - +hook_args+: +[cmd]+
   # - +event_args+: +[object (optional)]+
   def hook_run_cmd(hook_args, event_args)
     Process.wait(
       spawn hook_args.first.miga_variables(
-        project: path, miga: MiGA::MiGA.root_path, object: event_args.first
+        project: path, project_name: name,
+        miga: MiGA::MiGA.root_path, object: event_args.first
       )
     )
   end

data/lib/miga/remote_dataset/download.rb

@@ -94,12 +94,13 @@ class MiGA::RemoteDataset
       @timeout_try = 0
       begin
         DEBUG 'GET: ' + url
-        open(url, read_timeout: 600) { |f| doc = f.read }
+        URI.parse(url).open(read_timeout: 600) { |f| doc = f.read }
       rescue => e
         @timeout_try += 1
         raise e if @timeout_try >= 3
 
         sleep 5 # <- For: 429 Too Many Requests
+        DEBUG "RETRYING after: #{e}"
         retry
       end
       doc

data/lib/miga/result.rb

@@ -164,7 +164,9 @@ class MiGA::Result < MiGA::MiGA
   # Unlink result by removing the .done and .start timestamps and the
   # .json descriptor, but don't remove any other associated files
   def unlink
-    %i(start done).each { |i| f = path(i) and File.unlink(f) }
+    %i(start done).each do |i|
+      f = path(i) and File.exists?(f) and File.unlink(f)
+    end
     File.unlink path
   end
 

data/lib/miga/result/stats.rb

@@ -8,6 +8,7 @@ module MiGA::Result::Stats
   # (Re-)calculate and save the statistics for the result
   def compute_stats
     method = :"compute_stats_#{key}"
+    MiGA::MiGA.DEBUG "Result(#{key}).compute_stats"
     stats = self.respond_to?(method, true) ? send(method) : nil
     unless stats.nil?
       self[:stats] = stats
@@ -16,32 +17,45 @@ module MiGA::Result::Stats
     self[:stats]
   end
 
+  ##
+  # Access the stats entry of results
+  def stats
+    self[:stats]
+  end
+
   private
 
   def compute_stats_raw_reads
     stats = {}
+    seq_opts = { gc: true, x: true, skew: true }
     if self[:files][:pair1].nil?
-      s = MiGA::MiGA.seqs_length(file_path(:single), :fastq, gc: true, x: true)
+      s = MiGA::MiGA.seqs_length(file_path(:single), :fastq, seq_opts)
       stats = {
         reads: s[:n],
         length_average: [s[:avg], 'bp'],
         length_standard_deviation: [s[:sd], 'bp'],
         g_c_content: [s[:gc], '%'],
-        x_content: [s[:x], '%']
+        x_content: [s[:x], '%'],
+        g_c_skew: [s[:gc_skew], '%'],
+        a_t_skew: [s[:at_skew], '%']
       }
     else
-      s1 = MiGA::MiGA.seqs_length(file_path(:pair1), :fastq, gc: true, x: true)
-      s2 = MiGA::MiGA.seqs_length(file_path(:pair2), :fastq, gc: true, x: true)
+      s1 = MiGA::MiGA.seqs_length(file_path(:pair1), :fastq, seq_opts)
+      s2 = MiGA::MiGA.seqs_length(file_path(:pair2), :fastq, seq_opts)
       stats = {
         read_pairs: s1[:n],
         forward_length_average: [s1[:avg], 'bp'],
         forward_length_standard_deviation: [s1[:sd], 'bp'],
         forward_g_c_content: [s1[:gc], '%'],
         forward_x_content: [s1[:x], '%'],
+        forward_g_c_skew: [s1[:gc_skew], '%'],
+        forward_a_t_skew: [s1[:at_skew], '%'],
         reverse_length_average: [s2[:avg], 'bp'],
         reverse_length_standard_deviation: [s2[:sd], 'bp'],
         reverse_g_c_content: [s2[:gc], '%'],
-        reverse_x_content: [s2[:x], '%']
+        reverse_x_content: [s2[:x], '%'],
+        reverse_g_c_skew: [s2[:gc_skew], '%'],
+        reverse_a_t_skew: [s2[:at_skew], '%']
       }
     end
     stats
@@ -49,19 +63,22 @@ module MiGA::Result::Stats
 
   def compute_stats_trimmed_fasta
     f = self[:files][:coupled].nil? ? file_path(:single) : file_path(:coupled)
-    s = MiGA::MiGA.seqs_length(f, :fasta, gc: true, x: true)
+    s = MiGA::MiGA.seqs_length(f, :fasta, gc: true, x: true, skew: true)
     {
       reads: s[:n],
       length_average: [s[:avg], 'bp'],
       length_standard_deviation: [s[:sd], 'bp'],
       g_c_content: [s[:gc], '%'],
-      x_content: [s[:x], '%']
+      x_content: [s[:x], '%'],
+      g_c_skew: [s[:gc_skew], '%'],
+      a_t_skew: [s[:at_skew], '%']
     }
   end
 
   def compute_stats_assembly
     s = MiGA::MiGA.seqs_length(
-      file_path(:largecontigs), :fasta, n50: true, gc: true, x: true
+      file_path(:largecontigs), :fasta,
+      n50: true, gc: true, x: true, skew: true
     )
     {
       contigs: s[:n],
@@ -69,7 +86,9 @@ module MiGA::Result::Stats
       total_length: [s[:tot], 'bp'],
       longest_sequence: [s[:max], 'bp'],
       g_c_content: [s[:gc], '%'],
-      x_content: [s[:x], '%']
+      x_content: [s[:x], '%'],
+      g_c_skew: [s[:gc_skew], '%'],
+      a_t_skew: [s[:at_skew], '%']
     }
   end
 
@@ -109,20 +128,8 @@ module MiGA::Result::Stats
         end
       end
     else
-      # Fix estimate by domain
-      if !(tax = source.metadata[:tax]).nil? &&
-         %w[Archaea Bacteria].include?(tax[:d]) &&
-         file_path(:raw_report).nil?
-        scr = "#{MiGA::MiGA.root_path}/utils/domain-ess-genes.rb"
-        rep = file_path(:report)
-        rc_p = File.expand_path('.miga_rc', ENV['HOME'])
-        rc = File.exist?(rc_p) ? ". '#{rc_p}' && " : ''
-        $stderr.print `#{rc} ruby '#{scr}' \
-          '#{rep}' '#{rep}.domain' '#{tax[:d][0]}'`
-        add_file(:raw_report, "#{source.name}.ess/log")
-        add_file(:report, "#{source.name}.ess/log.domain")
-      end
-      # Extract/compute quality values
+      # Estimate quality metrics
+      fix_essential_genes_by_domain
       stats = { completeness: [0.0, '%'], contamination: [0.0, '%'] }
       File.open(file_path(:report), 'r') do |fh|
         fh.each_line do |ln|
@@ -131,6 +138,8 @@ module MiGA::Result::Stats
           end
         end
       end
+
+      # Determine qualitative range
       stats[:quality] = stats[:completeness][0] - stats[:contamination][0] * 5
       source.metadata[:quality] =
         case stats[:quality]
@@ -140,6 +149,12 @@ module MiGA::Result::Stats
         else; :low
         end
       source.save
+
+      # Inactivate low-quality datasets
+      min_qual = (project.metadata[:min_qual] || 25)
+      if min_qual != 'no' && stats[:quality] < min_qual
+        source.inactivate! 'Low quality genome'
+      end
     end
     stats
   end
@@ -175,4 +190,21 @@ module MiGA::Result::Stats
     end
     stats
   end
+
+  # Fix estimates based on essential genes based on taxonomy
+  def fix_essential_genes_by_domain
+    return if (tax = source.metadata[:tax]).nil? ||
+       !%w[Archaea Bacteria].include?(tax[:d]) ||
+       file_path(:raw_report)
+
+    MiGA::MiGA.DEBUG "Fixing essential genes by domain"
+    scr = "#{MiGA::MiGA.root_path}/utils/domain-ess-genes.rb"
+    rep = file_path(:report)
+    rc_p = File.expand_path('.miga_rc', ENV['HOME'])
+    rc = File.exist?(rc_p) ? ". '#{rc_p}' && " : ''
+    $stderr.print `#{rc} ruby '#{scr}' \
+      '#{rep}' '#{rep}.domain' '#{tax[:d][0]}'`
+    add_file(:raw_report, "#{source.name}.ess/log")
+    add_file(:report, "#{source.name}.ess/log.domain")
+  end
 end

data/lib/miga/version.rb

@@ -8,7 +8,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.7, 8, 0]
+  VERSION = [0.7, 12, 0]
 
   ##
   # Nickname for the current major.minor version.
@@ -16,7 +16,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2020, 6, 7)
+  VERSION_DATE = Date.new(2020, 7, 22)
 
   ##
   # Reference of MiGA.

data/scripts/cds.bash

@@ -20,7 +20,6 @@ fi
 TYPE=$(miga ls -P "$PROJECT" -D "$DATASET" -m type | cut -f 2)
 case "$TYPE" in
   metagenome|virome)
-    $CMD -p meta
     prodigal -a "${DATASET}.faa" -d "${DATASET}.fna" -o "${DATASET}.gff3" \
       -f gff -q -i "../05.assembly/${DATASET}.LargeContigs.fna" -p meta
     ;;

data/scripts/distances.bash

@@ -9,7 +9,12 @@ cd "$PROJECT/data/09.distances"
 # Initialize
 miga date > "$DATASET.start"
 
-# Run
+# Check quality first
+miga stats -P "$PROJECT" -D "$DATASET" -r essential_genes --compute-and-save
+inactive=$(miga ls -P "$PROJECT" -D "$DATASET" -m inactive | cut -f 2)
+[[ "$inactive" == "true" ]] && exit
+
+# Run distances
 ruby -I "$MIGA/lib" "$MIGA/utils/distances.rb" "$PROJECT" "$DATASET"
 
 # Finalize

data/test/daemon_test.rb

@@ -93,7 +93,7 @@ class DaemonTest < Test::Unit::TestCase
       0 => /-{20}\n/,
       1 => /MiGA:#{p.name} launched/,
       2 => /-{20}\n/,
-      5 => /Probing running jobs\n/
+      6 => /Probing running jobs\n/
     }.each { |k, v| assert_match(v, l[k], "unexpected line: #{k}") }
   ensure
     begin

data/test/dataset_test.rb

@@ -185,11 +185,13 @@ class DatasetTest < Test::Unit::TestCase
     d = dataset
     assert_equal(:incomplete, d.status)
     assert_predicate(d, :active?)
-    d.inactivate!
+    d.inactivate! 'Too annoying'
     assert_equal(:inactive, d.status)
+    assert_equal('Inactive: Too annoying', d.metadata[:warn])
     assert_not_predicate(d, :active?)
     d.activate!
     assert_equal(:incomplete, d.status)
+    assert_nil(d.metadata[:warn])
     assert_predicate(d, :active?)
   end
 

data/test/project_test.rb

@@ -108,7 +108,7 @@ class ProjectTest < Test::Unit::TestCase
     d1 = p1.add_dataset('BAH')
     assert_not_predicate(p1, :done_preprocessing?)
     FileUtils.touch(File.join(p1.path, 'data', '90.stats', "#{d1.name}.done"))
-    assert_predicate(p1, :done_preprocessing?)
+    assert { p1.done_preprocessing? true }
     assert_nil(p1.next_inclade)
     p1.metadata[:type] = :clade
     assert_equal(:subclades, p1.next_inclade)

data/test/remote_dataset_test.rb

@@ -101,7 +101,7 @@ class RemoteDatasetTest < Test::Unit::TestCase
 
   def test_ref_type_status
     declare_remote_access
-    rd = MiGA::RemoteDataset.new('GCA_002849345', :assembly, :ncbi)
+    rd = MiGA::RemoteDataset.new('GCA_003144295.1', :assembly, :ncbi)
     assert { !rd.get_metadata[:is_type] }
     assert { rd.get_metadata[:is_ref_type] }
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.7.8.0
+  version: 0.7.12.0
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-07 00:00:00.000000000 Z
+date: 2020-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
@@ -118,6 +118,7 @@ files:
 - lib/miga/cli/action/add.rb
 - lib/miga/cli/action/add_result.rb
 - lib/miga/cli/action/archive.rb
+- lib/miga/cli/action/browse.rb
 - lib/miga/cli/action/classify_wf.rb
 - lib/miga/cli/action/console.rb
 - lib/miga/cli/action/daemon.rb
@@ -529,7 +530,7 @@ homepage: http://enve-omics.ce.gatech.edu/miga
 licenses:
 - Artistic-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options:
 - lib
 - README.md
@@ -550,8 +551,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: MiGA
 test_files: []