miga-base 1.3.8.2 → 1.3.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e4abdd418e396b20fcfea4beaf4d70f3b0022808714478581e2e1f6f8a85478
-  data.tar.gz: 881884c4d5a933b64ac29e93397b62b7d1dc30fd599baaa8b6ac04a3ad3ab051
+  metadata.gz: 7a4aa208ac4dfe7ff6edbedb7aa7b3444c909c77e47b2ef1b93282adc83192d6
+  data.tar.gz: 78bdce8752b3a1a1a281123a99cccdc002d0f95a2a345aa16b1c4ee684220c29
 SHA512:
-  metadata.gz: 317022ffe39818af13d36e3bf1e1adea80b3ab3aafb265b3df6371b6b5e87efdb75199cf8bd22aea37a63e941f824d1686f85bdb7a38eda3f4ba2a76c535b86b
-  data.tar.gz: 0d42dadd982374f0c618d76fc030f5665075e9973ca8e8615c5b090ab1dcc6bf663901a8f7355ce3bfa6874941d9098957d31704b6369474e516dd5ebf8c0b6b
+  metadata.gz: 335e7a8715a6c561b618e21d1e273381f88d384daf16360c01f9b6818ec374f05f6ae824a36628fbe4dcb0c7d6e082f633e8372137edfa035389cb92922363fa
+  data.tar.gz: 76bdf75348e711bdfcca6b973acd3768559acde7ee80233833ae8b8caa595020599756b7913774dc53c038290592d20710b1b67bd5fa8bb93fd334206bf21b06

data/README.md

@@ -12,7 +12,7 @@
 For additional information on MiGA, visit:
 
 * [MiGA Online][miga-online]: The Microbial Genomes Atlas Online
-* [MiGA@XSEDE][miga-at-xsede]: The MiGA@XSEDE Gateway
+* [MiGA Gateway][miga-gatewat]: The MiGA Science Gateway
 * [MiGA users list][mailing-list]:
   Forum to discuss with other users and developers
 * [MiGA manual][manual]: The definitive guide to MiGA
@@ -57,6 +57,6 @@ See [LICENSE](LICENSE).
 [miga-web]: https://github.com/bio-miga/miga-web
 [miga-gui]: https://github.com/bio-miga/miga-gui
 [miga-online]: http://microbial-genomes.org/
-[miga-at-xsede]: https://xsede.microbial-genomes.org/
+[miga-gateway]: https://gateway.microbial-genomes.org/
 [kostas]: http://enve-omics.gatech.edu/
 [rdp]: http://rdp.cme.msu.edu/

data/lib/miga/cli/action/add_result.rb

@@ -5,13 +5,17 @@ require 'miga/cli/action'
 
 class MiGA::Cli::Action::AddResult < MiGA::Cli::Action
   def parse_cli
-    cli.defaults = { force: false }
+    cli.defaults = { force: false, stdin_versions: false }
     cli.parse do |opt|
       cli.opt_object(opt, [:project, :dataset_opt, :result])
       opt.on(
         '-f', '--force',
         'Force re-indexing of the result even if it\'s already registered'
       ) { |v| cli[:force] = v }
+      opt.on(
+        '--stdin-versions',
+        'Read Software versions from STDIN'
+      ) { |v| cli[:stdin_versions] = v }
     end
   end
 
@@ -21,5 +25,22 @@ class MiGA::Cli::Action::AddResult < MiGA::Cli::Action
     cli.say "Registering result: #{cli[:result]}"
     r = obj.add_result(cli[:result], true, force: cli[:force])
     raise 'Cannot add result, incomplete expected files' if r.nil?
+
+    # Add Software version data
+    if cli[:stdin_versions]
+      versions = {}
+      sw = nil
+      $stdin.each do |ln|
+        ln = ln.chomp.strip
+        if ln =~ /^=> (.*)/
+          sw = $1
+          versions[sw] = ''
+        else
+          versions[sw] += ln
+        end
+      end
+      r.add_versions(versions)
+      r.save
+    end
   end
 end

data/lib/miga/cli/action/browse/about.html

@@ -12,10 +12,12 @@ terms of the terms of the
 <p>
 MiGA is the result of a collaboration between the
 <a href='http://enve-omics.gatech.edu/'>Kostas Lab</a>
-(<a href='http://www.gatech.edu/'>Georgia Institute of Technology</a>) and the
+(<a href='http://www.gatech.edu/'>Georgia Institute of Technology</a>), the
 <a href='http://rdp.cme.msu.edu/'>RDP team</a>
 (<a href='http://cme.msu.edu/'>Center for Microbial Ecology</a>,
-<a href='https://msu.edu/'>Michigan State University</a>).
+<a href='https://msu.edu/'>Michigan State University</a>), and the
+<a href="https://disc-genomics.uibk.ac.at/">Rodriguez-R lab</a>
+(<a href="https://uibk.ac.at/">University of Innsbruck</a>).
 The MiGA project is funded by the
 <a href='http://nsf.gov/'>US National Science Foundation</a>
 (Awards <a href='http://nsf.gov/awardsearch/showAward?AWD_ID=1356288'>#1356288</a> &amp;

data/lib/miga/cli/action/download/gtdb.rb

@@ -31,7 +31,7 @@ module MiGA::Cli::Action::Download::Gtdb
 
   def remote_list
     cli.say 'Downloading genome list'
-    extra = ['sp_reps_only=' + cli[:reference].to_s]
+    extra = { sp_reps_only: cli[:reference].to_s }
     json = MiGA::RemoteDataset.download(
       :gtdb, :taxon, cli[:taxon], :genomes, nil, extra
     )

data/lib/miga/cli/action/download/ncbi.rb

@@ -34,11 +34,8 @@ module MiGA::Cli::Action::Download::Ncbi
       'Do not add sequence version to the dataset name',
       'Only affects --complete and --chromosome'
     ) { |v| cli[:add_version] = v }
-    cli.opt_flag(
-      opt, 'legacy-name',
-      'Use dataset names based on chromosome entries instead of assembly',
-      :legacy_name
-    )
+    # For backwards compatibility
+    cli.opt_flag(opt, 'legacy-name', '::HIDE::', :legacy_name)
   end
 
   def sanitize_cli
@@ -52,89 +49,67 @@ module MiGA::Cli::Action::Download::Ncbi
   end
 
   def remote_list
-    doc =
-      if cli[:ncbi_table_file]
-        cli.say 'Reading genome list from file'
-        File.open(cli[:ncbi_table_file], 'r')
-      else
-        cli.say 'Downloading genome list'
-        url = remote_list_url
-        MiGA::RemoteDataset.download_url(url)
-      end
-    ds = parse_csv_as_datasets(doc)
-    doc.close if cli[:ncbi_table_file]
-    ds
+    list  = {}
+    query = remote_list_query
+    loop do
+      # Query the remote collection
+      page = MiGA::Json.parse(
+        MiGA::RemoteDataset.download(:ncbi_datasets, :genome, query, :json),
+        contents: true
+      )
+      break unless page&.any? && page[:reports]&.any?
+
+      # Process reports in this page
+      list.merge!(parse_reports_as_datasets(page[:reports]))
+
+      # Next page
+      break unless page[:next_page_token]
+      query[:page_token] = page[:next_page_token]
+    end
+    list
   end
   
-  def parse_csv_as_datasets(doc)
+  def parse_reports_as_datasets(reports)
     ds = {}
-    CSV.parse(doc, headers: true).each do |r|
-      asm = r['assembly']
+    reports.each do |r|
+      asm = r[:accession]
       next if asm.nil? || asm.empty? || asm == '-'
 
-      rep = remote_row_replicons(r)
-      n = remote_row_name(r, rep, asm)
-
       # Register for download
+      n = remote_report_name(r, asm)
       ds[n] = {
         ids: [asm], db: :assembly, universe: :ncbi,
         md: {
-          type: :genome, ncbi_asm: asm, strain: r['strain']
+          type: :genome, ncbi_asm: asm, strain: r.dig(:organism, :infraspecific_names, :strain)
         }
       }
-      ds[n][:md][:ncbi_nuccore] = rep.join(',') unless rep.nil?
-      unless r['release_date'].nil?
-        ds[n][:md][:release_date] = Time.parse(r['release_date']).to_s
-      end
+      date = r.dig(:assembly_info, :release_date)
+      ds[n][:md][:release_date] = Time.parse(date).to_s if date
+      ds[n][:md][:ncbi_dataset] = r
     end
     ds
   end
 
-  def remote_row_replicons(r)
-    return if r['replicons'].nil?
-
-    r['replicons']
-      .split('; ')
-      .map { |i| i.gsub(/.*:/, '') }
-      .map { |i| i.gsub(%r{/.*}, '') }
-  end
-
-  def remote_row_name(r, rep, asm)
-    return r['#organism'].miga_name if cli[:legacy_name] && cli[:reference]
-
-    if cli[:legacy_name] && ['Complete', ' Chromosome'].include?(r['level'])
-      acc = rep.nil? ? '' : rep.first
-    else
-      acc = asm
-    end
+  def remote_report_name(r, asm)
+    acc = "#{asm}"
     acc.gsub!(/\.\d+\Z/, '') unless cli[:add_version]
-    "#{r['#organism']}_#{acc}".miga_name
+    org = r.dig(:organism, :organism_name)
+    acc = "#{org}_#{acc}" if org
+    acc.miga_name
   end
 
-  def remote_list_url
-    url_base = 'https://www.ncbi.nlm.nih.gov/genomes/solr2txt.cgi?'
-    url_param = {
-      q: '[display()].' \
-        'from(GenomeAssemblies).' \
-        'usingschema(/schema/GenomeAssemblies).' \
-        'matching(tab==["Prokaryotes"] and q=="' \
-          "#{cli[:taxon]&.tr('"', "'")}\"",
-      fields: 'organism|organism,assembly|assembly,replicons|replicons,' \
-        'level|level,release_date|release_date,strain|strain',
-      nolimit: 'on'
-    }
+  def remote_list_query
+    q = { taxons: [cli[:taxon]], filters: {} }
     if cli[:reference]
-      url_param[:q] += ' and refseq_category==["representative"]'
+      q[:filters][:reference_only] = true
     else
-      status = {
-        complete: 'Complete',
-        chromosome: ' Chromosome', # <- The leading space is *VERY* important!
-        scaffold: 'Scaffold',
-        contig: 'Contig'
-      }.map { |k, v| '"' + v + '"' if cli[k] }.compact.join(',')
-      url_param[:q] += ' and level==[' + status + ']'
+      q[:assembly_level] = {
+        contig: 'contig',
+        scaffold: 'scaffold',
+        chromosome: 'chromosome',
+        complete: 'complete_genome'
+      }.map { |k, v| '"' + v + '"' if cli[k] }.compact
     end
-    url_param[:q] += ')'
-    url_base + URI.encode_www_form(url_param)
+    q
   end
 end

data/lib/miga/cli/action/download/seqcode.rb

@@ -29,8 +29,7 @@ module MiGA::Cli::Action::Download::Seqcode
 
     while current_page <= total_pages
       json = MiGA::RemoteDataset.download(
-        :seqcode, :'type-genomes', nil, :json, nil,
-        ["page=#{current_page}"]
+        :seqcode, :'type-genomes', nil, :json, nil, page: current_page
       )
       doc = MiGA::Json.parse(json, contents: true)
       current_page = doc[:current_page] + 1

data/lib/miga/cli/action/ncbi_get.rb

@@ -8,8 +8,7 @@ class MiGA::Cli::Action::NcbiGet < MiGA::Cli::Action
 
   def parse_cli
     cli.defaults = {
-      query: false, unlink: false,
-      reference: false, legacy_name: false,
+      query: false, unlink: false, reference: false,
       complete: false, chromosome: false,
       scaffold: false, contig: false, add_version: true, dry: false,
       get_md: false, only_md: false, save_every: 1
@@ -29,12 +28,6 @@ class MiGA::Cli::Action::NcbiGet < MiGA::Cli::Action
         '--api-key STRING',
         '::HIDE::' # For backwards compatibility
       ) { |v| ENV['NCBI_API_KEY'] = v }
-      opt.on(
-        '--ncbi-table-file STRING',
-        '::HIDE::' # Only meant for debugging
-        # It can take the table returned by NCBI and parse it from a file
-        # instead of downloading it directly
-      ) { |v| cli[:ncbi_table_file] = v }
       opt.on(
         '--ncbi-api-key STRING',
         'NCBI API key'

data/lib/miga/cli/action/wf.rb

@@ -8,7 +8,8 @@ module MiGA::Cli::Action::Wf
     cli.expect_files = true
     cli.defaults = {
       clean: false, project_type: :genomes, dataset_type: :popgenome,
-      ncbi_draft: true, min_qual: MiGA::Project.OPTIONS[:min_qual][:default],
+      ncbi_draft: true, ncbi_ref: false,
+      min_qual: MiGA::Project.OPTIONS[:min_qual][:default],
       prepare_and_exit: false
     }
   end
@@ -39,14 +40,21 @@ module MiGA::Cli::Action::Wf
         '-T', '--ncbi-taxon STRING',
         'Download all the genomes in NCBI classified as this taxon'
       ) { |v| cli[:ncbi_taxon] = v }
+      opt.on(
+        '--no-draft', '::HIDE::' # Deprecated
+      ) { |v| cli[:ncbi_draft] = v }
+      opt.on(
+        '--ncbi-complete',
+        'Only download complete genomes, not drafts (requires -T)'
+      ) { |v| cli[:ncbi_draft] = !v }
+      opt.on(
+        '--ncbi-ref',
+        'Only download RefSeq reference genomes (requires -T)'
+      ) { |v| cli[:ncbi_ref] = v }
       opt.on(
         '-G', '--gtdb-taxon STRING',
         'Download all the genomes in GTDB classified as this taxon'
       ) { |v| cli[:gtdb_taxon] = v }
-      opt.on(
-        '--no-draft',
-        'Only download complete genomes, not drafts (requires -T)'
-      ) { |v| cli[:ncbi_draft] = v }
       opt.on(
         '--gtdb-ref',
         'Only download reference anchor genomes in GTDB (requires -G)'
@@ -170,7 +178,8 @@ module MiGA::Cli::Action::Wf
   def download_datasets
     # Download datasets from NCBI
     unless cli[:ncbi_taxon].nil?
-      what = cli[:ncbi_draft] ? '--all' : '--complete'
+      what = cli[:ncbi_ref] ? '--reference' :
+             cli[:ncbi_draft] ? '--all' : '--complete'
       cmd = ['ncbi_get', '-P', cli[:outdir], '-T', cli[:ncbi_taxon], what]
       cmd += ['--max', cli[:max_download]] if cli[:max_download]
       call_cli(cmd)

data/lib/miga/cli/objects_helper.rb

@@ -60,6 +60,9 @@ module MiGA::Cli::ObjectsHelper
       o &&= (d.ref? == self[:ref]) unless self[:ref].nil?
       o &&= (d.active? == self[:active]) unless self[:active].nil?
       o &&= (self[:multi] ? d.multi? : d.nonmulti?) unless self[:multi].nil?
+      unless self[:markers].nil?
+        o &&= (self[:markers] ? d.markers? : !d.markers?)
+      end
       unless self[:taxonomy].nil?
         o &&= !d.metadata[:tax].nil? && d.metadata[:tax].in?(self[:taxonomy])
       end

data/lib/miga/cli/opt_helper.rb

@@ -43,7 +43,7 @@ module MiGA::Cli::OptHelper
       '-h', '--help',
       'Display this screen'
     ) do
-      puts opt.to_s.gsub(/^.*\s+::HIDE::\s*$/, '')
+      puts opt.to_a.select { |i| i !~ /\s::HIDE::\s/ }
       exit
     end
     opt.separator ''
@@ -120,10 +120,11 @@ module MiGA::Cli::OptHelper
   # as determined by +what+ an Array with any combination of:
   # - :ref To filter by reference (--ref) or query (--no-ref)
   # - :multi To filter by multiple (--multi) or single (--no-multi) species
+  # - :markers To filter by with (--markers) or without markers (--no-markers)
   # - :active To filter by active (--active) or inactive (--no-active)
   # - :taxonomy To filter by taxonomy (--taxonomy)
   # The "k-th" filter (--dataset-k) is always included
-  def opt_filter_datasets(opt, what = %i[ref multi active taxonomy])
+  def opt_filter_datasets(opt, what = %i[ref multi markers active taxonomy])
     what.each do |w|
       case w
       when :ref
@@ -136,6 +137,11 @@ module MiGA::Cli::OptHelper
           '--[no-]multi',
           'Use only multi-species (or only single-species) datasets'
         ) { |v| self[:multi] = v }
+      when :markers
+        opt.on(
+          '--[no-]markers',
+          'Use only datasets with (or without) markers'
+        ) { |v| self[:markers] = v }
       when :active
         opt.on(
           '--[no-]active',

data/lib/miga/common/net.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'net/http'
 require 'net/ftp'
 require 'open-uri'
 require 'fileutils'
@@ -10,6 +11,8 @@ Net::FTP.const_set('FTP_PORT', 21)
 ##
 # General web-access functions shared throughout MiGA.
 module MiGA::Common::Net
+  attr_accessor :remote_connection_uri
+
   ##
   # Returns the URL of the host +name+ (Symbol)
   def known_hosts(name)
@@ -21,7 +24,7 @@ module MiGA::Common::Net
     when :miga_dist
       "ftp://#{main_server}/dist"
     else
-      raise "Unrecognized server name: #{host}"
+      raise "Unrecognized server name: #{name}"
     end
   end
 
@@ -32,49 +35,128 @@ module MiGA::Common::Net
   end
 
   ##
-  # Connect to an FTP +host+ (String) or a known host name (Symbol, see
-  # +.known_hosts+)
+  # Connect to an FTP +host+ (String), a known host name (Symbol, see
+  # +.known_hosts+), or a parsed +URI+ object
+  #
+  # Sets the attribute +remote_connection_uri+ to the parsed +URI+ object
+  # silently
   def remote_connection(host)
     host = known_hosts(host) if host.is_a?(Symbol)
-    uri = URI.parse(host)
-    raise 'Only FTP hosts are currently supported' unless uri.scheme == 'ftp'
-
-    ftp = Net::FTP.new(uri.host)
-    ftp.passive = true
-    ftp.login
-    ftp.chdir(uri.path)
-    ftp
+    uri = host.is_a?(URI) ? host : URI.parse(host)
+    @remote_connection_uri = uri
+
+    case uri.scheme
+    when 'ftp'
+      ftp = Net::FTP.new(uri.host)
+      ftp.passive = true
+      ftp.login
+      ftp.chdir(uri.path) unless host.is_a?(URI)
+      ftp
+    when 'http', 'https'
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.read_timeout = 600
+      http.use_ssl = uri.scheme == 'https'
+      http
+    else
+      raise 'Only FTP, HTTP, and HTTPS are currently supported'
+    end
   end
 
   ##
   # Download a file via FTP using the +connection+ (returned by
-  # +.remote_connection+) with remote name +file+ into local +target+.
+  # +.remote_connection+) with remote name +file+ into local +target+. If +file+
+  # is +nil+, it tries to guess the file from +connection+. If +target+ is
+  # +nil+, it returns the read data instead
   #
-  # Alternatively, +connection+ can simply be the host (String) or a recognized
-  # Symbol (see +.remote_connection+), in which case the function opens the
-  # connection automatically
+  # Alternatively, +connection+ can simply be the host (String), a recognized
+  # Symbol (see +.remote_connection+), or a parsed +URI+ object, in which case
+  # the function opens the connection automatically
   #
   # Reports progress to the function block with two arguments: the
   # currently transferred size and the total file size
-  def download_file_ftp(connection, file, target)
+  def download_file_ftp(connection, file = nil, target = nil)
     # Open connection unless passed
     close_conn = false
-    if connection.is_a?(String) || connection.is_a?(Symbol)
+    if connection.is_a?(String) || connection.is_a?(Symbol) ||
+          connection.is_a?(URI)
       connection = remote_connection(connection)
+      file ||= remote_connection_uri.path
       close_conn = true
     end
 
     # Prepare download
-    FileUtils.mkdir_p(File.dirname(target))
+    FileUtils.mkdir_p(File.dirname(target)) if target
     filesize = connection.size(file)
     transferred = 0
 
     # Get in chunks of 1KiB
+    ret = ''
     connection.getbinaryfile(file, target, 1024) do |data|
       yield(transferred += data.size, filesize) if block_given?
+      ret += data unless target
     end
 
     # Close connection if automatically opened
     connection.close if close_conn
+    ret unless target
+  end
+
+  ##
+  # Submit an HTTP or HTTPS request using +url+, which should be a URL
+  # either as String or parsed URI. The request follows the +method+, which
+  # should be a Net::HTTP verb such as +:get+, +:post+, or +:patch+. All
+  # additional parameters for the corresponding method should be passed as
+  # +opts+.
+  def http_request(method, url, *opts)
+    doc = nil
+    remote_connection(url).start do |http|
+      res = http.send(method, remote_connection_uri.to_s, *opts)
+      if %w[301 302].include?(res.code)
+        DEBUG "REDIRECTION #{res.code}: #{res['location']}"
+        return http_request(method, res['location'], *opts)
+      end
+      res.value # To force exception unless success
+      doc = res.body
+    end
+    doc
+  end
+
+  def net_method(method, uri, *opts)
+    attempts ||= 0
+    DEBUG "#{method.to_s.upcase}: #{uri} #{opts}"
+    case method.to_sym
+    when :ftp
+      download_file_ftp(uri)
+    else
+      http_request(method, uri, *opts)
+    end
+  rescue => e
+    raise e if (attempts += 1) >= 3
+
+    sleep 5 # <- For: 429 Too Many Requests
+    DEBUG "RETRYING after: #{e}"
+    retry
+  end
+
+  alias :https_request :http_request
+
+  ##
+  # Normalize the encoding of +body+ to UTF-8 by attempting several
+  # common recodings. Code from https://github.com/seq-code/registry
+  def normalize_encoding(body)
+    # Test encodings
+    body.force_encoding('utf-8')
+    %w[iso8859-1 windows-1252 us-ascii ascii-8bit].each do |enc|
+      break if body.valid_encoding?
+      recode = body.force_encoding(enc).encode('utf-8')
+      body = recode if recode.valid_encoding?
+    end
+    # If nothing works, replace offending characters with '?'
+    unless body.valid_encoding?
+      body = body.encode(
+        'utf-8', invalid: :replace, undef: :replace, replace: '?'
+      )
+    end
+    body
   end
 end

data/lib/miga/dataset/base.rb

@@ -32,6 +32,12 @@ class MiGA::Dataset < MiGA::MiGA
       @@EXCLUDE_NOREF_TASKS
     end
 
+    ##
+    # Tasks to be excluded from datasets without markers
+    def EXCLUDE_NOMARKER_TASKS
+      @@EXCLUDE_NOMARKER_TASKS
+    end
+
     ##
     # Tasks to be executed only in datasets that are single-organism. These
     # tasks are ignored for multi-organism datasets or for unknown types
@@ -81,45 +87,67 @@ module MiGA::Dataset::Base
   # Supported dataset types
   @@KNOWN_TYPES = {
     genome: {
-      description: 'The genome from an isolate', multi: false
+      description: 'The genome from an isolate',
+      multi: false, markers: true,
+      project_types: %i[mixed genomes clade]
     },
     scgenome: {
-      description: 'A Single-cell Amplified Genome (SAG)', multi: false
+      description: 'A Single-cell Amplified Genome (SAG)',
+      multi: false, markers: true,
+      project_types: %i[mixed genomes clade]
     },
     popgenome: {
-      description: 'A Metagenome-Assembled Genome (MAG)', multi: false
+      description: 'A Metagenome-Assembled Genome (MAG)',
+      multi: false, markers: true,
+      project_types: %i[mixed genomes clade]
     },
     metagenome: {
-      description: 'A metagenome (excluding viromes)', multi: true
+      description: 'A metagenome (excluding viromes)',
+      multi: true, markers: true,
+      project_types: %i[mixed metagenomes]
     },
     virome: {
-      description: 'A viral metagenome', multi: true
+      description: 'A viral metagenome',
+      multi: true,
+      markers: true, # <- We don't expect, but can be useful for contamination
+      project_types: %i[mixed metagenomes]
+    },
+    plasmid: {
+      description: 'An individual plasmid',
+      multi: false, markers: false,
+      project_types: %i[mixed plasmids]
     }
   }
 
   ##
   # Returns an Array of tasks (Symbols) to be executed before project-wide tasks
-  @@PREPROCESSING_TASKS = [
-    :raw_reads, :trimmed_reads, :read_quality, :trimmed_fasta,
-    :assembly, :cds, :essential_genes, :mytaxa, :mytaxa_scan,
-    :taxonomy, :distances, :ssu, :stats
+  @@PREPROCESSING_TASKS = %i[
+    raw_reads trimmed_reads read_quality trimmed_fasta
+    assembly cds essential_genes mytaxa mytaxa_scan
+    taxonomy distances ssu stats
   ]
 
   ##
   # Tasks to be excluded from query datasets
-  @@EXCLUDE_NOREF_TASKS = [:mytaxa_scan, :taxonomy]
+  @@EXCLUDE_NOREF_TASKS = %i[mytaxa_scan taxonomy]
   @@_EXCLUDE_NOREF_TASKS_H = Hash[@@EXCLUDE_NOREF_TASKS.map { |i| [i, true] }]
 
+  ##
+  # Tasks to be excluded from datasets without markers
+  @@EXCLUDE_NOMARKER_TASKS = %i[essential_genes ssu]
+  @@_EXCLUDE_NOMARKER_TASKS_H =
+    Hash[@@EXCLUDE_NOMARKER_TASKS.map { |i| [i, true] }]
+
   ##
   # Tasks to be executed only in datasets that are single-organism. These
   # tasks are ignored for multi-organism datasets or for unknown types
-  @@ONLY_NONMULTI_TASKS = [:mytaxa_scan, :taxonomy, :distances]
+  @@ONLY_NONMULTI_TASKS = %i[mytaxa_scan taxonomy distances]
   @@_ONLY_NONMULTI_TASKS_H = Hash[@@ONLY_NONMULTI_TASKS.map { |i| [i, true] }]
 
   ##
   # Tasks to be executed only in datasets that are multi-organism. These
   # tasks are ignored for single-organism datasets or for unknwon types
-  @@ONLY_MULTI_TASKS = [:mytaxa]
+  @@ONLY_MULTI_TASKS = %i[mytaxa]
   @@_ONLY_MULTI_TASKS_H = Hash[@@ONLY_MULTI_TASKS.map { |i| [i, true] }]
 
   ##

data/lib/miga/dataset/hooks.rb

@@ -15,6 +15,7 @@ require 'miga/common/hooks'
 # Supported hooks:
 # - run_lambda(lambda, args...)
 # - recalculate_status()
+# - check_type()
 # - clear_run_counts()
 # - run_cmd(cmd)
 # Internal hooks:
@@ -27,6 +28,7 @@ module MiGA::Dataset::Hooks
   def default_hooks
     {
       on_create: [[:recalculate_status]],
+      on_save: [[:check_type]],
       on_activate: [[:clear_run_counts], [:recalculate_status]],
       on_inactivate: [[:recalculate_status]],
       on_result_ready: [[:_pull_result_hooks]],
@@ -51,6 +53,12 @@ module MiGA::Dataset::Hooks
     recalculate_status
   end
 
+  ##
+  # Ensure that the dataset type exists and is compatible with the project type
+  def hook_check_type(_hook_args, _event_args)
+    check_type
+  end
+
   ##
   # Run +cmd+ in the command-line with {{variables}}:
   # dataset, project, project_name, miga, object (if defined for the event)