miga-base 1.3.8.1 → 1.3.9.0

data/lib/miga/dataset/hooks.rb

@@ -15,6 +15,7 @@ require 'miga/common/hooks'
 # Supported hooks:
 # - run_lambda(lambda, args...)
 # - recalculate_status()
+# - check_type()
 # - clear_run_counts()
 # - run_cmd(cmd)
 # Internal hooks:
@@ -27,6 +28,7 @@ module MiGA::Dataset::Hooks
   def default_hooks
     {
       on_create: [[:recalculate_status]],
+      on_save: [[:check_type]],
       on_activate: [[:clear_run_counts], [:recalculate_status]],
       on_inactivate: [[:recalculate_status]],
       on_result_ready: [[:_pull_result_hooks]],
@@ -51,6 +53,12 @@ module MiGA::Dataset::Hooks
     recalculate_status
   end
 
+  ##
+  # Ensure that the dataset type exists and is compatible with the project type
+  def hook_check_type(_hook_args, _event_args)
+    check_type
+  end
+
   ##
   # Run +cmd+ in the command-line with {{variables}}:
   # dataset, project, project_name, miga, object (if defined for the event)

data/lib/miga/dataset/result/ignore.rb

@@ -17,10 +17,14 @@ module MiGA::Dataset::Result::Ignore
   # - project: incompatible project
   # - noref: incompatible dataset, only for reference
   # - multi: incompatible dataset, only for multi
+  # - nomarkers: incompatible dataset, only for markers
   # - nonmulti: incompatible dataset, only for nonmulti
   # - complete: the task is already complete
   def ignore_reasons
-    %i[empty inactive upstream force project noref multi nonmulti complete]
+    %i[
+      empty inactive upstream force project
+      noref multi nonmulti nomarkers complete
+    ]
   end
 
   ##
@@ -91,9 +95,15 @@ module MiGA::Dataset::Result::Ignore
     ignore_by_type?(task, :nonmulti)
   end
 
+  ##
+  # Ignore +task+ because it's not a markers dataset
+  def ignore_nomarkers?(task)
+    ignore_by_type?(task, :nomarkers)
+  end
+
   ##
   # Ignore +task+ by +type+ of dataset, one of: +:noref+, +:multi+, or
-  # +:nonmulti+
+  # +:nonmulti+, +:nomarkers+
   def ignore_by_type?(task, type)
     return false if force_task?(task)
 
@@ -105,6 +115,8 @@ module MiGA::Dataset::Result::Ignore
         [:multi?, self.class.ONLY_MULTI_TASKS]
       when :nonmulti
         [:nonmulti?, self.class.ONLY_NONMULTI_TASKS]
+      when :nomarkers
+        [:markers?, self.class.EXCLUDE_NOMARKER_TASKS]
       else
         raise "Unexpected error, unknown type reason: #{type}"
       end

data/lib/miga/dataset/type.rb

@@ -0,0 +1,51 @@
+##
+# Helper module including specific functions for dataset type
+module MiGA::Dataset::Type
+  ##
+  # Get the type of dataset as Symbol
+  def type
+    metadata[:type]
+  end
+
+  ##
+  # Is this dataset known to be multi-organism?
+  def multi?
+    self.class.KNOWN_TYPES.dig(type, :multi)
+  end
+
+  ##
+  # Is this dataset known to be single-organism?
+  def nonmulti?
+    y = self.class.KNOWN_TYPES.dig(type, :multi)
+    y.nil? ? nil : !y
+  end
+
+  ##
+  # Are universal marker genes expected to be found in this dataset?
+  def markers?
+    self.class.KNOWN_TYPES.dig(type, :markers)
+  end
+
+  ##
+  # Check that the dataset type is defined, known, and compatible with the
+  # project type and raise an exception if any of these checks fail
+  #
+  # If the dataset type is +:empty+, it returns +false+ without raising an
+  # exception, and true otherwise (and no tests are failed)
+  def check_type
+    raise MiGA::Error.new('Undefined dataset type') unless type
+    return false if type == :empty
+
+    unless self.class.KNOWN_TYPES[type]
+      raise MiGA::Error.new("Unknown dataset type: #{type}")
+    end
+    unless self.class.KNOWN_TYPES[type][:project_types].include? project.type
+      raise MiGA::Error.new(
+        "Dataset type (#{type}) incompatible with project (#{project.type})"
+      )
+    end
+
+    true
+  end
+
+end

data/lib/miga/dataset.rb

@@ -6,6 +6,7 @@
 require 'miga/metadata'
 require 'miga/dataset/result'
 require 'miga/dataset/status'
+require 'miga/dataset/type'
 require 'miga/dataset/hooks'
 
 # This library is only required by +#closest_relatives+, so it is now
@@ -18,6 +19,7 @@ require 'miga/dataset/hooks'
 class MiGA::Dataset < MiGA::MiGA
   include MiGA::Dataset::Result
   include MiGA::Dataset::Status
+  include MiGA::Dataset::Type
   include MiGA::Dataset::Hooks
 
   # Class-level
@@ -56,6 +58,7 @@ class MiGA::Dataset < MiGA::MiGA
             name.to_s
     @project, @name, @metadata = project, name, nil
     metadata[:ref] = is_ref
+    metadata[:type] ||= :empty
     @metadata_future = [
       File.join(project.path, 'metadata', "#{name}.json"),
       metadata
@@ -89,12 +92,6 @@ class MiGA::Dataset < MiGA::MiGA
   # +Project+ interface
   alias :save! :save
 
-  ##
-  # Get the type of dataset as Symbol
-  def type
-    metadata[:type]
-  end
-
   ##
   # Delete the dataset with all it's contents (including results) and returns
   # nil
@@ -146,22 +143,6 @@ class MiGA::Dataset < MiGA::MiGA
     !metadata[:ref]
   end
 
-  ##
-  # Is this dataset known to be multi-organism?
-  def multi?
-    return false if metadata[:type].nil? || @@KNOWN_TYPES[type].nil?
-
-    @@KNOWN_TYPES[type][:multi]
-  end
-
-  ##
-  # Is this dataset known to be single-organism?
-  def nonmulti?
-    return false if metadata[:type].nil? || @@KNOWN_TYPES[type].nil?
-
-    !@@KNOWN_TYPES[type][:multi]
-  end
-
   ##
   # Is this dataset active?
   def active?

data/lib/miga/json.rb

@@ -69,5 +69,14 @@ class MiGA::Json < MiGA::MiGA
       File.open(path, 'w') { |fh| fh.print y } unless path.nil?
       y
     end
+
+    ##
+    # Generates and returns plain JSON to represent +obj+.
+    # If +path+ is passed, it saves the JSON in that file.
+    def generate_plain(obj, path = nil)
+      y = JSON.generate(obj)
+      File.open(path, 'w') { |fh| fh.print y } unless path.nil?
+      y
+    end
   end
 end

data/lib/miga/project/base.rb

@@ -89,32 +89,36 @@ module MiGA::Project::Base
   @@KNOWN_TYPES = {
     mixed: {
       description: 'Mixed collection of genomes, metagenomes, and viromes',
-      single: true, multi: true
+      single: true, multi: true, markers: true
     },
     genomes: {
       description: 'Collection of genomes',
-      single: true, multi: false
+      single: true, multi: false, markers: true
     },
     clade: {
       description: 'Collection of closely-related genomes (ANI >= 90%)',
-      single: true, multi: false
+      single: true, multi: false, markers: true
     },
     metagenomes: {
       description: 'Collection of metagenomes and/or viromes',
-      single: false, multi: true
+      single: false, multi: true, markers: true
+    },
+    plasmids: {
+      description: 'Collection of plasmids',
+      single: true, multi: false, markers: false
     }
   }
 
   ##
   # Project-wide distance estimations
-  @@DISTANCE_TASKS = [
-    :project_stats, :haai_distances, :aai_distances, :ani_distances,
-    :clade_finding
+  @@DISTANCE_TASKS = %i[
+    project_stats haai_distances aai_distances ani_distances
+    clade_finding
   ]
 
   ##
   # Project-wide tasks for :clade projects
-  @@INCLADE_TASKS = [:subclades, :ogs]
+  @@INCLADE_TASKS = %i[subclades ogs]
 
   ##
   # Options supported by projects
@@ -131,7 +135,9 @@ module MiGA::Project::Base
     },
     haai_p: {
       desc: 'Value of aai.rb -p on hAAI', type: String,
-      default: proc { |project| project.clade? ? 'no' : 'fastaai' },
+      default: proc { |project|
+        project.clade? || !project.markers? ? 'no' : 'fastaai'
+      },
       in: %w[blast+ blast blat diamond fastaai no]
     },
     aai_p: {

data/lib/miga/project.rb

@@ -98,7 +98,7 @@ class MiGA::Project < MiGA::MiGA
   ##
   # Is this a clade project?
   def clade?
-    type == :clade
+    %i[clade plasmids].include? type
   end
 
   ##
@@ -115,6 +115,12 @@ class MiGA::Project < MiGA::MiGA
   # Same as multi? For backward compatibility
   alias is_multi? multi?
 
+  ##
+  # Does the project support the use of universal markers?
+  def markers?
+    @@KNOWN_TYPES[type][:markers]
+  end
+
   ##
   # Is this project active? Currently a dummy function, returns
   # always true.

data/lib/miga/remote_dataset/base.rb

@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'cgi'
 
 class MiGA::RemoteDataset < MiGA::MiGA
@@ -10,13 +9,24 @@ class MiGA::RemoteDataset < MiGA::MiGA
   end
 end
 
+def uri_safe_join(*parts)
+  safe = parts.map { |i| i.is_a?(Array) ? i.join(',') : i.to_s }
+  last = safe.pop
+  safe.map! { |i| i[-1] == '/' ? i : "#{i}/" }
+  safe << last
+  URI::join(*safe)
+end
+
 module MiGA::RemoteDataset::Base
-  @@_EUTILS = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/'
-  @@_EBI_API = 'https://www.ebi.ac.uk/Tools'
-  @@_GTDB_API = 'https://api.gtdb.ecogenomic.org'
-  @@_SEQCODE_API = 'https://disc-genomics.uibk.ac.at/seqcode'
-  @@_NCBI_API_KEY = lambda { |url|
-    ENV['NCBI_API_KEY'].nil? ? url : "#{url}&api_key=#{ENV['NCBI_API_KEY']}"
+  @@_NCBI_DATASETS = 'https://api.ncbi.nlm.nih.gov/datasets/v2alpha/'
+  @@_EUTILS        = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/'
+  @@_EBI_API       = 'https://www.ebi.ac.uk/Tools/'
+  @@_GTDB_API      = 'https://api.gtdb.ecogenomic.org/'
+  @@_SEQCODE_API   = 'https://disc-genomics.uibk.ac.at/seqcode/'
+  @@_EUTILS_BUILD  = lambda { |service, q|
+    q[:api_key] = ENV['NCBI_API_KEY'] if ENV['NCBI_API_KEY']
+    uri_safe_join(@@_EUTILS, "#{service}.fcgi")
+      .tap { |uri| uri.query = URI.encode_www_form(q) }
   }
 
   ##
@@ -25,15 +35,13 @@ module MiGA::RemoteDataset::Base
   # supported keys as Symbol:
   # - +:dbs+ => Hash with keys being the database name and the values a Hash of
   #   properties such as +stage+, +format+, +map_to+, and +getter+.
-  # - +url+ => Pattern of the URL where the data can be obtained, where +%1$s+
-  #   is the name of the database, +%2$s+ is the IDs, and +%3$s+ is format.
-  #   Additional parameters can be passed to certain functions using the +extra+
-  #   option.
+  # - +uri+ => Function producing a parsed URI object, accepting one parameter:
+  #   a Hash of options.
   # - +method+ => Method used to query the URL. Only +:rest+ and +:net+ are
   #   currently supported.
-  # - +api_key+ => A lambda function that takes a URL as input and returns the
-  #   URL to be downloaded with an API Key (if available).
   # - +map_to_universe+ => Universe where results map to. Currently unsupported.
+  # - +scheme+ => Function returning the scheme used as a String (ftp, http,
+  #   https). Mandatory if method is :net.
   @@UNIVERSE = {
     web: {
       dbs: {
@@ -41,13 +49,18 @@ module MiGA::RemoteDataset::Base
         assembly_gz: { stage: :assembly, format: :fasta_gz },
         text: { stage: :metadata, format: :text }
       },
-      url: '%2$s',
+      uri: lambda { |opts| URI.parse(opts[:ids][0]) },
+      scheme: lambda { |opts| opts[:ids][0].split(':', 2)[0] },
       method: :net
     },
     ebi: {
       dbs: { embl: { stage: :assembly, format: :fasta } },
-      url: "#{@@_EBI_API}/dbfetch/dbfetch/%1$s/%2$s/%3$s",
-      method: :rest
+      uri: lambda do |opts|
+        uri_safe_join(
+          @@_EBI_API, 'dbfetch', 'dbfetch', opts[:db], opts[:ids], opts[:format]
+        )
+      end,
+      method: :get
     },
     gtdb: {
       dbs: {
@@ -56,15 +69,18 @@ module MiGA::RemoteDataset::Base
         # The 'taxon' namespace actually returns a list of genomes (+format+)
         taxon: {
           stage: :metadata, format: :genomes, map_to: [:assembly],
-          extra: ['sp_reps_only=false']
+          extra: { sp_reps_only: false }
         },
         # The 'genome' namespace actually returns the taxonomy (+format+)
         genome: { stage: :metadata, format: 'taxon-history' }
       },
-      url: "#{@@_GTDB_API}/%1$s/%2$s/%3$s?%4$s",
-      method: :rest,
+      uri: lambda do |opts|
+        uri_safe_join(@@_GTDB_API, opts[:db], opts[:ids], opts[:format])
+          .tap { |uri| uri.query = URI.encode_www_form(opts[:extra]) }
+      end,
+      method: :get,
       map_to_universe: :ncbi,
-      headers: 'accept: application/json' # < TODO not currently supported
+      headers: lambda { |_opts| { 'Accept' => 'application/json' } }
     },
     seqcode: {
       dbs: {
@@ -74,8 +90,11 @@ module MiGA::RemoteDataset::Base
         # This is the list of type genomes
         :'type-genomes' => { stage: :metadata, format: :json }
       },
-      url: "#{@@_SEQCODE_API}/%1$s.json?%4$s",
-      method: :rest,
+      uri: lambda do |opts|
+        uri_safe_join(@@_SEQCODE_API, "#{opts[:db]}.json")
+          .tap { |uri| uri.query = URI.encode_www_form(opts[:extra]) }
+      end,
+      method: :get,
       map_to_universe: :ncbi
     },
     ncbi: {
@@ -84,9 +103,12 @@ module MiGA::RemoteDataset::Base
         assembly: { stage: :assembly, format: :fasta_gz, getter: :ncbi_asm },
         taxonomy: { stage: :metadata, format: :xml }
       },
-      url: "#{@@_EUTILS}efetch.fcgi?db=%1$s&id=%2$s&rettype=%3$s&retmode=text",
-      method: :rest,
-      api_key: @@_NCBI_API_KEY
+      uri: lambda do |opts|
+        @@_EUTILS_BUILD[:efetch,
+          db: opts[:db], id: opts[:ids], rettype: opts[:format], retmode: :text
+        ]
+      end,
+      method: :get
     },
     ncbi_map: {
       dbs: {
@@ -95,22 +117,81 @@ module MiGA::RemoteDataset::Base
         },
         biosample: { stage: :metadata, map_to: [:assembly], format: :json }
       },
-      url: "#{@@_EUTILS}elink.fcgi?dbfrom=%1$s&id=%2$s&db=%4$s&retmode=%3$s",
-      method: :net,
-      map_to_universe: :ncbi,
-      api_key: @@_NCBI_API_KEY
+      uri: lambda do |opts|
+        @@_EUTILS_BUILD[:elink, {
+          dbfrom: opts[:db], id: opts[:ids], retmode: opts[:format]
+        }.merge(opts[:extra])]
+      end,
+      method: :get,
+      map_to_universe: :ncbi
     },
     ncbi_summary: {
       dbs: { assembly: { stage: :metadata, format: :json } },
-      url: "#{@@_EUTILS}esummary.fcgi?db=%1$s&id=%2$s&retmode=%3$s",
-      method: :rest,
-      api_key: @@_NCBI_API_KEY
+      uri: lambda do |opts|
+        @@_EUTILS_BUILD[:esummary,
+          db: opts[:db], id: opts[:ids], retmode: opts[:format]
+        ]
+      end,
+      method: :get
     },
     ncbi_search: {
-      dbs: { assembly: { stage: :metadata, format: :json } },
-      url: "#{@@_EUTILS}esearch.fcgi?db=%1$s&term=%2$s&retmode=%3$s",
-      method: :rest,
-      api_key: @@_NCBI_API_KEY
+      dbs: {
+        assembly: { stage: :metadata, format: :json },
+        taxonomy: { stage: :metadata, format: :json }
+      },
+      uri: lambda do |opts|
+        @@_EUTILS_BUILD[:esearch,
+          db: opts[:db], term: opts[:ids], retmode: opts[:format]
+        ]
+      end,
+      method: :get
+    },
+    ncbi_datasets_download: {
+      dbs: { genome: { stage: :assembly, format: :zip } },
+      uri: lambda do |opts|
+        q = { include_annotation_type: 'GENOME_FASTA' }
+        uri_safe_join(
+          @@_NCBI_DATASETS, opts[:db], :accession, opts[:ids], :download
+        ).tap { |uri| uri.query = URI.encode_www_form(q) }
+      end,
+      method: :get,
+      headers: lambda do |opts|
+        {}.tap do |h|
+          h['Accept'] = 'application/zip' if opts[:format] == :zip
+          h['api-key'] = ENV['NCBI_API_KEY'] if ENV['NCBI_API_KEY']
+        end
+      end
+    },
+    ncbi_datasets: {
+      dbs: {
+        genome: {
+          stage: :metadata, format: :json, extra: { action: 'dataset_report' }
+        }
+      },
+      uri: lambda do |opts|
+        uri_safe_join(@@_NCBI_DATASETS, opts[:db], opts[:extra][:action])
+      end,
+      payload: lambda do |opts|
+        query = opts[:ids][0]
+        q = {
+          filters: {
+            assembly_version: 'current',
+            exclude_paired_reports: true
+          }.merge(query[:filters] || {}),
+          page_size: query[:page_size] || 1_000,
+          returned_content: 'COMPLETE'
+        }
+        q[:page_token] = query[:page_token] if query[:page_token]
+        q[:taxons] = query[:taxons] if query[:taxons]
+        MiGA::Json.generate_plain(q)
+      end,
+      headers: lambda do |opts|
+        {}.tap do |h|
+          h['api-key'] = ENV['NCBI_API_KEY'] if ENV['NCBI_API_KEY']
+          h['Content-Type'] = 'application/json' if opts[:format] == :json
+        end
+      end,
+      method: :post
     }
   }
 end