miga-base 1.2.10.2 → 1.2.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc04109959a53156953d2a20365a0af4555010b119bfcf632558d2a6c22b70e4
-  data.tar.gz: b7aa96e71d322c590289f72d8735153f23741335c657dab8343f68835da21952
+  metadata.gz: bca3141b79964a880f22ba6e4e724b4821466321942a02f6443dcb1e5ba3b6cc
+  data.tar.gz: 45af5251d8bced591ba493ca7b9cac5ae45d2c527f19902773a122f436b93808
 SHA512:
-  metadata.gz: 5c2485fd256fd2cb2861ddf5510213aac302ec5fdb4e8429e752ea2cf2c184ba0ddb44f84f9f4077cef72be345ef0b204dcf019f7b5403c9f03974ba3ccd785d
-  data.tar.gz: 84974ec7a124769a7fffa8bdcd703720e0f9bfb9b842d92ac8996b4cad4dbb3667a806fc164adf8e73414db15b8a083376a6ec89cf3aaf40a8773d7b9aa59f43
+  metadata.gz: 7e648e5f3068348bd0064f8c3f8e3691c9c978aa9c06bb35bcf1b9a3b9a8d8eb0d1d67b5520e52537cf3ef6940414f88d24602cfaa47129fa39face66a95b375
+  data.tar.gz: aab0fb4e5764d76eb78a908e6997761e346d8027323bf305528ccf5abcd258b3b84d6ef356352a270b2e273e8563a28e6e9e0f6993bd7bd648ac28c4a6bce6d7

data/lib/miga/cli/action/download/base.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require 'miga/remote_dataset'
+module MiGA::Cli::Action::Download
+end
+
+##
+# Helper module including download functions for the *_get actions
+module MiGA::Cli::Action::Download::Base
+  def cli_filters(opt)
+    opt.on(
+      '--blacklist PATH',
+      'A file with dataset names to blacklist'
+    ) { |v| cli[:blacklist] = v }
+    cli.opt_flag(opt, 'dry', 'Do not download or save the datasets')
+    opt.on(
+      '--ignore-until STRING',
+      'Ignores all datasets until a name is found (useful for large reruns)'
+    ) { |v| cli[:ignore_until] = v }
+    cli.opt_flag(
+      opt, 'get-metadata',
+      'Only download and update metadata for existing datasets', :get_md
+    )
+  end
+
+  def cli_save_actions(opt)
+    cli.opt_flag(
+      opt, '--only-metadata',
+      'Create datasets without input data but retrieve all metadata',
+      :only_md
+    )
+    opt.on(
+      '--save-every INT', Integer,
+      'Save project every this many downloaded datasets',
+      'If zero, it saves the project only once upon completion',
+      "By default: #{cli[:save_every]}"
+    ) { |v| cli[:save_every] = v }
+    opt.on(
+      '-q', '--query',
+      'Register the datasets as queries, not reference datasets'
+    ) { |v| cli[:query] = v }
+    opt.on(
+      '-u', '--unlink',
+      'Unlink all datasets in the project missing from the download list'
+    ) { |v| cli[:unlink] = v }
+    opt.on(
+      '-R', '--remote-list PATH',
+      'Path to an output file with the list of all datasets listed remotely'
+    ) { |v| cli[:remote_list] = v }
+  end
+
+  def discard_blacklisted(ds)
+    unless cli[:blacklist].nil?
+      cli.say "Discarding datasets in #{cli[:blacklist]}"
+      File.readlines(cli[:blacklist])
+          .select { |i| i !~ /^#/ }
+          .map(&:chomp)
+          .each { |i| ds.delete i }
+    end
+    ds
+  end
+
+  def impose_limit(ds)
+    max = cli[:max_datasets].to_i
+    if !max.zero? && max < ds.size
+      cli.say "Subsampling list from #{ds.size} to #{max} datasets"
+      sample = ds.keys.sample(max)
+      ds.select! { |k, _| sample.include? k }
+    end
+    ds
+  end
+
+  def download_entries(ds, p)
+    cli.say "Downloading #{ds.size} " + (ds.size == 1 ? 'entry' : 'entries')
+    p.do_not_save = true if cli[:save_every] != 1
+    ignore = !cli[:ignore_until].nil?
+    downloaded = 0
+    d = []
+    ds.each do |name, body|
+      d << name
+      cli.puts name
+      ignore = false if ignore && name == cli[:ignore_until]
+      next if ignore || p.dataset(name).nil? == cli[:get_md]
+
+      downloaded += 1
+      unless cli[:dry]
+        save_entry(name, body, p)
+        p.save! if cli[:save_every] > 1 && (downloaded % cli[:save_every]).zero?
+      end
+    end
+    p.do_not_save = false
+    p.save! if cli[:save_every] != 1
+    [d, downloaded]
+  end
+
+  def save_entry(name, body, p)
+    cli.say '  Locating remote dataset'
+    body[:md][:metadata_only] = true if cli[:only_md]
+    rd = MiGA::RemoteDataset.new(body[:ids], body[:db], body[:universe])
+    if cli[:get_md]
+      cli.say '  Updating dataset'
+      rd.update_metadata(p.dataset(name), body[:md])
+    else
+      cli.say '  Creating dataset'
+      rd.save_to(p, name, !cli[:query], body[:md])
+      cli.add_metadata(p.add_dataset(name))
+    end
+  end
+end

data/lib/miga/cli/action/download/gtdb.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'miga/cli/action/download/base'
+
+##
+# Helper module including download functions for the gtdb_get action
+module MiGA::Cli::Action::Download::Gtdb
+  include MiGA::Cli::Action::Download::Base
+
+  def cli_task_flags(opt)
+    cli.opt_flag(
+      opt, 'reference',
+      'Download only reference genomes. By default: download all'
+    )
+  end
+
+  def cli_name_modifiers(opt)
+    opt.on(
+      '--no-version-name',
+      'Do not add sequence version to the dataset name'
+    ) { |v| cli[:add_version] = v }
+  end
+
+  def sanitize_cli
+    cli.ensure_par(taxon: '-T')
+    cli[:save_every] = 1 if cli[:dry]
+  end
+
+  def remote_list
+    cli.say 'Downloading genome list'
+    ds = {}
+    extra = ['sp_reps_only=' + cli[:reference].to_s]
+    json = MiGA::RemoteDataset.download(
+      :gtdb, :taxon, cli[:taxon], :genomes, nil, extra
+    )
+    doc = MiGA::Json.parse(json, contents: true)
+
+    Hash[
+      doc.map do |acc|
+        [
+          remote_row_name(acc),
+          {
+            ids: [acc], db: :assembly, universe: :gtdb,
+            md: { type: :genome, gtdb_assembly: acc }
+          }
+        ]
+      end
+    ]
+  end
+
+  def remote_row_name(asm)
+    acc = "#{asm}"
+    acc.gsub!(/\.\d+\Z/, '') unless cli[:add_version]
+    acc.miga_name
+  end
+end

data/lib/miga/cli/action/{ncbi_get/downloads.rb → download/ncbi.rb}

@@ -1,11 +1,13 @@
 # frozen_string_literal: true
 
-require 'miga/remote_dataset'
+require 'miga/cli/action/download/base'
 require 'csv'
 
 ##
 # Helper module including download functions for the ncbi_get action
-module MiGA::Cli::Action::NcbiGet::Downloads
+module MiGA::Cli::Action::Download::Ncbi
+  include MiGA::Cli::Action::Download::Base
+
   def cli_task_flags(opt)
     cli.opt_flag(
       opt, 'reference',
@@ -39,48 +41,6 @@ module MiGA::Cli::Action::NcbiGet::Downloads
     )
   end
 
-  def cli_filters(opt)
-    opt.on(
-      '--blacklist PATH',
-      'A file with dataset names to blacklist'
-    ) { |v| cli[:blacklist] = v }
-    cli.opt_flag(opt, 'dry', 'Do not download or save the datasets')
-    opt.on(
-      '--ignore-until STRING',
-      'Ignores all datasets until a name is found (useful for large reruns)'
-    ) { |v| cli[:ignore_until] = v }
-    cli.opt_flag(
-      opt, 'get-metadata',
-      'Only download and update metadata for existing datasets', :get_md
-    )
-  end
-
-  def cli_save_actions(opt)
-    cli.opt_flag(
-      opt, 'only-metadata',
-      'Create datasets without input data but retrieve all metadata',
-      :only_md
-    )
-    opt.on(
-      '--save-every INT', Integer,
-      'Save project every this many downloaded datasets',
-      'If zero, it saves the project only once upon completion',
-      "By default: #{cli[:save_every]}"
-    ) { |v| cli[:save_every] = v }
-    opt.on(
-      '-q', '--query',
-      'Register the datasets as queries, not reference datasets'
-    ) { |v| cli[:query] = v }
-    opt.on(
-      '-u', '--unlink',
-      'Unlink all datasets in the project missing from the download list'
-    ) { |v| cli[:unlink] = v }
-    opt.on(
-      '-R', '--remote-list PATH',
-      'Path to an output file with the list of all datasets listed remotely'
-    ) { |v| cli[:remote_list] = v }
-  end
-
   def sanitize_cli
     cli.ensure_par(taxon: '-T')
     tasks = %w[reference complete chromosome scaffold contig]
@@ -169,62 +129,4 @@ module MiGA::Cli::Action::NcbiGet::Downloads
     url_param[:q] += ')'
     url_base + URI.encode_www_form(url_param)
   end
-
-  def discard_blacklisted(ds)
-    unless cli[:blacklist].nil?
-      cli.say "Discarding datasets in #{cli[:blacklist]}"
-      File.readlines(cli[:blacklist])
-          .select { |i| i !~ /^#/ }
-          .map(&:chomp)
-          .each { |i| ds.delete i }
-    end
-    ds
-  end
-
-  def impose_limit(ds)
-    max = cli[:max_datasets].to_i
-    if !max.zero? && max < ds.size
-      cli.say "Subsampling list from #{ds.size} to #{max} datasets"
-      sample = ds.keys.sample(max)
-      ds.select! { |k, _| sample.include? k }
-    end
-    ds
-  end
-
-  def download_entries(ds, p)
-    cli.say "Downloading #{ds.size} " + (ds.size == 1 ? 'entry' : 'entries')
-    p.do_not_save = true if cli[:save_every] != 1
-    ignore = !cli[:ignore_until].nil?
-    downloaded = 0
-    d = []
-    ds.each do |name, body|
-      d << name
-      cli.puts name
-      ignore = false if ignore && name == cli[:ignore_until]
-      next if ignore || p.dataset(name).nil? == cli[:get_md]
-
-      downloaded += 1
-      unless cli[:dry]
-        save_entry(name, body, p)
-        p.save! if cli[:save_every] > 1 && (downloaded % cli[:save_every]).zero?
-      end
-    end
-    p.do_not_save = false
-    p.save! if cli[:save_every] != 1
-    [d, downloaded]
-  end
-
-  def save_entry(name, body, p)
-    cli.say '  Locating remote dataset'
-    body[:md][:metadata_only] = true if cli[:only_md]
-    rd = MiGA::RemoteDataset.new(body[:ids], body[:db], body[:universe])
-    if cli[:get_md]
-      cli.say '  Updating dataset'
-      rd.update_metadata(p.dataset(name), body[:md])
-    else
-      cli.say '  Creating dataset'
-      rd.save_to(p, name, !cli[:query], body[:md])
-      cli.add_metadata(p.add_dataset(name))
-    end
-  end
 end

data/lib/miga/cli/action/get.rb

@@ -17,7 +17,8 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
       ) { |v| cli[:ids] = v }
       opt.on(
         '-U', '--universe STRING',
-        "Universe of the remote database. By default: #{cli[:universe]}"
+        "Universe of the remote database. By default: #{cli[:universe]}",
+        "Supported: #{MiGA::RemoteDataset.UNIVERSE.keys.join(', ')}"
       ) { |v| cli[:universe] = v.to_sym }
       opt.on(
         '--db STRING',

data/lib/miga/cli/action/gtdb_get.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::GtdbGet < MiGA::Cli::Action
+  require 'miga/cli/action/download/gtdb'
+  include MiGA::Cli::Action::Download::Gtdb
+
+  def parse_cli
+    cli.defaults = {
+      query: false, unlink: false,
+      reference: false, add_version: true, dry: false,
+      get_md: false, only_md: false, save_every: 1
+    }
+    cli.parse do |opt|
+      cli.opt_object(opt, [:project])
+      opt.on(
+        '-T', '--taxon STRING',
+        '(Mandatory) Taxon name in GTDB format (e.g., g__Escherichia)'
+      ) { |v| cli[:taxon] = v }
+      opt.on(
+        '--max INT', Integer,
+        'Maximum number of datasets to download (by default: unlimited)'
+      ) { |v| cli[:max_datasets] = v }
+      opt.on(
+        '-m', '--metadata STRING',
+        'Metadata as key-value pairs separated by = and delimited by comma',
+        'Values are saved as strings except for booleans (true / false) or nil'
+      ) { |v| cli[:metadata] = v }
+      cli_task_flags(opt)
+      cli_name_modifiers(opt)
+      cli_filters(opt)
+      cli_save_actions(opt)
+    end
+  end
+
+  def perform
+    sanitize_cli
+    p = cli.load_project
+    ds = remote_list
+    ds = discard_blacklisted(ds)
+    ds = impose_limit(ds)
+    d, downloaded = download_entries(ds, p)
+
+    # Finalize
+    cli.say "Datasets listed: #{d.size}"
+    act = cli[:dry] ? 'to download' : 'downloaded'
+    cli.say "Datasets #{act}: #{downloaded}"
+    unless cli[:remote_list].nil?
+      File.open(cli[:remote_list], 'w') do |fh|
+        d.each { |i| fh.puts i }
+      end
+    end
+    return unless cli[:unlink]
+
+    unlink = p.dataset_names - d
+    unlink.each { |i| p.unlink_dataset(i).remove! }
+    cli.say "Datasets unlinked: #{unlink.size}"
+  end
+
+end

data/lib/miga/cli/action/ncbi_get.rb

@@ -3,8 +3,8 @@
 require 'miga/cli/action'
 
 class MiGA::Cli::Action::NcbiGet < MiGA::Cli::Action
-  require 'miga/cli/action/ncbi_get/downloads'
-  include MiGA::Cli::Action::NcbiGet::Downloads
+  require 'miga/cli/action/download/ncbi'
+  include MiGA::Cli::Action::Download::Ncbi
 
   def parse_cli
     cli.defaults = {

data/lib/miga/cli/base.rb

@@ -19,6 +19,7 @@ module MiGA::Cli::Base
     add: 'Create a dataset in a MiGA project',
     get: 'Download a dataset from public databases into a MiGA project',
     ncbi_get: 'Download all genomes in a taxon from NCBI into a MiGA project',
+    gtdb_get: 'Download all genomes in a taxon from GTDB into a MiGA project',
     rm: 'Remove a dataset from a MiGA project',
     find: 'Find unregistered datasets based on result files',
     ln: 'Link datasets (including results) from one project to another',

data/lib/miga/remote_dataset/base.rb

@@ -12,6 +12,8 @@ end
 
 module MiGA::RemoteDataset::Base
   @@_EUTILS = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/'
+  @@_EBI_API = 'https://www.ebi.ac.uk/Tools'
+  @@_GTDB_API = 'https://api.gtdb.ecogenomic.org'
   @@_NCBI_API_KEY = lambda { |url|
     ENV['NCBI_API_KEY'].nil? ? url : "#{url}&api_key=#{ENV['NCBI_API_KEY']}"
   }
@@ -43,9 +45,26 @@ module MiGA::RemoteDataset::Base
     },
     ebi: {
       dbs: { embl: { stage: :assembly, format: :fasta } },
-      url: 'https://www.ebi.ac.uk/Tools/dbfetch/dbfetch/%1$s/%2$s/%3$s',
+      url: "#{@@_EBI_API}/dbfetch/dbfetch/%1$s/%2$s/%3$s",
       method: :rest
     },
+    gtdb: {
+      dbs: {
+        # This is a dummy entry plugged directly to +ncbi_asm_rest+
+        assembly: { stage: :assembly, format: :fasta_gz, getter: :ncbi_asm },
+        # The 'taxon' namespace actually returns a list of genomes (+format+)
+        taxon: {
+          stage: :metadata, format: :genomes, map_to: [:assembly],
+          extra: ['sp_reps_only=false']
+        },
+        # The 'genome' namespace actually returns the taxonomy (+format+)
+        genome: { stage: :metadata, format: 'taxon-history' }
+      },
+      url: "#{@@_GTDB_API}/%1$s/%2$s/%3$s?%4$s",
+      method: :rest,
+      map_to_universe: :ncbi,
+      headers: 'accept: application/json' # < TODO not currently supported
+    },
     ncbi: {
       dbs: {
         nuccore: { stage: :assembly, format: :fasta, getter: :ncbi_gb },

data/lib/miga/remote_dataset.rb

@@ -49,7 +49,7 @@ class MiGA::RemoteDataset < MiGA::MiGA
     @@UNIVERSE.keys.include?(@universe) or
       raise "Unknown Universe: #{@universe}. Try: #{@@UNIVERSE.keys}"
     @@UNIVERSE[@universe][:dbs].include?(@db) or
-      raise "Unknown Database: #{@db}. Try: #{@@UNIVERSE[@universe][:dbs]}"
+      raise "Unknown Database: #{@db}. Try: #{@@UNIVERSE[@universe][:dbs].keys}"
     @_ncbi_asm_json_doc = nil
     # FIXME: Part of the +map_to+ support:
     # unless @@UNIVERSE[@universe][:dbs][@db][:map_to].nil?
@@ -104,6 +104,9 @@ class MiGA::RemoteDataset < MiGA::MiGA
     when :ebi, :ncbi, :web
       # Get taxonomy
       @metadata[:tax] = get_ncbi_taxonomy
+    when :gtdb
+      # Get taxonomy
+      @metadata[:tax] = get_gtdb_taxonomy
     end
     @metadata = get_type_status(metadata)
   end
@@ -129,10 +132,9 @@ class MiGA::RemoteDataset < MiGA::MiGA
   end
 
   ##
-  # Get NCBI taxonomy as MiGA::Taxonomy.
+  # Get NCBI taxonomy as MiGA::Taxonomy
   def get_ncbi_taxonomy
-    tax_id = get_ncbi_taxid
-    return nil if tax_id.nil?
+    tax_id = get_ncbi_taxid or return
 
     lineage = { ns: 'ncbi' }
     doc = MiGA::RemoteDataset.download(:ncbi, :taxonomy, tax_id, :xml)
@@ -147,12 +149,34 @@ class MiGA::RemoteDataset < MiGA::MiGA
     MiGA::Taxonomy.new(lineage)
   end
 
+  ##
+  # Get GTDB taxonomy as MiGA::Taxonomy
+  def get_gtdb_taxonomy
+    gtdb_genome = metadata[:gtdb_assembly] or return
+
+    doc = MiGA::Json.parse(
+      MiGA::RemoteDataset.download(
+        :gtdb, :genome, gtdb_genome, 'taxon-history', nil, ['']
+      ),
+      contents: true
+    )
+    lineage = { ns: 'gtdb' }
+    lineage.merge!(doc.first) # Get only the latest available classification
+    release = lineage.delete(:release)
+    @metadata[:gtdb_release] = release
+    lineage.transform_values! { |v| v.gsub(/^\S__/, '') }
+    MiGA.DEBUG "Got lineage from #{release}: #{lineage}"
+    MiGA::Taxonomy.new(lineage)
+  end
+
   ##
   # Get the JSON document describing an NCBI assembly entry.
   def ncbi_asm_json_doc
     return @_ncbi_asm_json_doc unless @_ncbi_asm_json_doc.nil?
 
-    metadata[:ncbi_asm] ||= ids.first if universe == :ncbi and db == :assembly
+    if db == :assembly && %i[ncbi gtdb].include?(universe)
+      metadata[:ncbi_asm] ||= ids.first
+    end
     return nil unless metadata[:ncbi_asm]
 
     ncbi_asm_id = self.class.ncbi_asm_acc2id metadata[:ncbi_asm]

data/lib/miga/version.rb

@@ -12,7 +12,7 @@ module MiGA
   # - String indicating release status:
   #   - rc* release candidate, not released as gem
   #   - [0-9]+ stable release, released as gem
-  VERSION = [1.2, 10, 2].freeze
+  VERSION = [1.2, 11, 0].freeze
 
   ##
   # Nickname for the current major.minor version.
@@ -20,7 +20,7 @@ module MiGA
 
   ##
   # Date of the current gem relese.
-  VERSION_DATE = Date.new(2022, 12, 14)
+  VERSION_DATE = Date.new(2022, 12, 28)
 
   ##
   # References of MiGA

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 1.2.10.2
+  version: 1.2.11.0
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-12-14 00:00:00.000000000 Z
+date: 2022-12-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
@@ -141,6 +141,9 @@ files:
 - lib/miga/cli/action/derep_wf.rb
 - lib/miga/cli/action/doctor.rb
 - lib/miga/cli/action/doctor/base.rb
+- lib/miga/cli/action/download/base.rb
+- lib/miga/cli/action/download/gtdb.rb
+- lib/miga/cli/action/download/ncbi.rb
 - lib/miga/cli/action/edit.rb
 - lib/miga/cli/action/env.rb
 - lib/miga/cli/action/files.rb
@@ -148,6 +151,7 @@ files:
 - lib/miga/cli/action/generic.rb
 - lib/miga/cli/action/get.rb
 - lib/miga/cli/action/get_db.rb
+- lib/miga/cli/action/gtdb_get.rb
 - lib/miga/cli/action/index_wf.rb
 - lib/miga/cli/action/init.rb
 - lib/miga/cli/action/init/daemon_helper.rb
@@ -156,7 +160,6 @@ files:
 - lib/miga/cli/action/ln.rb
 - lib/miga/cli/action/ls.rb
 - lib/miga/cli/action/ncbi_get.rb
-- lib/miga/cli/action/ncbi_get/downloads.rb
 - lib/miga/cli/action/new.rb
 - lib/miga/cli/action/next_step.rb
 - lib/miga/cli/action/option.rb