miga-base 1.2.16.1 → 1.2.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a66176a9b04ef20b3c415e195d4acdb38c18f681b3f7ee52899d79fe0672b570
-  data.tar.gz: d6b98ea8416dec7abd29d1255f512d5ca352efd58dd002616377b88079beb09b
+  metadata.gz: a33ac43db8c09ad7cad760c1bc82102ca1629beba12bea7c15d5391569210d95
+  data.tar.gz: 5104348e8471729c6ce59612a88e624f44d382480e313e00329e8766ead0e8a5
 SHA512:
-  metadata.gz: dea24c16ab06cb21d0f8130cccbfe8d9edddbf96f7de9329a10218e086bcd90c25d1f672f61b94ae115e8cdc4fbf24d3febf3009d3c11f0f0e3129c134100d39
-  data.tar.gz: 7a40aa1e7fc58efd768aec30e9e21110d2b8ab7ab938b8ccc8f6ff5737ef9b44624e926f955d05239099289407f990dcd0f13da36ae065064dbf301bbe355091
+  metadata.gz: fd9c726cc1da17dc2146772491a580d471739876f036368fb5a2c2fab475604f0ae858b3e02cebcb7dd22c029d01ade2010a68acb86e752a85e7723dabcebec0
+  data.tar.gz: 898c7b240b5d1c8e8f911f2111d9e326752cdb25b36c646aa21402c140c52187382fc9bf4cf621b5a0b3cfc715f0d596beefaaece3827822b4ad515d6e420aac

data/LICENSE

@@ -1,6 +1,6 @@
-               The Artistic License 2.0
+                       The Artistic License 2.0
 
-           Copyright (c) 2016 Luis M Rodriguez-R
+  Copyright (c) 2016-2023 Luis M Rodriguez-R (Universität Innsbruck)
 
      Everyone is permitted to copy and distribute verbatim copies
       of this license document, but changing it is not allowed.

data/lib/miga/cli/action/download/seqcode.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'miga/cli/action/download/base'
+
+##
+# Helper module including download functions for the seqcode_get action
+module MiGA::Cli::Action::Download::Seqcode
+  include MiGA::Cli::Action::Download::Base
+
+  def cli_task_flags(_opt)
+  end
+
+  def cli_name_modifiers(opt)
+    opt.on(
+      '--no-version-name',
+      'Do not add sequence version to the dataset name'
+    ) { |v| cli[:add_version] = v }
+  end
+
+  def sanitize_cli
+    cli[:save_every] = 1 if cli[:dry]
+  end
+
+  def remote_list
+    cli.say 'Downloading genome list'
+    current_page = 1
+    total_pages  = 1
+    ds = {}
+
+    while current_page <= total_pages
+      json = MiGA::RemoteDataset.download(
+        :seqcode, :'type-genomes', nil, :json, nil,
+        ["page=#{current_page}"]
+      )
+      doc = MiGA::Json.parse(json, contents: true)
+      current_page = doc[:current_page] + 1
+      total_pages  = doc[:total_pages]
+
+      doc[:values].each do |name|
+        next unless name[:type_material]
+        acc = name[:type_material].values.first
+        db  = name[:type_material].keys.first
+        next unless %i[assembly nuccore].include?(db) # No INSDC genome, ignore
+
+        classif = name[:classification] || {}
+        tax = MiGA::Taxonomy.new(Hash[classif.map { |i| [i[:rank], i[:name]] }])
+        tax << { 'ns' => 'seqcode', name[:rank] => name[:name] }
+        d = {
+          ids: [acc], db: db, universe: :seqcode,
+          md: {
+            type: :genome, tax: tax, is_type: true,
+            type_rel: 'SeqCode type genome',
+            seqcode_url: "https://seqco.de/i:#{name[:id]}"
+          }
+        }
+        d[:md][:get_ncbi_taxonomy] = true if cli[:get_ncbi_taxonomy]
+        ds[remote_row_name(tax, db, acc)] = d
+      end
+    end
+    ds
+  end
+
+  def remote_row_name(tax, db, asm)
+    acc = asm.to_s
+    acc.gsub!(/\.\d+\Z/, '') unless cli[:add_version]
+    db_short = { assembly: 'asm', nuccore: 'gb' }[db]
+    "#{tax.lowest[1]}_#{db_short}_#{acc}".miga_name
+  end
+end

data/lib/miga/cli/action/seqcode_get.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::SeqcodeGet < MiGA::Cli::Action
+  require 'miga/cli/action/download/seqcode'
+  include MiGA::Cli::Action::Download::Seqcode
+
+  def parse_cli
+    cli.defaults = {
+      query: false, unlink: false,
+      reference: false, add_version: true, dry: false,
+      get_md: false, only_md: false, save_every: 1
+    }
+    cli.parse do |opt|
+      cli.opt_object(opt, [:project])
+      cli_base_flags(opt)
+      opt.on(
+        '--ncbi-taxonomy',
+        'Retrieve NCBI taxonomy instead of SeqCode taxonomy'
+      ) { |v| cli[:get_ncbi_taxonomy] = v }
+      cli_task_flags(opt)
+      cli_name_modifiers(opt)
+      cli_filters(opt)
+      cli_save_actions(opt)
+      opt.on(
+        '--ncbi-api-key STRING',
+        'NCBI API key'
+      ) { |v| ENV['NCBI_API_KEY'] = v }
+    end
+  end
+
+  def perform
+    generic_perform
+  end
+
+end

data/lib/miga/cli/action/wf.rb

@@ -47,10 +47,15 @@ module MiGA::Cli::Action::Wf
         '--gtdb-ref',
         'Only download reference anchor genomes in GTDB (requires -G)'
       ) { |v| cli[:gtdb_ref] = v }
+      opt.on(
+        '-S', '--seqcode-type',
+        'Download all type genomes from the SeqCode Registry'
+      ) { |v| cli[:seqcode_type] = v }
       opt.on(
         '--max-download INT', Integer,
-        'Maximum number of genomes to download (by default: unlimited)'
-      ) { |v| cli[:ncbi_max] = v }
+        'Maximum number of genomes to download (by default: unlimited)',
+        'It applies independently to -T, -G and --S'
+      ) { |v| cli[:max_download] = v }
     end
     if params[:qual]
       opt.on(
@@ -131,24 +136,39 @@ module MiGA::Cli::Action::Wf
     paired = cli[:input_type].to_s.include?('_paired')
     cli[:regexp] ||= MiGA::Cli.FILE_REGEXP(paired)
 
-    # Create empty project
+    # Create empty project and populate with datasets
+    p = initialize_empty_project(p_metadata)
+    download_datasets
+    import_datasets(stage)
+
+    # Define datasets metadata
+    p.load
+    d_metadata[:type] = cli[:dataset_type]
+    p.each_dataset { |d| transfer_metadata(d, d_metadata) }
+    p
+  end
+
+  def initialize_empty_project(metadata)
     call_cli(
       ['new', '-P', cli[:outdir], '-t', cli[:project_type]]
     ) unless MiGA::Project.exist? cli[:outdir]
 
     # Define project metadata
     p = cli.load_project(:outdir, '-o')
-    p_metadata[:type] = cli[:project_type]
-    transfer_metadata(p, p_metadata)
+    metadata[:type] = cli[:project_type]
+    transfer_metadata(p, metadata)
     %i[haai_p aai_p ani_p ess_coll min_qual].each do |i|
       p.set_option(i, cli[i])
     end
+    p
+  end
 
+  def download_datasets
     # Download datasets from NCBI
     unless cli[:ncbi_taxon].nil?
       what = cli[:ncbi_draft] ? '--all' : '--complete'
       cmd = ['ncbi_get', '-P', cli[:outdir], '-T', cli[:ncbi_taxon], what]
-      cmd += ['--max', cli[:ncbi_max]] if cli[:ncbi_max]
+      cmd += ['--max', cli[:max_download]] if cli[:max_download]
       call_cli(cmd)
     end
 
@@ -156,11 +176,19 @@ module MiGA::Cli::Action::Wf
     unless cli[:gtdb_taxon].nil?
       cmd = ['gtdb_get', '-P', cli[:outdir], '-T', cli[:gtdb_taxon]]
       cmd << '--reference' if cli[:gtdb_ref]
-      cmd += ['--max', cli[:ncbi_max]] if cli[:ncbi_max]
+      cmd += ['--max', cli[:max_download]] if cli[:max_download]
       call_cli(cmd)
     end
 
-    # Add datasets
+    # Download datasets from SeqCode Registry
+    if cli[:seqcode_type]
+      cmd = ['seqcode_get', '-P', cli[:outdir]]
+      cmd += ['--max', cli[:max_download]] if cli[:max_download]
+      call_cli(cmd)
+    end
+  end
+
+  def import_datasets(stage)
     call_cli(
       [
         'add',
@@ -171,12 +199,6 @@ module MiGA::Cli::Action::Wf
         '-R', cli[:regexp]
       ] + cli.files
     ) unless cli.files.empty?
-
-    # Define datasets metadata
-    p.load
-    d_metadata[:type] = cli[:dataset_type]
-    p.each_dataset { |d| transfer_metadata(d, d_metadata) }
-    p
   end
 
   def summarize(which = %w[cds assembly essential_genes ssu])

data/lib/miga/cli/base.rb

@@ -20,6 +20,7 @@ module MiGA::Cli::Base
     get: 'Download a dataset from public databases into a MiGA project',
     ncbi_get: 'Download all genomes in a taxon from NCBI into a MiGA project',
     gtdb_get: 'Download all genomes in a taxon from GTDB into a MiGA project',
+    seqcode_get: 'Download all type genomes from SeqCode into a MiGA project',
     rm: 'Remove a dataset from a MiGA project',
     find: 'Find unregistered datasets based on result files',
     ln: 'Link datasets (including results) from one project to another',

data/lib/miga/common/net.rb

@@ -15,16 +15,22 @@ module MiGA::Common::Net
   def known_hosts(name)
     case name.to_sym
     when :miga_online_ftp
-      'ftp://microbial-genomes.org//' # <- // to simplify chdir in connection
+      "ftp://#{main_server}//" # <- // to simplify chdir in connection
     when :miga_db
-      'ftp://microbial-genomes.org/db'
+      "ftp://#{main_server}/db"
     when :miga_dist
-      'ftp://microbial-genomes.org/dist'
+      "ftp://#{main_server}/dist"
     else
       raise "Unrecognized server name: #{host}"
     end
   end
 
+  ##
+  # Returns the address of the main MiGA server
+  def main_server
+    'gatech.microbial-genomes.org'
+  end
+
   ##
   # Connect to an FTP +host+ (String) or a known host name (Symbol, see
   # +.known_hosts+)

data/lib/miga/remote_dataset/base.rb

@@ -14,6 +14,7 @@ module MiGA::RemoteDataset::Base
   @@_EUTILS = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/'
   @@_EBI_API = 'https://www.ebi.ac.uk/Tools'
   @@_GTDB_API = 'https://api.gtdb.ecogenomic.org'
+  @@_SEQCODE_API = 'https://disc-genomics.uibk.ac.at/seqcode'
   @@_NCBI_API_KEY = lambda { |url|
     ENV['NCBI_API_KEY'].nil? ? url : "#{url}&api_key=#{ENV['NCBI_API_KEY']}"
   }
@@ -65,6 +66,18 @@ module MiGA::RemoteDataset::Base
       map_to_universe: :ncbi,
       headers: 'accept: application/json' # < TODO not currently supported
     },
+    seqcode: {
+      dbs: {
+        # These are dummy entries plugged directly to +ncbi_*_rest+
+        assembly: { stage: :assembly, format: :fasta_gz, getter: :ncbi_asm },
+        nuccore:  { stage: :assembly, format: :fasta, getter: :ncbi_gb },
+        # This is the list of type genomes
+        :'type-genomes' => { stage: :metadata, format: :json }
+      },
+      url: "#{@@_SEQCODE_API}/%1$s.json?%4$s",
+      method: :rest,
+      map_to_universe: :ncbi
+    },
     ncbi: {
       dbs: {
         nuccore: { stage: :assembly, format: :fasta, getter: :ncbi_gb },

data/lib/miga/remote_dataset.rb

@@ -133,6 +133,8 @@ class MiGA::RemoteDataset < MiGA::MiGA
     when :gtdb
       # Get taxonomy
       @metadata[:tax] = get_gtdb_taxonomy
+    when :seqcode
+      # Do nothing, taxonomy already defined
     end
 
     if metadata[:get_ncbi_taxonomy]
@@ -206,7 +208,7 @@ class MiGA::RemoteDataset < MiGA::MiGA
   def ncbi_asm_json_doc
     return @_ncbi_asm_json_doc unless @_ncbi_asm_json_doc.nil?
 
-    if db == :assembly && %i[ncbi gtdb].include?(universe)
+    if db == :assembly && %i[ncbi gtdb seqcode].include?(universe)
       metadata[:ncbi_asm] ||= ids.first
     end
     return nil unless metadata[:ncbi_asm]
@@ -244,6 +246,7 @@ class MiGA::RemoteDataset < MiGA::MiGA
     ln
   end
 
+  alias :get_ncbi_taxid_from_seqcode :get_ncbi_taxid_from_ncbi
   alias :get_ncbi_taxid_from_gtdb :get_ncbi_taxid_from_ncbi
 
   def get_ncbi_taxid_from_ebi

data/lib/miga/version.rb

@@ -12,7 +12,7 @@ module MiGA
   # - String indicating release status:
   #   - rc* release candidate, not released as gem
   #   - [0-9]+ stable release, released as gem
-  VERSION = [1.2, 16, 1].freeze
+  VERSION = [1.2, 17, 0].freeze
 
   ##
   # Nickname for the current major.minor version.
@@ -20,7 +20,7 @@ module MiGA
 
   ##
   # Date of the current gem relese.
-  VERSION_DATE = Date.new(2023, 2, 13)
+  VERSION_DATE = Date.new(2023, 2, 14)
 
   ##
   # References of MiGA

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 1.2.16.1
+  version: 1.2.17.0
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-02-13 00:00:00.000000000 Z
+date: 2023-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
@@ -147,6 +147,7 @@ files:
 - lib/miga/cli/action/download/base.rb
 - lib/miga/cli/action/download/gtdb.rb
 - lib/miga/cli/action/download/ncbi.rb
+- lib/miga/cli/action/download/seqcode.rb
 - lib/miga/cli/action/edit.rb
 - lib/miga/cli/action/env.rb
 - lib/miga/cli/action/files.rb
@@ -170,6 +171,7 @@ files:
 - lib/miga/cli/action/quality_wf.rb
 - lib/miga/cli/action/rm.rb
 - lib/miga/cli/action/run.rb
+- lib/miga/cli/action/seqcode_get.rb
 - lib/miga/cli/action/stats.rb
 - lib/miga/cli/action/summary.rb
 - lib/miga/cli/action/tax_dist.rb