miga-base 1.2.14.2 → 1.2.15.1

data/lib/miga/dataset/result.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'miga/result'
 require 'miga/dataset/base'
 require 'miga/common/with_result'
@@ -13,56 +15,17 @@ module MiGA::Dataset::Result
   include MiGA::Dataset::Base
   include MiGA::Common::WithResult
 
+  require 'miga/dataset/result/ignore'
+  require 'miga/dataset/result/add'
+  include MiGA::Dataset::Result::Ignore
+  include MiGA::Dataset::Result::Add
+
   ##
   # Return the basename for results
   def result_base
     name
   end
 
-  ##
-  # Should I ignore +task+ for this dataset?
-  def ignore_task?(task)
-    why_ignore(task) != :execute
-  end
-
-  ##
-  # Return a code explaining why a task is ignored.
-  # The values are symbols:
-  # - empty: the dataset has no data
-  # - inactive: the dataset is inactive
-  # - upstream: the task is upstream from dataset's input
-  # - force: forced to ignore by metadata
-  # - project: incompatible project
-  # - noref: incompatible dataset, only for reference
-  # - multi: incompatible dataset, only for multi
-  # - nonmulti: incompatible dataset, only for nonmulti
-  # - complete: the task is already complete
-  # - execute: do not ignore, execute the task
-  def why_ignore(task)
-    if !get_result(task).nil?
-      :complete
-    elsif !active?
-      :inactive
-    elsif first_preprocessing.nil?
-      :empty
-    elsif @@PREPROCESSING_TASKS.index(task) <
-          @@PREPROCESSING_TASKS.index(first_preprocessing)
-      :upstream
-    elsif !metadata["run_#{task}"].nil?
-      metadata["run_#{task}"] ? :execute : :force
-    elsif task == :taxonomy && project.option(:ref_project).nil?
-      :project
-    elsif @@_EXCLUDE_NOREF_TASKS_H[task] && !ref?
-      :noref
-    elsif @@_ONLY_MULTI_TASKS_H[task] && !multi?
-      :multi
-    elsif @@_ONLY_NONMULTI_TASKS_H[task] && !nonmulti?
-      :nonmulti
-    else
-      :execute
-    end
-  end
-
   ##
   # Returns the key symbol of the first registered result (sorted by the
   # execution order). This typically corresponds to the result used as the
@@ -95,18 +58,18 @@ module MiGA::Dataset::Result
   # - 2 for a queued result (a task yet to be executed).
   # It passes +save+ to #add_result
   def profile_advance(save = false)
-    first_task = first_preprocessing(save)
-    return Array.new(@@PREPROCESSING_TASKS.size, 0) if first_task.nil?
+    # Determine the start point
+    first_task = first_preprocessing(save) or
+      return Array.new(self.class.PREPROCESSING_TASKS.size, 0)
 
-    adv = []
+    # Traverse all tasks and determine the corresponding state
     state = 0
     next_task = next_preprocessing(save)
-    @@PREPROCESSING_TASKS.each do |task|
+    self.class.PREPROCESSING_TASKS.map do |task|
       state = 1 if first_task == task
       state = 2 if !next_task.nil? && next_task == task
-      adv << state
+      state
     end
-    adv
   end
 
   ##
@@ -125,9 +88,9 @@ module MiGA::Dataset::Result
   def result_status(task)
     reason = why_ignore(task)
     case reason
-    when :upstream; :-
-    when :execute; :pending
-    when :complete; :complete
+    when :upstream then :-
+    when :execute  then :pending
+    when :complete then :complete
     else; :"ignore_#{reason}"
     end
   end
@@ -136,306 +99,32 @@ module MiGA::Dataset::Result
   # Clean-up all the stored distances, removing values for datasets no longer in
   # the project as reference datasets.
   def cleanup_distances!
-    r = get_result(:distances)
-    return if r.nil?
+    return if get_result(:distances).nil?
 
     require 'miga/sqlite'
     ref = project.datasets.select(&:ref?).select(&:active?).map(&:name)
-    %i[haai_db aai_db ani_db].each do |db_type|
-      db = r.file_path(db_type)
-      next if db.nil? || !File.size?(db)
-
-      sqlite_db = MiGA::SQLite.new(db)
-      table = db_type[-6..-4]
-      val = sqlite_db.run("select seq2 from #{table}")
-      next if val.empty?
-
-      (val.map(&:first) - ref).each do |extra|
-        sqlite_db.run("delete from #{table} where seq2=?", extra)
-      end
+    %i[haai aai ani].each do |metric|
+      cleanup_distances_by_metric!(ref, metric)
     end
   end
 
   private
 
   ##
-  # Add result type +:raw_reads+ at +base+ (no +_opts+ supported)
-  def add_result_raw_reads(base, _opts)
-    return nil unless result_files_exist?(base, '.1.fastq')
-
-    add_files_to_ds_result(
-      MiGA::Result.new("#{base}.json"), name,
-      if result_files_exist?(base, '.2.fastq')
-        { pair1: '.1.fastq', pair2: '.2.fastq' }
-      else
-        { single: '.1.fastq' }
-      end
-    )
-  end
-
-  ##
-  # Add result type +:trimmed_reads+ at +base+ (no +_opts+ supported)
-  def add_result_trimmed_reads(base, _opts)
-    return nil unless result_files_exist?(base, '.1.clipped.fastq')
-
-    add_files_to_ds_result(
-      MiGA::Result.new("#{base}.json"), name,
-      if result_files_exist?(base, '.2.clipped.fastq')
-        { pair1: '.1.clipped.fastq', pair2: '.2.clipped.fastq' }
-      else
-        { single: '.1.clipped.fastq' }
-      end
-    ).tap do |r|
-      # Legacy files
-      r.add_file(:trimming_sumary, "#{name}.1.fastq.trimmed.summary.txt")
-      r.add_file(:single, "#{name}.1.clipped.single.fastq")
-    end
-  end
-
-  ##
-  # Add result type +:read_quality+ at +base+ (no +_opts+ supported)
-  def add_result_read_quality(base, _opts)
-    return nil unless
-      result_files_exist?(base, %w[.post.1.html]) ||
-      result_files_exist?(base, %w[.solexaqa .fastqc])
-
-    add_files_to_ds_result(
-      MiGA::Result.new("#{base}.json"), name,
-      pre_qc_1: '.pre.1.html', pre_qc_2: '.pre.2.html',
-      post_qc_1: '.post.1.html', post_qc_2: '.post.2.html',
-      adapter_detection: '.adapters.txt',
-      # Legacy files
-      solexaqa: '.solexaqa', fastqc: '.fastqc'
-    )
-  end
-
-  ##
-  # Add result type +:trimmed_fasta+ at +base+ (no +_opts+ supported)
-  def add_result_trimmed_fasta(base, _opts)
-    return nil unless
-      result_files_exist?(base, '.CoupledReads.fa') ||
-      result_files_exist?(base, '.SingleReads.fa')  ||
-      result_files_exist?(base, %w[.1.fasta .2.fasta])
-
-    add_files_to_ds_result(
-      MiGA::Result.new("#{base}.json"), name,
-      coupled: '.CoupledReads.fa',
-      single: '.SingleReads.fa',
-      pair1: '.1.fasta',
-      pair2: '.2.fasta'
-    )
-  end
-
-  ##
-  # Add result type +:assembly+ at +base+. Hash +opts+ supports
-  # +is_clean: Boolean+.
-  def add_result_assembly(base, opts)
-    return nil unless result_files_exist?(base, '.LargeContigs.fna')
-
-    r = add_files_to_ds_result(
-      MiGA::Result.new("#{base}.json"), name,
-      largecontigs: '.LargeContigs.fna',
-      allcontigs: '.AllContigs.fna',
-      assembly_data: ''
-    )
-    opts[:is_clean] ||= false
-    r.clean! if opts[:is_clean]
-    unless r.clean?
-      MiGA::MiGA.clean_fasta_file(r.file_path(:largecontigs))
-      r.clean!
-    end
-    r
-  end
-
-  ##
-  # Add result type +:cds+ at +base+. Hash +opts+ supports +is_clean: Boolean+
-  def add_result_cds(base, opts)
-    return nil unless result_files_exist?(base, %w[.faa])
-
-    r = add_files_to_ds_result(
-      MiGA::Result.new("#{base}.json"), name,
-      proteins: '.faa',
-      genes: '.fna',
-      gff2: '.gff2',
-      gff3: '.gff3',
-      tab: '.tab'
-    )
-    opts[:is_clean] ||= false
-    r.clean! if opts[:is_clean]
-    unless r.clean?
-      MiGA::MiGA.clean_fasta_file(r.file_path(:proteins))
-      MiGA::MiGA.clean_fasta_file(r.file_path(:genes)) if r.file_path(:genes)
-      r.clean!
-    end
-    r
-  end
-
-  ##
-  # Add result type +:essential_genes+ at +base+ (no +_opts+ supported).
-  def add_result_essential_genes(base, _opts)
-    return nil unless result_files_exist?(base, %w[.ess.faa .ess .ess/log])
+  # Cleanup the tables of a specific +metric+ (symbol) removing all values
+  # against dataset names not in +ref+ (Array of string)
+  def cleanup_distances_by_metric!(ref, metric)
+    db_type = :"#{metric}_db"
+    db = get_result(:distances).file_path(db_type)
+    return if db.nil? || !File.size?(db)
 
-    add_files_to_ds_result(
-      MiGA::Result.new("#{base}.json"), name,
-      ess_genes: '.ess.faa',
-      collection: '.ess',
-      report: '.ess/log',
-      alignments: '.ess/proteins.aln',
-      fastaai_index: '.faix.db.gz',
-      fastaai_index_2: '.faix'
-    )
-  end
-
-  ##
-  # Add result type +:ssu+ at +base+. Hash +opts+ supports +is_clean: Boolean+
-  def add_result_ssu(base, opts)
-    return MiGA::Result.new("#{base}.json") if result(:assembly).nil?
-    return nil unless result_files_exist?(base, '.ssu.fa')
-
-    r = add_files_to_ds_result(
-      MiGA::Result.new("#{base}.json"), name,
-      longest_ssu_gene: '.ssu.fa',
-      ssu_gff: '.ssu.gff', # DEPRECATED
-      gff: '.gff',
-      all_ssu_genes: '.ssu.all.fa',
-      classification: '.rdp.tsv',
-      trna_list: '.trna.txt'
-    )
-    opts[:is_clean] ||= false
-    r.clean! if opts[:is_clean]
-    unless r.clean?
-      MiGA::MiGA.clean_fasta_file(r.file_path(:longest_ssu_gene))
-      r.clean!
-    end
-    r
-  end
-
-  ##
-  # Add result type +:mytaxa+ at +base+ (no +_opts+ supported)
-  def add_result_mytaxa(base, _opts)
-    if multi?
-      return nil unless
-        result_files_exist?(base, '.mytaxa') ||
-        result_files_exist?(base, '.nomytaxa.txt')
-
-      add_files_to_ds_result(
-        MiGA::Result.new("#{base}.json"), name,
-        mytaxa: '.mytaxa',
-        blast: '.blast',
-        mytaxain: '.mytaxain',
-        nomytaxa: '.nomytaxa.txt',
-        species: '.mytaxa.Species.txt',
-        genus: '.mytaxa.Genus.txt',
-        phylum: '.mytaxa.Phylum.txt',
-        innominate: '.mytaxa.innominate',
-        kronain: '.mytaxa.krona',
-        krona: '.html'
-      )
-    else
-      MiGA::Result.new("#{base}.json")
-    end
-  end
-
-  ##
-  # Add result type +:mytaxa_scan+ at +base+ (no +_opts+ supported)
-  def add_result_mytaxa_scan(base, _opts)
-    if nonmulti?
-      return nil unless
-        result_files_exist?(base, %w[.pdf .mytaxa]) ||
-        result_files_exist?(base, '.nomytaxa.txt')
-
-      add_files_to_ds_result(
-        MiGA::Result.new("#{base}.json"), name,
-        nomytaxa: '.nomytaxa.txt',
-        mytaxa: '.mytaxa',
-        report: '.pdf',
-        regions_archive: '.reg.tar',
-        # Intermediate / Deprecated:
-        blast: '.blast',
-        mytaxain: '.mytaxain',
-        wintax: '.wintax',
-        gene_ids: '.wintax.genes',
-        region_ids: '.wintax.regions',
-        regions: '.reg'
-      )
-    else
-      MiGA::Result.new("#{base}.json")
-    end
-  end
-
-  ##
-  # Add result type +:distances+ at +base+ (no +_opts+ supported)
-  def add_result_distances(base, _opts)
-    if nonmulti?
-      if ref?
-        add_result_distances_ref(base)
-      else
-        add_result_distances_nonref(base)
-      end
-    else
-      add_result_distances_multi(base)
-    end
-  end
-
-  ##
-  # Add result type +:taxonomy+ at +base+ (no +_opts+ supported)
-  def add_result_taxonomy(base, _opts)
-    add_result_distances_nonref(base)
-  end
-
-  ##
-  # Add result type +:stats+ at +base+ (no +_opts+ supported)
-  def add_result_stats(base, _opts)
-    MiGA::Result.new("#{base}.json")
-  end
-
-  ##
-  # Add result type +:distances+ for _multi_ datasets at +base+
-  def add_result_distances_multi(base)
-    MiGA::Result.new("#{base}.json")
-  end
+    sqlite_db = MiGA::SQLite.new(db)
+    table = db_type[-6..-4]
+    val = sqlite_db.run("select seq2 from #{table}")
+    return if val.empty?
 
-  ##
-  # Add result type +:distances+ for _nonmulti_ reference datasets at +base+
-  def add_result_distances_ref(base)
-    pref = File.dirname(base)
-    return nil unless File.exist?("#{pref}/01.haai/#{name}.db")
-
-    MiGA::Result.new("#{base}.json").tap do |r|
-      r.add_files(
-        haai_db: "01.haai/#{name}.db",
-        aai_db: "02.aai/#{name}.db",
-        ani_db: "03.ani/#{name}.db"
-      )
+    (val.map(&:first) - ref).each do |extra|
+      sqlite_db.run("delete from #{table} where seq2=?", extra)
     end
   end
-
-  ##
-  # Add result type +:distances+ for _nonmulti_ query datasets at +base+
-  def add_result_distances_nonref(base)
-    return nil unless
-      result_files_exist?(base, %w[.aai-medoids.tsv .aai.db]) ||
-      result_files_exist?(base, %w[.ani-medoids.tsv .ani.db])
-
-    add_files_to_ds_result(
-      MiGA::Result.new("#{base}.json"), name,
-      aai_medoids: '.aai-medoids.tsv',
-      haai_db: '.haai.db',
-      aai_db: '.aai.db',
-      ani_medoids: '.ani-medoids.tsv',
-      ani_db: '.ani.db',
-      ref_tree: '.nwk',
-      ref_tree_pdf: '.nwk.pdf',
-      intax_test: '.intax.txt'
-    )
-  end
-
-  ##
-  # Add files in +rel_files+ Hash to the result +r+ with dataset name +name+
-  def add_files_to_ds_result(r, name, rel_files)
-    files = {}
-    rel_files.each { |k, v| files[k] = name + v }
-    r.add_files(files)
-    r
-  end
 end

data/lib/miga/remote_dataset/download.rb

@@ -43,6 +43,12 @@ class MiGA::RemoteDataset
     # +format+: String, passed to download
     def ncbi_asm_rest(opts)
       url_dir = opts[:obj].ncbi_asm_json_doc['ftppath_genbank']
+      if url_dir.nil? || url_dir.empty?
+        raise MiGA::RemoteDataMissingError.new(
+          "Missing ftppath_genbank in NCBI Assembly JSON"
+        )
+      end
+
       url = "#{url_dir}/#{File.basename url_dir}_genomic.fna.gz"
       download(
         :web, :assembly_gz, url,

data/lib/miga/version.rb

@@ -12,7 +12,7 @@ module MiGA
   # - String indicating release status:
   #   - rc* release candidate, not released as gem
   #   - [0-9]+ stable release, released as gem
-  VERSION = [1.2, 14, 2].freeze
+  VERSION = [1.2, 15, 1].freeze
 
   ##
   # Nickname for the current major.minor version.
@@ -20,7 +20,7 @@ module MiGA
 
   ##
   # Date of the current gem relese.
-  VERSION_DATE = Date.new(2023, 1, 16)
+  VERSION_DATE = Date.new(2023, 1, 24)
 
   ##
   # References of MiGA

data/test/remote_dataset_test.rb

@@ -106,6 +106,12 @@ class RemoteDatasetTest < Test::Unit::TestCase
     assert { rd.get_metadata[:is_ref_type] }
   end
 
+  def test_missing_data
+    declare_remote_access
+    rd = MiGA::RemoteDataset.new('GCA_000484975.1', :assembly, :ncbi)
+    assert_raise(MiGA::RemoteDataMissingError) { rd.save_to(project, 'bad') }
+  end
+
   # This test is too expensive (too much time to run it!)
   # def test_net_timeout
   #   declare_remote_access

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 1.2.14.2
+  version: 1.2.15.1
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-01-16 00:00:00.000000000 Z
+date: 2023-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
@@ -141,6 +141,9 @@ files:
 - lib/miga/cli/action/derep_wf.rb
 - lib/miga/cli/action/doctor.rb
 - lib/miga/cli/action/doctor/base.rb
+- lib/miga/cli/action/doctor/databases.rb
+- lib/miga/cli/action/doctor/distances.rb
+- lib/miga/cli/action/doctor/operations.rb
 - lib/miga/cli/action/download/base.rb
 - lib/miga/cli/action/download/gtdb.rb
 - lib/miga/cli/action/download/ncbi.rb
@@ -196,6 +199,8 @@ files:
 - lib/miga/dataset/base.rb
 - lib/miga/dataset/hooks.rb
 - lib/miga/dataset/result.rb
+- lib/miga/dataset/result/add.rb
+- lib/miga/dataset/result/ignore.rb
 - lib/miga/dataset/status.rb
 - lib/miga/json.rb
 - lib/miga/lair.rb