miga-base 0.7.3.0 → 0.7.7.0

data/lib/miga/daemon/base.rb

@@ -1,11 +1,9 @@
-
 require 'shellwords'
 
 class MiGA::Daemon < MiGA::MiGA
 end
 
 module MiGA::Daemon::Base
-
   ##
   # Set/get #options, where +k+ is the Symbol of the option and +v+ is the value
   # (or nil to use as getter). Skips consistency tests if +force+. Returns new
@@ -14,19 +12,21 @@ module MiGA::Daemon::Base
     k = k.to_sym
     unless v.nil?
       case k
-      when :latency, :maxjobs, :ppn, :format_version
+      when :latency, :maxjobs, :ppn, :format_version, :verbosity
         v = v.to_i
+        if !force && v == 0 && k != :verbosity
+          raise "Daemon's #{k} cannot be set to zero"
+        end
       when :shutdown_when_done
         v = !!v
       when :nodelist
         if v =~ /^\$/
-          vv = ENV[v.sub('$','')] or raise "Unset environment variable: #{v}"
+          vv = ENV[v.sub('$', '')] or raise "Unset environment variable: #{v}"
           v = vv
         end
         say "Reading node list: #{v}"
         v = File.readlines(v).map(&:chomp)
       end
-      raise "Daemon's #{k} cannot be set to zero." if !force and v == 0
       @runopts[k] = v
     end
     @runopts[k]
@@ -34,24 +34,43 @@ module MiGA::Daemon::Base
 
   ##
   # Returns Integer indicating the number of seconds to sleep between checks
-  def latency() runopts(:latency); end
+  def latency
+    runopts(:latency)
+  end
 
   ##
   # Returns Integer indicating the maximum number of concurrent jobs to run
-  def maxjobs() runopts(:maxjobs); end
+  def maxjobs
+    runopts(:maxjobs)
+  end
 
   ##
   # Returns the path to the list of execution hostnames
-  def nodelist() runopts(:nodelist); end
+  def nodelist
+    runopts(:nodelist)
+  end
 
   ##
   # Returns Integer indicating the number of CPUs per job
-  def ppn() runopts(:ppn); end
+  def ppn
+    runopts(:ppn)
+  end
 
   ##
   # Returns Boolean indicating if the daemon should shutdown when processing is
   # complete
-  def shutdown_when_done?() !!runopts(:shutdown_when_done); end
+  def shutdown_when_done?
+    !!runopts(:shutdown_when_done)
+  end
 
+  ##
+  # Returns the level of verbosity for the daemon as an Integer, or 1 if unset.
+  # Verbosity levels are:
+  # 0: No output
+  # 1: General daemon and job information
+  # 2: Same, and indicate when each task is performed (even if nothing happens)
+  # 3: Same, and indicate when each loop begins and ends
+  def verbosity
+    runopts(:verbosity) || 1
+  end
 end
-

data/lib/miga/dataset.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # @package MiGA
 # @license Artistic-2.0
 
@@ -8,7 +10,7 @@ require 'miga/dataset/hooks'
 require 'sqlite3'
 
 ##
-# Dataset representation in MiGA.
+# Dataset representation in MiGA
 class MiGA::Dataset < MiGA::MiGA
   include MiGA::Dataset::Result
   include MiGA::Dataset::Status
@@ -23,20 +25,20 @@ class MiGA::Dataset < MiGA::MiGA
     end
 
     ##
-    # Standard fields of metadata for datasets.
+    # Standard fields of metadata for datasets
     def INFO_FIELDS
-      %w(name created updated type ref user description comments)
+      %w[name created updated type ref user description comments]
     end
   end
 
   # Instance-level
 
   ##
-  # MiGA::Project that contains the dataset.
+  # MiGA::Project that contains the dataset
   attr_reader :project
 
   ##
-  # Datasets are uniquely identified by +name+ in a project.
+  # Datasets are uniquely identified by +name+ in a project
   attr_reader :name
 
   ##
@@ -45,22 +47,19 @@ class MiGA::Dataset < MiGA::MiGA
   # be treated as reference (true, default) or query (false). Pass any
   # additional +metadata+ as a Hash.
   def initialize(project, name, is_ref = true, metadata = {})
-    unless name.miga_name?
+    name.miga_name? or
       raise 'Invalid name, please use only alphanumerics and underscores: ' +
-        name.to_s
-    end
-    @project = project
-    @name = name
-    @metadata = nil
+            name.to_s
+    @project, @name, @metadata = project, name, nil
     metadata[:ref] = is_ref
     @metadata_future = [
-      File.expand_path("metadata/#{name}.json", project.path),
+      File.join(project.path, 'metadata', "#{name}.json"),
       metadata
     ]
-    unless File.exist? @metadata_future[0]
-      save
-      pull_hook :on_create
-    end
+    return if File.exist? @metadata_future[0]
+
+    save
+    pull_hook :on_create
   end
 
   ##
@@ -74,7 +73,7 @@ class MiGA::Dataset < MiGA::MiGA
   end
 
   ##
-  # Save any changes you've made in the dataset.
+  # Save any changes you've made in the dataset
   def save
     MiGA.DEBUG "Dataset.metadata: #{metadata.data}"
     metadata.save
@@ -82,61 +81,69 @@ class MiGA::Dataset < MiGA::MiGA
   end
 
   ##
-  # Get the type of dataset as Symbol.
-  def type ; metadata[:type] ; end
+  # Get the type of dataset as Symbol
+  def type
+    metadata[:type]
+  end
 
   ##
   # Delete the dataset with all it's contents (including results) and returns
-  # nil.
+  # nil
   def remove!
-    self.results.each{ |r| r.remove! }
-    self.metadata.remove!
+    results.each(&:remove!)
+    metadata.remove!
     pull_hook :on_remove
   end
 
   ##
-  # Inactivate a dataset. This halts automated processing by the daemon.
+  # Inactivate a dataset. This halts automated processing by the daemon
   def inactivate!
-    self.metadata[:inactive] = true
-    self.metadata.save
+    metadata[:inactive] = true
+    metadata.save
     pull_hook :on_inactivate
   end
 
   ##
-  # Activate a dataset. This removes the +:inactive+ flag.
+  # Activate a dataset. This removes the +:inactive+ flag
   def activate!
-    self.metadata[:inactive] = nil
-    self.metadata.save
+    metadata[:inactive] = nil
+    metadata.save
     pull_hook :on_activate
   end
 
   ##
-  # Get standard metadata values for the dataset as Array.
+  # Get standard metadata values for the dataset as Array
   def info
     MiGA::Dataset.INFO_FIELDS.map do |k|
-      (k == 'name') ? self.name : metadata[k.to_sym]
+      k == 'name' ? name : metadata[k]
     end
   end
 
   ##
   # Is this dataset a reference?
-  def ref? ; !!metadata[:ref] ; end
+  def ref?
+    !query?
+  end
 
   ##
   # Is this dataset a query (non-reference)?
-  def query? ; !metadata[:ref] ; end
+  def query?
+    !metadata[:ref]
+  end
 
   ##
   # Is this dataset known to be multi-organism?
   def multi?
-    return false if metadata[:type].nil? or @@KNOWN_TYPES[type].nil?
+    return false if metadata[:type].nil? || @@KNOWN_TYPES[type].nil?
+
     @@KNOWN_TYPES[type][:multi]
   end
 
   ##
   # Is this dataset known to be single-organism?
   def nonmulti?
-    return false if metadata[:type].nil? or @@KNOWN_TYPES[type].nil?
+    return false if metadata[:type].nil? || @@KNOWN_TYPES[type].nil?
+
     !@@KNOWN_TYPES[type][:multi]
   end
 
@@ -174,12 +181,15 @@ class MiGA::Dataset < MiGA::MiGA
   # +ref_project+ is false (default), and only for reference dataset when
   # +ref_project+ is true. It returns +nil+ if this analysis is not supported.
   def closest_relatives(how_many = 1, ref_project = false)
-    return nil if (ref? != ref_project) or multi?
+    return nil if (ref? != ref_project) || multi?
+
     r = result(ref_project ? :taxonomy : :distances)
     return nil if r.nil?
-    db = SQLite3::Database.new(r.file_path :aai_db)
+
+    db = SQLite3::Database.new(r.file_path(:aai_db))
     db.execute(
       'SELECT seq2, aai FROM aai WHERE seq2 != ? ' \
-      'GROUP BY seq2 ORDER BY aai DESC LIMIT ?', [name, how_many])
+      'GROUP BY seq2 ORDER BY aai DESC LIMIT ?', [name, how_many]
+    )
   end
 end

data/lib/miga/dataset/base.rb

@@ -2,72 +2,87 @@
 # @license Artistic-2.0
 
 class MiGA::Dataset < MiGA::MiGA
-
   # Class-level
   class << self
-    def RESULT_DIRS ; @@RESULT_DIRS ; end
-    def KNOWN_TYPES ; @@KNOWN_TYPES ; end
-    def PREPROCESSING_TASKS ; @@PREPROCESSING_TASKS ; end
+    def RESULT_DIRS
+      @@RESULT_DIRS
+    end
+
+    def KNOWN_TYPES
+      @@KNOWN_TYPES
+    end
+
+    def PREPROCESSING_TASKS
+      @@PREPROCESSING_TASKS
+    end
   end
-  
 end
 
 module MiGA::Dataset::Base
-  
   ##
-  # Directories containing the results from dataset-specific tasks.
+  # Directories containing the results from dataset-specific tasks
   @@RESULT_DIRS = {
     # Preprocessing
-    raw_reads: "01.raw_reads", trimmed_reads: "02.trimmed_reads",
-    read_quality: "03.read_quality", trimmed_fasta: "04.trimmed_fasta",
-    assembly: "05.assembly", cds: "06.cds",
+    raw_reads: '01.raw_reads',
+    trimmed_reads: '02.trimmed_reads',
+    read_quality: '03.read_quality',
+    trimmed_fasta: '04.trimmed_fasta',
+    assembly: '05.assembly',
+    cds: '06.cds',
     # Annotation
-    essential_genes: "07.annotation/01.function/01.essential",
-    ssu: "07.annotation/01.function/02.ssu",
-    mytaxa: "07.annotation/02.taxonomy/01.mytaxa",
-    mytaxa_scan: "07.annotation/03.qa/02.mytaxa_scan",
+    essential_genes: '07.annotation/01.function/01.essential',
+    ssu: '07.annotation/01.function/02.ssu',
+    mytaxa: '07.annotation/02.taxonomy/01.mytaxa',
+    mytaxa_scan: '07.annotation/03.qa/02.mytaxa_scan',
     # Distances (for single-species datasets)
-    distances: "09.distances", taxonomy: "09.distances/05.taxonomy",
+    distances: '09.distances',
+    taxonomy: '09.distances/05.taxonomy',
     # General statistics
-    stats: "90.stats"
+    stats: '90.stats'
   }
 
   ##
-  # Supported dataset types.
+  # Supported dataset types
   @@KNOWN_TYPES = {
-    genome: {description: 'The genome from an isolate', multi: false},
-    scgenome: {description: 'A Single-cell Amplified Genome (SAG)',
-      multi: false},
-    popgenome: {description: 'A Metagenome-Assembled Genome (MAG)',
-      :multi=>false},
-    metagenome: {description: 'A metagenome (excluding viromes)',
-      multi: true},
-    virome: {description: 'A viral metagenome', multi: true}
+    genome: {
+      description: 'The genome from an isolate', multi: false
+    },
+    scgenome: {
+      description: 'A Single-cell Amplified Genome (SAG)', multi: false
+    },
+    popgenome: {
+      description: 'A Metagenome-Assembled Genome (MAG)', multi: false
+    },
+    metagenome: {
+      description: 'A metagenome (excluding viromes)', multi: true
+    },
+    virome: {
+      description: 'A viral metagenome', multi: true
+    }
   }
 
   ##
-  # Returns an Array of tasks to be executed before project-wide tasks.
-  @@PREPROCESSING_TASKS = [:raw_reads, :trimmed_reads, :read_quality,
-    :trimmed_fasta, :assembly, :cds, :essential_genes, :ssu, :mytaxa,
-    :mytaxa_scan, :distances, :taxonomy, :stats]
-  
+  # Returns an Array of tasks to be executed before project-wide tasks
+  @@PREPROCESSING_TASKS = [
+    :raw_reads, :trimmed_reads, :read_quality, :trimmed_fasta,
+    :assembly, :cds, :essential_genes, :ssu, :mytaxa, :mytaxa_scan,
+    :distances, :taxonomy, :stats
+  ]
+
   ##
   # Tasks to be excluded from query datasets.
   @@EXCLUDE_NOREF_TASKS = [:mytaxa_scan, :taxonomy]
-  @@_EXCLUDE_NOREF_TASKS_H = Hash[@@EXCLUDE_NOREF_TASKS.map{ |i| [i,true] }]
-  
+  @@_EXCLUDE_NOREF_TASKS_H = Hash[@@EXCLUDE_NOREF_TASKS.map { |i| [i, true] }]
+
   ##
   # Tasks to be executed only in datasets that are not multi-organism. These
   # tasks are ignored for multi-organism datasets or for unknown types.
   @@ONLY_NONMULTI_TASKS = [:mytaxa_scan, :distances, :taxonomy]
-  @@_ONLY_NONMULTI_TASKS_H = Hash[@@ONLY_NONMULTI_TASKS.map{ |i| [i,true] }]
+  @@_ONLY_NONMULTI_TASKS_H = Hash[@@ONLY_NONMULTI_TASKS.map { |i| [i, true] }]
 
   ##
   # Tasks to be executed only in datasets that are multi-organism. These
   # tasks are ignored for single-organism datasets or for unknwon types.
   @@ONLY_MULTI_TASKS = [:mytaxa]
-  @@_ONLY_MULTI_TASKS_H = Hash[@@ONLY_MULTI_TASKS.map{ |i| [i,true] }]
-
-
+  @@_ONLY_MULTI_TASKS_H = Hash[@@ONLY_MULTI_TASKS.map { |i| [i, true] }]
 end
-

data/lib/miga/dataset/hooks.rb

@@ -1,4 +1,3 @@
-
 require 'miga/common/hooks'
 
 ##
@@ -20,7 +19,7 @@ require 'miga/common/hooks'
 # - run_cmd(cmd)
 # Internal hooks:
 # - _pull_result_hooks()
-module MiGA::Dataset::Hooks 
+module MiGA::Dataset::Hooks
   include MiGA::Common::Hooks
 
   ##
@@ -38,7 +37,8 @@ module MiGA::Dataset::Hooks
   ##
   # Clear metadata from run counts
   def hook_clear_run_counts(_hook_args, _event_args)
-    metadata.data.keys
+    metadata
+      .data.keys
       .select { |k| k.to_s =~ /^_try_/ }
       .each { |k| metadata[k] = nil }
     metadata[:_step] = nil
@@ -72,5 +72,4 @@ module MiGA::Dataset::Hooks
     pull_hook(:"on_result_ready_#{event_args.first}", *event_args)
     pull_hook(:on_preprocessing_ready) if done_preprocessing?
   end
-
 end

data/lib/miga/dataset/result.rb

@@ -65,7 +65,7 @@ module MiGA::Dataset::Result
   # Passes +save+ to #add_result.
   def next_preprocessing(save = false)
     first = first_preprocessing(save) or return nil
-    @@PREPROCESSING_TASKS[@@PREPROCESSING_TASKS.index(first) .. -1].find do |t|
+    @@PREPROCESSING_TASKS[@@PREPROCESSING_TASKS.index(first)..-1].find do |t|
       if ignore_task? t
         false
       elsif add_result(t, save).nil?
@@ -95,6 +95,7 @@ module MiGA::Dataset::Result
   def profile_advance(save = false)
     first_task = first_preprocessing(save)
     return Array.new(@@PREPROCESSING_TASKS.size, 0) if first_task.nil?
+
     adv = []
     state = 0
     next_task = next_preprocessing(save)
@@ -141,13 +142,16 @@ module MiGA::Dataset::Result
     r = get_result(:distances)
     ref = project.datasets.select(&:ref?).select(&:active?).map(&:name)
     return if r.nil?
+
     %i[haai_db aai_db ani_db].each do |db_type|
       db = r.file_path(db_type)
       next if db.nil? || !File.size?(db)
+
       sqlite_db = SQLite3::Database.new db
       table = db_type[-6..-4]
       val = sqlite_db.execute "select seq2 from #{table}"
       next if val.empty?
+
       (val.map(&:first) - ref).each do |extra|
         sqlite_db.execute "delete from #{table} where seq2=?", extra
       end
@@ -160,6 +164,7 @@ module MiGA::Dataset::Result
   # Add result type +:raw_reads+ at +base+ (no +_opts+ supported)
   def add_result_raw_reads(base, _opts)
     return nil unless result_files_exist?(base, '.1.fastq')
+
     add_files_to_ds_result(
       MiGA::Result.new("#{base}.json"), name,
       if result_files_exist?(base, '.2.fastq')
@@ -174,6 +179,7 @@ module MiGA::Dataset::Result
   # Add result type +:trimmed_reads+ at +base+ (no +_opts+ supported)
   def add_result_trimmed_reads(base, _opts)
     return nil unless result_files_exist?(base, '.1.clipped.fastq')
+
     add_files_to_ds_result(
       MiGA::Result.new("#{base}.json"), name,
       if result_files_exist?(base, '.2.clipped.fastq')
@@ -194,6 +200,7 @@ module MiGA::Dataset::Result
   # Add result type +:read_quality+ at +base+ (no +_opts+ supported)
   def add_result_read_quality(base, _opts)
     return nil unless result_files_exist?(base, %w[.solexaqa .fastqc])
+
     add_files_to_ds_result(
       MiGA::Result.new("#{base}.json"), name,
       solexaqa: '.solexaqa', fastqc: '.fastqc'
@@ -207,6 +214,7 @@ module MiGA::Dataset::Result
       result_files_exist?(base, '.CoupledReads.fa') ||
       result_files_exist?(base, '.SingleReads.fa')  ||
       result_files_exist?(base, %w[.1.fasta .2.fasta])
+
     add_files_to_ds_result(
       MiGA::Result.new("#{base}.json"), name,
       coupled: '.CoupledReads.fa',
@@ -221,6 +229,7 @@ module MiGA::Dataset::Result
   # +is_clean: Boolean+.
   def add_result_assembly(base, opts)
     return nil unless result_files_exist?(base, '.LargeContigs.fna')
+
     r = add_files_to_ds_result(
       MiGA::Result.new("#{base}.json"), name,
       largecontigs: '.LargeContigs.fna',
@@ -240,6 +249,7 @@ module MiGA::Dataset::Result
   # Add result type +:cds+ at +base+. Hash +opts+ supports +is_clean: Boolean+
   def add_result_cds(base, opts)
     return nil unless result_files_exist?(base, %w[.faa])
+
     r = add_files_to_ds_result(
       MiGA::Result.new("#{base}.json"), name,
       proteins: '.faa',
@@ -262,6 +272,7 @@ module MiGA::Dataset::Result
   # Add result type +:essential_genes+ at +base+ (no +_opts+ supported).
   def add_result_essential_genes(base, _opts)
     return nil unless result_files_exist?(base, %w[.ess.faa .ess .ess/log])
+
     add_files_to_ds_result(
       MiGA::Result.new("#{base}.json"), name,
       ess_genes: '.ess.faa',
@@ -276,6 +287,7 @@ module MiGA::Dataset::Result
   def add_result_ssu(base, opts)
     return MiGA::Result.new("#{base}.json") if result(:assembly).nil?
     return nil unless result_files_exist?(base, '.ssu.fa')
+
     r = add_files_to_ds_result(
       MiGA::Result.new("#{base}.json"), name,
       longest_ssu_gene: '.ssu.fa',
@@ -298,6 +310,7 @@ module MiGA::Dataset::Result
       return nil unless
         result_files_exist?(base, '.mytaxa') ||
         result_files_exist?(base, '.nomytaxa.txt')
+
       add_files_to_ds_result(
         MiGA::Result.new("#{base}.json"), name,
         mytaxa: '.mytaxa',
@@ -323,6 +336,7 @@ module MiGA::Dataset::Result
       return nil unless
         result_files_exist?(base, %w[.pdf .mytaxa]) ||
         result_files_exist?(base, '.nomytaxa.txt')
+
       add_files_to_ds_result(
         MiGA::Result.new("#{base}.json"), name,
         nomytaxa: '.nomytaxa.txt',
@@ -379,6 +393,7 @@ module MiGA::Dataset::Result
   def add_result_distances_ref(base)
     pref = File.dirname(base)
     return nil unless File.exist?("#{pref}/01.haai/#{name}.db")
+
     MiGA::Result.new("#{base}.json").tap do |r|
       r.add_files(
         haai_db: "01.haai/#{name}.db",
@@ -394,6 +409,7 @@ module MiGA::Dataset::Result
     return nil unless
       result_files_exist?(base, %w[.aai-medoids.tsv .aai.db]) ||
       result_files_exist?(base, %w[.ani-medoids.tsv .ani.db])
+
     add_files_to_ds_result(
       MiGA::Result.new("#{base}.json"), name,
       aai_medoids: '.aai-medoids.tsv',