miga-base 0.4.3.0 → 0.5.0.0

data/lib/miga/cli/action/tax_test.rb

@@ -29,49 +29,56 @@ class MiGA::Cli::Action::TaxTest < MiGA::Cli::Action
       raise 'Action not supported for the project or dataset' if cr.nil?
       raise 'No close relatives found'
     else
-      cli.say 'Querying probability distributions'
-      cr = cr[0]
-      cli.puts "Closest relative: #{cr[0]} with AAI: #{cr[1]}."
-      if self[:ref_project]
-        if (ref = p.metadata[:ref_project]).nil?
-          raise '--ref-project requested but no reference project has been set'
-        end
-        if (q = MiGA::Project.load(ref)).nil?
-          raise '--ref-project requested but reference project doesn\'t exist'
-        end
-        cr_d = q.dataset(cr[0])
-      else
-        cr_d = p.dataset(cr[0])
-      end
-      tax = cr_d.metadata[:tax] unless cr_d.nil?
-      tax ||= {}
+      query_probability_distributions(d, cr[0])
+    end
+  end
 
-      if %w[intax both].include? cli[:test]
-        # Intax
-        r = TaxDist.aai_pvalues(cr[1], :intax).map do |k,v|
-          sig = ''
-          [0.5, 0.1, 0.05, 0.01].each{ |i| sig << '*' if v<i }
-          [Taxonomy.LONG_RANKS[k], (tax[k] || '?'), v, sig]
-        end
-        cli.puts ''
-        cli.puts 'Taxonomic classification'
-        cli.puts MiGA.tabulate(%w[Rank Taxonomy P-value Signif.], r)
-      end
+  private
 
-      if %w[novel both].include? cli[:test]
-        # Novel
-        r = TaxDist.aai_pvalues(cr[1], :novel).map do |k,v|
-          sig = ''
-          [0.5, 0.1, 0.05, 0.01].each{ |i| sig << '*' if v<i }
-          [Taxonomy.LONG_RANKS[k], v, sig]
-        end
-        cli.puts ''
-        cli.puts 'Taxonomic novelty'
-        cli.puts MiGA.tabulate(%w[Rank P-value Signif.], r)
+  def query_probability_distributions(d, cr)
+    cli.say 'Querying probability distributions'
+    tax = closest_relative_tax(cr)
+    if %w[intax both].include? cli[:test]
+      r = test_closest_relative(cr, tax, :intax)
+      cli.puts ''
+      cli.puts 'Taxonomic classification'
+      cli.puts MiGA.tabulate(%w[Rank Taxonomy P-value Signif.], r)
+    end
+    if %w[novel both].include? cli[:test]
+      r = test_closest_relative(cr, tax, :novel)
+      r.map! { |i| i.tap { |j| j.delete_at(1) } }
+      cli.puts ''
+      cli.puts 'Taxonomic novelty'
+      cli.puts MiGA.tabulate(%w[Rank P-value Signif.], r)
+    end
+    cli.puts ''
+    cli.puts 'Significance at p-value below: *0.5, **0.1, ***0.05, ****0.01.'
+  end
+
+  def closest_relative_tax(cr)
+    cli.puts "Closest relative: #{cr[0]} with AAI: #{cr[1]}."
+    p = cli.load_project
+    if cli[:ref_project]
+      if (ref = p.metadata[:ref_project]).nil?
+        raise '--ref-project requested but no reference project has been set'
       end
+      if (q = MiGA::Project.load(ref)).nil?
+        raise '--ref-project requested but reference project doesn\'t exist'
+      end
+      cr_d = q.dataset(cr[0])
+    else
+      cr_d = p.dataset(cr[0])
+    end
+    tax = cr_d.metadata[:tax] unless cr_d.nil?
+    tax ||= {}
+    tax
+  end
 
-      cli.puts ''
-      cli.puts 'Significance at p-value below: *0.5, **0.1, ***0.05, ****0.01.'
+  def test_closest_relative(cr, tax, test)
+    TaxDist.aai_pvalues(cr[1], test).map do |k,v|
+      sig = ''
+      [0.5, 0.1, 0.05, 0.01].each { |i| sig << '*' if v < i }
+      [Taxonomy.LONG_RANKS[k], (tax[k] || '?'), v, sig]
     end
   end
 end

data/lib/miga/cli/action/wf.rb

@@ -0,0 +1,178 @@
+# @package MiGA
+# @license Artistic-2.0
+
+##
+# Helper module for workflows
+module MiGA::Cli::Action::Wf
+  def default_opts_for_wf
+    cli.expect_files = true
+    cli.defaults = {
+      clean: false, regexp: MiGA::Cli.FILE_REGEXP,
+      project_type: :genomes, dataset_type: :popgenome,
+      ncbi_draft: true }
+  end
+
+  def opts_for_wf(opt, files_desc, params = {})
+    {
+      multi: false, cleanup: true, project_type: false, ncbi: true
+    }.each { |k, v| params[k] = v if params[k].nil? }
+    opt.on(
+      '-o', '--out_dir PATH',
+      '(Mandatory) Directory to be created with all output data'
+    ) { |v| cli[:outdir] = v }
+    opt.separator ''
+    opt.separator "    FILES...: #{files_desc}"
+    opt.separator ''
+    opt.separator 'Workflow Control Options'
+    if params[:ncbi]
+      opt.on(
+        '-T', '--ncbi-taxon STRING',
+        'Download all the genomes in NCBI classified as this taxon'
+      ) { |v| cli[:ncbi_taxon] = v }
+      opt.on(
+        '--no-draft',
+        'Only download complete genomes, not drafts'
+      ) { |v| cli[:ncbi_draft] = v }
+    end
+    if params[:cleanup]
+      opt.on(
+        '-c', '--clean',
+        'Clean all intermediate files after generating the reports'
+      ) { |v| cli[:clean] = v }
+    end
+    opt.on(
+      '-R', '--name-regexp REGEXP', Regexp,
+      'Regular expression indicating how to extract the name from the path',
+      "By default: '#{cli[:regexp]}'"
+    ) { |v| cli[:regexp] = v }
+    opt.on(
+      '--type STRING',
+      "Type of datasets. By default: #{cli[:dataset_type]}",
+      'Recognized types:',
+      *MiGA::Dataset.KNOWN_TYPES
+        .map do |k, v|
+          "~ #{k}: #{v[:description]}" unless !params[:multi] && v[:multi]
+        end.compact
+    ) { |v| cli[:dataset_type] = v.downcase.to_sym }
+    if params[:project_type]
+      opt.on(
+        '--project-type STRING',
+        "Type of project. By default: #{cli[:project_type]}",
+        'Recognized types:',
+        *MiGA::Project.KNOWN_TYPES
+          .map do |k, v|
+            "~ #{k}: #{v[:description]}" unless !params[:multi] && v[:multi]
+          end.compact
+      ) { |v| cli[:project_type] = v.downcase.to_sym }
+    end
+    opt.on(
+      '--daemon PATH',
+      'Use custom daemon configuration in JSON format',
+      'By default: ~/.miga_daemon.json'
+    ) { |v| cli[:daemon_json] = v }
+    opt.on(
+      '-j', '--jobs INT',
+      'Number of parallel jobs to execute',
+      'By default controlled by the daemon configuration (maxjobs)'
+    ) { |v| cli[:jobs] = v.to_i }
+    opt.on(
+      '-t', '--threads INT',
+      'Number of CPUs to use per job',
+      'By default controlled by the daemon configuration (ppn)'
+    ) { |v| cli[:threads] = v.to_i }
+  end
+
+  def opts_for_wf_distances(opt)
+    opt.on('--fast', 'Alias to: --aai-p diamond --ani-p fastani') do
+      cli[:aai_p] = 'diamond'
+      cli[:ani_p] = 'fastani'
+    end
+    opt.on(
+      '--haai-p STRING',
+      'hAAI search engine. One of: blast+ (default), blat, diamond, no'
+    ) { |v| cli[:haai_p] = v }
+    opt.on(
+      '--aai-p STRING',
+      'AAI search engine. One of: blast+ (default), blat, diamond'
+    ) { |v| cli[:aai_p] = v }
+    opt.on(
+      '--ani-p STRING',
+      'ANI search engine. One of: blast+ (default), blat, fastani'
+    ) { |v| cli[:ani_p] = v }
+  end
+
+  def create_project(stage, p_metadata = {}, d_metadata = {})
+    cli.ensure_par(
+      outdir: '-o',
+      project_type: '--project-type',
+      dataset_type: '--dataset-type')
+    # Create empty project
+    call_cli([
+      'new',
+      '-P', cli[:outdir],
+      '-t', cli[:project_type],
+      '-m', p_metadata.map{ |k,v| "#{k}=#{v}" }.join(',')
+    ])
+    # Download datasets
+    call_cli([
+      'ncbi_get',
+      '-P', cli[:outdir],
+      '-T', cli[:ncbi_taxon],
+      (cli[:ncbi_draft] ? '--all' : '--complete'),
+      '-m', d_metadata.map{ |k,v| "#{k}=#{v}" }.join(',')
+    ]) unless cli[:ncbi_taxon].nil?
+    # Add datasets
+    call_cli([
+      'add',
+      '-P', cli[:outdir],
+      '-t', cli[:dataset_type],
+      '-i', stage,
+      '-R', cli[:regexp],
+      '-m', d_metadata.map{ |k,v| "#{k}=#{v}" }.join(',')
+    ] + cli.files) unless cli.files.empty?
+    p = MiGA::Project.load(cli[:outdir])
+    raise "Impossible to create project: #{cli[:outdir]}" if p.nil?
+    [:haai_p, :aai_p, :ani_p].each do |i|
+      p.metadata[i] = cli[i] unless cli[i].nil?
+    end
+    p.save
+    p
+  end
+
+  def summarize(which = %w[cds assembly essential_genes ssu])
+    which.each do |r|
+      cli.say "Summary: #{r}"
+      call_cli([
+        'summary',
+        '-P', cli[:outdir],
+        '-r', r,
+        '-o', File.expand_path("#{r}.tsv", cli[:outdir]),
+        '--tab'
+      ])
+    end
+  end
+
+  def cleanup
+    return unless cli[:clean]
+    cli.say "Cleaning up intermediate files"
+    %w[data daemon metadata miga.project.json].each do |f|
+      FileUtils.rm_rf(File.expand_path(f, cli[:outdir]))
+    end
+  end
+
+  def call_cli(cmd)
+    cmd << '-v' if cli[:verbose]
+    MiGA::Cli.new(cmd.map(&:to_s)).launch
+  end
+
+  def run_daemon
+    cmd  = ['daemon', 'run', '-P', cli[:outdir], '--shutdown-when-done']
+    cmd += ['--json', cli[:daemon_json]] unless cli[:daemon_json].nil?
+    cmd += ['--max-jobs', cli[:jobs]] unless cli[:jobs].nil?
+    cmd += ['--ppn', cli[:threads]] unless cli[:threads].nil?
+    cwd = Dir.pwd
+    call_cli cmd
+    Dir.chdir(cwd)
+  end
+
+end

data/lib/miga/cli/base.rb

@@ -5,11 +5,18 @@ module MiGA::Cli::Base
 
   @@TASK_DESC = {
     generic:  'MiGA: The Microbial Genomes Atlas',
+    # Workflows
+    quality_wf:  'Evaluate the quality of input genomes',
+    derep_wf:    'Dereplicate a collection of input genomes',
+    classify_wf: 'Classify input genomes against a reference database',
+    preproc_wf:  'Preprocess input genomes or metagenomes',
+    index_wf:    'Generate distance indexing of input genomes',
     # Projects
     new:      'Creates an empty MiGA project',
     about:    'Displays information about a MiGA project',
     plugins:  'Lists or (un)installs plugins in a MiGA project',
     doctor:   'Performs consistency checks on a MiGA project',
+    get_db:   'Downloads a pre-indexed database',
     # Datasets
     add:      'Creates a dataset in a MiGA project',
     get:      'Downloads a dataset from public databases into a MiGA project',
@@ -43,6 +50,7 @@ module MiGA::Cli::Base
     # Projects
     create_project: :new,
     project_info: :about,
+    download: :get_db,
     # Datasets
     create_dataset: :add,
     download_dataset: :get,
@@ -76,6 +84,8 @@ module MiGA::Cli::Base
 
   @@EXECS = @@TASK_DESC.keys
 
+  @@FILE_REGEXP = %r{^(?:.*/)?(.+?)(?:\..*(?:[12]|Reads|Contigs))?(?:\.f[nastq]+)?$}i
+
 end
 
 class MiGA::Cli < MiGA::MiGA
@@ -86,5 +96,6 @@ class MiGA::Cli < MiGA::MiGA
     def TASK_DESC; @@TASK_DESC end
     def TASK_ALIAS; @@TASK_ALIAS end
     def EXECS; @@EXECS end
+    def FILE_REGEXP; @@FILE_REGEXP end
   end
 end

data/lib/miga/cli/objects_helper.rb

@@ -0,0 +1,88 @@
+# @package MiGA
+# @license Artistic-2.0
+
+module MiGA::Cli::ObjectsHelper 
+  ##
+  # Get the project defined in the CLI by parameter +name+ and +flag+
+  def load_project(name = :project, flag = '-P')
+    return @objects[name] unless @objects[name].nil?
+    ensure_par(name => flag)
+    say "Loading project: #{self[name]}"
+    @objects[name] = MiGA::Project.load(self[name])
+    raise "Cannot load project: #{self[name]}" if @objects[name].nil?
+    @objects[name]
+  end
+
+  ##
+  # Load the dataset defined in the CLI
+  # If +silent=true+, it allows failures silently
+  def load_dataset(silent = false)
+    return @objects[:dataset] unless @objects[:dataset].nil?
+    ensure_par(dataset: '-D')
+    @objects[:dataset] = load_project.dataset(self[:dataset])
+    if !silent && @objects[:dataset].nil?
+      raise "Cannot load dataset: #{self[:dataset]}"
+    end
+    return @objects[:dataset]
+  end
+
+  ##
+  # Load an a project or (if defined) a dataset
+  def load_project_or_dataset
+    self[:dataset].nil? ? load_project : load_dataset
+  end
+
+  ##
+  # Load and filter a list of datasets as requested in the CLI
+  # If +silent=true+, it allows failures silently
+  def load_and_filter_datasets(silent = false)
+    return @objects[:filtered_datasets] unless @objects[:filtered_datasets].nil?
+    say 'Listing datasets'
+    ds = self[:dataset].nil? ?
+      load_project.datasets : [load_dataset(silent)].compact
+    ds.select! { |d| d.is_ref? == self[:ref] } unless self[:ref].nil?
+    ds.select! { |d| d.is_active? == self[:active] } unless self[:active].nil?
+    ds.select! do |d|
+      self[:multi] ? d.is_multi? : d.is_nonmulti?
+    end unless self[:multi].nil?
+    ds.select! do |d|
+      (not d.metadata[:tax].nil?) && d.metadata[:tax].in?(self[:taxonomy])
+    end unless self[:taxonomy].nil?
+    ds = ds.values_at(self[:dataset_k]-1) unless self[:dataset_k].nil?
+    @objects[:filtered_datasets] = ds
+  end
+
+  def load_result
+    return @objects[:result] unless @objects[:result].nil?
+    ensure_par(result: '-r')
+    obj = load_project_or_dataset
+    if obj.class.RESULT_DIRS[self[:result]].nil?
+      klass = obj.class.to_s.gsub(/.*::/,'')
+      raise "Unsupported result for #{klass}: #{self[:result]}"
+    end
+    r = obj.add_result(self[:result], false)
+    raise "Cannot load result: #{self[:result]}" if r.nil?
+    @objects[:result] = r
+  end
+
+  def add_metadata(obj, cli = self)
+    cli[:metadata].split(',').each do |pair|
+      (k,v) = pair.split('=')
+      case v
+        when 'true';  v = true
+        when 'false'; v = false
+        when 'nil';   v = nil
+      end
+      if k == '_step'
+        obj.metadata["_try_#{v}"] ||= 0
+        obj.metadata["_try_#{v}"]  += 1
+      end
+      obj.metadata[k] = v
+    end unless cli[:metadata].nil?
+    [:type, :name, :user, :description, :comments].each do |k|
+      obj.metadata[k] = cli[k] unless cli[k].nil?
+    end
+    obj
+  end
+end
+

data/lib/miga/cli/opt_helper.rb

@@ -0,0 +1,160 @@
+# @package MiGA
+# @license Artistic-2.0
+
+module MiGA::Cli::OptHelper 
+  ##
+  # Send MiGA's banner to OptionParser +opt+
+  def banner(opt)
+    usage = "Usage: miga #{action.name}"
+    usage += ' {operation}' if expect_operation
+    usage += ' [options]'
+    usage += ' {FILES...}' if expect_files
+    opt.banner = "\n#{task_description}\n\n#{usage}\n"
+    opt.separator ''
+  end
+
+  ##
+  # Common options at the end of most actions, passed to OptionParser +opt+
+  # No action is performed if +#opt_common = false+ is passed
+  # Executes only once, unless +#opt_common = true+ is passed between calls
+  def opt_common(opt)
+    return unless @opt_common
+    if interactive
+      opt.on(
+        '--auto',
+        'Accept all defaults as answers'
+      ) { |v| self[:auto] = v }
+    end
+    opt.on(
+      '-v', '--verbose',
+      'Print additional information to STDERR'
+    ) { |v| self[:verbose] = v }
+    opt.on(
+      '-d', '--debug INT', Integer,
+      'Print debugging information to STDERR (1: debug, 2: trace)'
+    ) { |v| v > 1 ? MiGA::MiGA.DEBUG_TRACE_ON : MiGA::MiGA.DEBUG_ON }
+    opt.on(
+      '-h', '--help',
+      'Display this screen'
+    ) do
+      puts opt
+      exit
+    end
+    opt.separator ''
+    self.opt_common = false
+  end
+
+  ##
+  # Options to load an object passed to OptionParser +opt+, as determined
+  # by +what+ an Array with any combination of:
+  # - :project To require a project
+  # - :dataset To require a dataset
+  # - :dataset_opt To allow (optionally) a dataset
+  # - :dataset_type To allow (optionally) a type of dataset
+  # - :dataset_type_req To require a type of dataset
+  # - :project_type To allow (optionally) a type of project
+  # - :project_type_req To require a type of project
+  # - :result To require a type of project or dataset result
+  # - :result_dataset To require a type of dataset result
+  # - :result_project To require a type of project result
+  # The options :result, :result_dataset, and :result_project are mutually
+  # exclusive
+  def opt_object(opt, what = [:project, :dataset])
+    what.each do |w|
+      case w
+      when :project
+        opt.on(
+          '-P', '--project PATH',
+          '(Mandatory) Path to the project'
+        ) { |v| self[:project] = v }
+      when :dataset, :dataset_opt
+        opt.on(
+          '-D', '--dataset STRING',
+          (w == :dataset ? '(Mandatory) ' : '') + 'Name of the dataset'
+        ) { |v| self[:dataset] = v }
+      when :dataset_type, :dataset_type_req, :project_type, :project_type_req
+        obj = w.to_s.gsub(/_.*/, '')
+        klass = Object.const_get("MiGA::#{obj.capitalize}")
+        req = w.to_s =~ /_req$/ ? '(Mandatory) ' : ''
+        opt.on(
+          '-t', '--type STRING',
+          "#{req}Type of #{obj}. Recognized types include:",
+          *klass.KNOWN_TYPES.map { |k, v| "~ #{k}: #{v[:description]}" }
+        ) { |v| self[:type] = v.downcase.to_sym }
+      when :result
+        opt.on(
+          '-r', '--result STRING',
+          '(Mandatory) Name of the result',
+          'Recognized names for dataset-specific results include:',
+          *MiGA::Dataset.RESULT_DIRS.keys.map { |n| " ~ #{n}" },
+          'Recognized names for project-wide results include:',
+          *MiGA::Project.RESULT_DIRS.keys.map { |n| " ~ #{n}" }
+        ) { |v| self[:result] = v.downcase.to_sym }
+      when :result_dataset
+        opt.on(
+          '-r', '--result STRING',
+          '(Mandatory) Name of the result, one of:',
+          *MiGA::Dataset.RESULT_DIRS.keys.map { |n| " ~ #{n}" }
+        ) { |v| self[:result] = v.downcase.to_sym }
+      when :result_project
+        opt.on(
+          '-r', '--result STRING',
+          '(Mandatory) Name of the result, one of:',
+          *MiGA::Project.RESULT_DIRS.keys.map { |n| " ~ #{n}" }
+        ) { |v| self[:result] = v.downcase.to_sym }
+      else
+        raise "Internal error: Unrecognized option: #{w}"
+      end
+    end
+  end
+
+  ##
+  # Options to filter a list of datasets passed to OptionParser +opt+,
+  # as determined by +what+ an Array with any combination of:
+  # - :ref To filter by reference (--ref) or query (--no-ref)
+  # - :multi To filter by multiple (--multi) or single (--no-multi) species
+  # - :active To filter by active (--active) or inactive (--no-active)
+  # - :taxonomy To filter by taxonomy (--taxonomy)
+  # The "k-th" filter (--dataset-k) is always included
+  def opt_filter_datasets(opt, what = [:ref, :multi, :active, :taxonomy])
+    what.each do |w|
+      case w
+      when :ref
+        opt.on(
+          '--[no-]ref',
+          'Use only reference (or only non-reference) datasets'
+        ) { |v| self[:ref] = v }
+      when :multi
+        opt.on(
+          '--[no-]multi',
+          'Use only multi-species (or only single-species) datasets'
+        ) { |v| self[:multi] = v }
+      when :active
+        opt.on(
+          '--[no-]active',
+          'Use only active (or inactive) datasets'
+        ) { |v| self[:active] = v }
+      when :taxonomy
+        opt.on(
+          '-t', '--taxonomy RANK:TAXON',
+          'Filter by taxonomy'
+        ) { |v| self[:taxonomy] = MiGA::Taxonomy.new(v) }
+      else
+        raise "Internal error: Unrecognized option: #{w}"
+      end
+    end
+    opt.on(
+      '--dataset-k INTEGER', Integer,
+      'Use only the k-th dataset in the list'
+    ) { |v| self[:dataset_k] = v }
+  end
+
+  ##
+  # Add a flag (true/false) to the OptionParser +opt+ defined by
+  # +flag+ (without --) and +description+, and save it in the CLI as +sym+.
+  # If +sym+ is nil, +flag+ is used as Symbol
+  def opt_flag(opt, flag, description, sym = nil)
+    sym = flag.to_sym if sym.nil?
+    opt.on("--#{flag}", description) { |v| self[sym] = v }
+  end
+end