miga-base 0.4.3.0 → 0.5.0.0

data/lib/miga/cli/action/classify_wf.rb

@@ -0,0 +1,97 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::ClassifyWf < MiGA::Cli::Action
+  require 'miga/cli/action/wf'
+  include MiGA::Cli::Action::Wf
+
+  def parse_cli
+    default_opts_for_wf
+    cli.defaults = {
+      download: false, summaries: true, pvalue: 0.05,
+      local: File.expand_path('.miga_db', ENV['MIGA_HOME'])
+    }
+    cli.parse do |opt|
+      opt.on(
+        '--download-db',
+        'Attempt to download the reference database (all default options)',
+        'It is recommended to use "miga get_db" separately instead'
+      ) { |v| cli[:download] = v }
+      opt.on(
+        '-n', '--database STRING',
+        'Name of the reference database to use',
+        'By default, the first locally listed database is used'
+      ) { |v| cli[:database] = v.to_sym }
+      opt.on(
+        '-p', '--p-value FLOAT', Float,
+        'Maximum p-value to transfer taxonomy',
+        "By default: #{cli[:pvalue]}"
+      ) { |v| cli[:pvalue] = v }
+      opt.on(
+        '-l', '--local-dir PATH',
+        "Local directory to store the database. By default: #{cli[:local]}"
+      ) { |v| cli[:local] = v }
+      opt.on(
+        '--db-path STRING',
+        'Path to the reference database to use, a fully indexed MiGA project',
+        'If defined, --local-dir and --database are ignored'
+      ) { |v| cli[:db_path] = v }
+      opt.on(
+        '--no-summaries',
+        'Do not generate intermediate step summaries'
+      ) { |v| cli[:summaries] = v }
+      opts_for_wf(opt, 'Input genome assemblies (nucleotides, FastA)')
+    end
+  end
+
+  def perform
+    # Input data
+    ref_db = reference_db
+    p_metadata = Hash[
+      %w[project_stats haai_distances aai_distances ani_distances clade_finding]
+        .map { |i| ["run_#{i}", false] }
+    ]
+    p_metadata[:ref_project] = ref_db.path
+    p_metadata[:tax_pvalue] = cli[:pvalue]
+    p = create_project(:assembly, p_metadata,
+      run_ssu: false, run_mytaxa_scan: false, run_distances: false)
+    # Run
+    run_daemon
+    summarize(%w[cds assembly essential_genes]) if cli[:summaries]
+    summarize(['taxonomy'])
+    cli.say "Summary: classification"
+    call_cli([
+      'ls', '-P', cli[:outdir], '-m', 'tax', '--tab',
+      '-o', File.expand_path('classification.tsv', cli[:outdir])
+    ])
+    cleanup
+  end
+  
+  private
+
+  def reference_db
+    cli.say "Locating reference database"
+    ref_db_path = cli[:db_path]
+    if ref_db_path.nil?
+      if cli[:download]
+        get_db_call  = ['get_db', '-l', cli[:local]]
+        get_db_call += ['-n', cli[:database]] unless cli[:database].nil?
+        call_cli(get_db_call)
+      end
+      if cli[:database].nil?
+        lm_f = File.expand_path('_local_manif.json', cli[:local])
+        unless File.size? lm_f
+          raise 'No locally listed databases, call "miga get_db" first'
+        end
+        cli[:database] = MiGA::Json.parse(lm_f)[:databases].keys.first
+      end
+      ref_db_path = File.expand_path(cli[:database].to_s, cli[:local])
+    end
+    ref_db = MiGA::Project.load(ref_db_path)
+    raise "Cannot locate reference database: #{ref_db_path}" if ref_db.nil?
+    cli.say "Reference database: #{ref_db.name}"
+    ref_db
+  end
+end

data/lib/miga/cli/action/daemon.rb

@@ -18,7 +18,7 @@ class MiGA::Cli::Action::Daemon < MiGA::Cli::Action
         run:     'Start the application and stay on top.',
         zap:     'Set the application to a stopped state.',
         status:  'Show status (PID) of application instances.'
-      }.each{ |k,v| opt.separator sprintf '    %*s%s', -33, k, v }
+      }.each { |k,v| opt.separator sprintf '    %*s%s', -33, k, v }
       opt.separator ''
 
       opt.separator 'MiGA options:'
@@ -27,45 +27,49 @@ class MiGA::Cli::Action::Daemon < MiGA::Cli::Action
         '--shutdown-when-done',
         'Exit the daemon when all processing is done',
         'Otherwise, it will stay idle awaiting for new data (default)'
-        ){ |v| cli[:shutdown_when_done] = v }
+      ) { |v| cli[:shutdown_when_done] = v }
       opt.on(
         '--latency INT',
         'Number of seconds the daemon will be sleeping'
-        ){ |v| cli[:latency]=v.to_i }
+      ) { |v| cli[:latency] = v.to_i }
       opt.on(
         '--max-jobs INT',
         'Maximum number of jobs to use simultaneously'
-        ){ |v| cli[:maxjobs]=v.to_i }
+      ) { |v| cli[:maxjobs] = v.to_i }
       opt.on(
         '--ppn INT',
         'Maximum number of cores to use in a single job'
-        ){ |v| cli[:ppn]=v.to_i }
+      ) { |v| cli[:ppn] = v.to_i }
+      opt.on(
+        '--json PATH',
+        'Path to a custom daemon definition in json format'
+      ) { |v| cli[:json] = v }
       cli.opt_common(opt)
 
       opt.separator 'Daemon options:'
       opt.on(
         '-t', '--ontop',
         'Stay on top (does not daemonize)'
-        ){ cli[:daemon_opts] << '-t' }
+      ) { cli[:daemon_opts] << '-t' }
       opt.on(
         '-f', '--force',
         'Force operation'
-        ){ cli[:daemon_opts] << '-f' }
+      ) { cli[:daemon_opts] << '-f' }
       opt.on(
         '-n', '--no_wait',
         'Do not wait for processes to stop'
-        ){ cli[:daemon_opts] << '-n' }
+      ) { cli[:daemon_opts] << '-n' }
       opt.on(
         '--shush',
         'Silence the daemon'
-        ){ cli[:daemon_opts] << '--shush' }
+      ) { cli[:daemon_opts] << '--shush' }
       opt.separator ''
     end
   end
 
   def perform
     p = cli.load_project
-    d = MiGA::Daemon.new(p)
+    d = MiGA::Daemon.new(p, cli[:json])
     [:latency, :maxjobs, :ppn, :shutdown_when_done].each do |k|
       d.runopts(k, cli[k]) unless cli[k].nil?
     end

data/lib/miga/cli/action/derep_wf.rb

@@ -0,0 +1,95 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::DerepWf < MiGA::Cli::Action
+  require 'miga/cli/action/wf'
+  include MiGA::Cli::Action::Wf
+
+  def parse_cli
+    default_opts_for_wf
+    cli.defaults = {
+      metric: :ani, threshold: 95.0, criterion: :quality,
+      summaries: true, collection: true
+    }
+    cli.parse do |opt|
+      opt.on(
+        '--aai',
+        'Use Average Amino Acid Identity (AAI) as genome similarity metric',
+        'By default: Use Average Nucleotide Identity (ANI)'
+      ) { cli[:metric] = :aai }
+      opt.on(
+        '--threshold FLOAT', Float,
+        "Metric threshold (%) to dereplicate. By default: #{cli[:threshold]}"
+      ) { |v| cli[:threshold] = v }
+      opt.on(
+        '--medoids',
+        'Use medoids as clade representatives',
+        'By default: Use genome with the highest quality'
+      ) { |v| cli[:criterion] = :medoids }
+      opt.on(
+        '--no-collection',
+        'Do not generate a dereplicated collection of assemblies'
+      ) { |v| cli[:collection] = v }
+      opt.on(
+        '--no-summaries',
+        'Do not generate intermediate step summaries'
+      ) { |v| cli[:summaries] = v }
+      opts_for_wf_distances(opt)
+      opts_for_wf(opt, 'Input genome assemblies (nucleotides, FastA)')
+    end
+  end
+
+  def perform
+    # Input data
+    p = create_project(:assembly,
+      { run_project_stats: false, run_clades: false,
+        gsp_metric: cli[:metric], :"gsp_#{cli[:metric]}" => cli[:threshold] },
+      { run_mytaxa_scan: false, run_ssu: false })
+    unless cli[:threshold] >= 0.0 && cli[:threshold] <= 100.0
+      raise "The threshold of identity must be in the range [0,100]"
+    end
+    # Run
+    run_daemon
+    dereplicate(p)
+    summarize(%w[cds assembly essential_genes]) if cli[:summaries]
+    cleanup
+  end
+
+  private
+
+  def dereplicate(p)
+    cli.say "Extracting genomospecies clades"
+    r = p.result(:clade_finding) or raise "Result unavailable: run failed"
+    c_f = r.file_path(:clades_gsp) or raise 'Result incomplete: run failed'
+    clades = File.readlines(c_f).map { |i| i.chomp.split("\t") }
+    rep = representatives(p)
+    File.open(File.expand_path('genomospecies.tsv', cli[:outdir]), 'w') do |fh|
+      fh.puts "Clade\tRepresentative\tMembers"
+      clades.each_with_index do |i, k|
+        fh.puts ["gsp_#{k+1}", rep[k], i.join(',')].join("\t")
+      end
+    end
+    if cli[:collection]
+      dir = File.expand_path('representatives', cli[:outdir])
+      FileUtils.mkdir_p(dir)
+      rep.each do |i|
+        f = p.dataset(i).result(:assembly).file_path(:largecontigs)
+        FileUtils.cp(f, dir)
+      end
+    end
+  end
+
+  def representatives(p)
+    cli.say "Identifying representatives"
+    f = File.expand_path('representatives.txt', cli[:outdir])
+    if cli[:criterion] == :medoids
+      FileUtils.cp(p.result(:clade_finding).file_path(:medoids_gsp), f)
+    else
+      src = File.expand_path('utils/representatives.rb', MiGA::MiGA.root_path)
+      `ruby '#{src}' '#{p.path}' | cut -f 2 > '#{f}'`
+    end
+    File.readlines(f).map(&:chomp)
+  end
+end

data/lib/miga/cli/action/doctor.rb

@@ -7,34 +7,32 @@ require 'sqlite3'
 class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
 
   def parse_cli
-    @@OPERATIONS.keys.each { |i| cli.defaults = {i => true} }
+    @@OPERATIONS.keys.each { |i| cli.defaults = { i => true } }
     cli.parse do |opt|
-      operation_n = Hash[@@OPERATIONS.map{ |k,v| [v[0], k] }]
+      operation_n = Hash[@@OPERATIONS.map { |k,v| [v[0], k] }]
       cli.opt_object(opt, [:project])
       opt.on(
         '--ignore TASK1,TASK2', Array,
         'Do not perform the task(s) listed. Available tasks are:',
-        * @@OPERATIONS.values.map{ |v| "~ #{v[0]}: #{v[1]}" }
-        ){ |v| v.map{ |i| cli[operation_n[i]] = false } }
+        * @@OPERATIONS.values.map { |v| "~ #{v[0]}: #{v[1]}" }
+      ) { |v| v.map { |i| cli[operation_n[i]] = false } }
       opt.on(
         '--only TASK',
         'Perform only the specified task (see --ignore)'
-        ) do |v|
-          op_k = @@OPERATIONS.find { |_, i| i[0] == v.downcase }.first
-          @@OPERATIONS.keys.each{ |i| cli[i] = false }
-          cli[op_k] = true
-        end
+      ) do |v|
+        op_k = @@OPERATIONS.find { |_, i| i[0] == v.downcase }.first
+        @@OPERATIONS.keys.each { |i| cli[i] = false }
+        cli[op_k] = true
+      end
     end
   end
 
   def check_sqlite3_database(db_file, metric)
-    begin
-      SQLite3::Database.new(db_file) do |conn|
-        conn.execute("select count(*) from #{metric}").first
-      end
-    rescue SQLite3::SQLException
-      yield
+    SQLite3::Database.new(db_file) do |conn|
+      conn.execute("select count(*) from #{metric}").first
     end
+  rescue SQLite3::SQLException
+    yield
   end
 
   def perform
@@ -48,6 +46,7 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
     db: ['databases', 'Check database files integrity'],
     dist: ['distances', 'Check distance summary tables'],
     files: ['files', 'Check for outdated files'],
+    cds: ['cds', 'Check for gzipped genes and proteins'],
     ess: ['essential-genes', 'Check for unarchived essential genes'],
     mts: ['mytaxa-scan', 'Check for unarchived MyTaxa scan'],
     start: ['start', 'Check for lingering .start files'],
@@ -84,41 +83,9 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
       res = p.result("#{dist}_distances")
       next if res.nil?
       cli.say "Checking #{dist} table for consistent datasets"
-      notok = {}
-      fix = {}
-      Zlib::GzipReader.open(res.file_path(:matrix)) do |fh|
-        lineno = 0
-        fh.each_line do |ln|
-          next if (lineno+=1)==1
-          r = ln.split("\t")
-          if [1,2].map{ |i| p.dataset(r[i]).nil? }.any?
-            [1,2].each do |i|
-              if p.dataset(r[i]).nil?
-                notok[r[i]] = true
-              else
-                fix[r[i]] = true
-              end
-            end
-          end
-        end
-      end
-
-      cli.say("- Fixing #{fix.size} datasets") unless fix.empty?
-      fix.keys.each do |d_n|
-        cli.say "  > Fixing #{d_n}."
-        p.dataset(d_n).cleanup_distances!
-      end
-
-      unless notok.empty?
-        cli.say '- Unregistered datasets detected: '
-        if notok.size <= 5
-          notok.keys.each { |i| cli.say "  > #{i}" }
-        else
-          cli.say "  > #{notok.size}, including #{notok.keys.first}"
-        end
-        cli.say '- Removing tables, recompute'
-        res.remove!
-      end
+      notok, fix = check_dist_eval(cli, p, res)
+      check_dist_fix(cli, p, fix)
+      check_dist_recompute(cli, res, notok)
     end
   end
 
@@ -142,6 +109,24 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
     end
   end
 
+  def check_cds(cli)
+    cli.say 'Looking for unzipped genes or proteins'
+    cli.load_project.each_dataset do |d|
+      res = d.result(:cds) or next
+      changed = false
+      [:genes, :proteins, :gff3, :gff2, :tab].each do |f|
+        file = res.file_path(f) or next
+        if file !~ /\.gz/
+          cli.say "  > Gzipping #{d.name} #{f}"
+          cmdo = `gzip -9 '#{file}'`.chomp
+          warn(cmdo) unless cmdo.empty?
+          changed = true
+        end
+      end
+      d.add_result(:cds, true, force: true) if changed
+    end
+  end
+
   def check_ess(cli)
     cli.say 'Looking for unarchived essential genes'
     cli.load_project.each_dataset do |d|
@@ -153,11 +138,10 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
         res.remove!
         next
       end
-      unless Dir["#{dir}/*.faa"].empty?
-        cli.say "  > Fixing #{d.name}"
-        cmdo = `cd '#{dir}' && tar -zcf proteins.tar.gz *.faa && rm *.faa`.chomp
-        warn(cmdo) unless cmdo.empty?
-      end
+      next if Dir["#{dir}/*.faa"].empty?
+      cli.say "  > Fixing #{d.name}"
+      cmdo = `cd '#{dir}' && tar -zcf proteins.tar.gz *.faa && rm *.faa`.chomp
+      warn(cmdo) unless cmdo.empty?
     end
   end
 
@@ -207,4 +191,48 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
     #cli.say 'o Checking for taxonomy/distances consistency'
     # TODO: Find 95%ANI clusters with entries from different species
   end
+
+  private
+
+  def check_dist_eval(cli, p, res)
+    notok = {}
+    fix = {}
+    Zlib::GzipReader.open(res.file_path(:matrix)) do |fh|
+      lineno = 0
+      fh.each_line do |ln|
+        next if (lineno += 1) == 1
+        r = ln.split("\t")
+        next unless [1, 2].map { |i| p.dataset(r[i]).nil? }.any?
+        [1, 2].each do |i|
+          if p.dataset(r[i]).nil?
+            notok[r[i]] = true
+          else
+            fix[r[i]] = true
+          end
+        end
+      end
+    end
+    [notok, fix]
+  end
+
+  def check_dist_fix(cli, p, fix)
+    return if fix.empty?
+    cli.say("- Fixing #{fix.size} datasets")
+    fix.keys.each do |d_n|
+      cli.say "  > Fixing #{d_n}."
+      p.dataset(d_n).cleanup_distances!
+    end
+  end
+
+  def check_dist_recompute(cli, p, notok)
+    return if notok.empty?
+    cli.say '- Unregistered datasets detected: '
+    if notok.size <= 5
+      notok.keys.each { |i| cli.say "  > #{i}" }
+    else
+      cli.say "  > #{notok.size}, including #{notok.keys.first}"
+    end
+    cli.say '- Removing tables, recompute'
+    res.remove!
+  end
 end