miga-base 0.4.1.0 → 0.4.2.0

data/lib/miga/cli/action/new.rb

@@ -0,0 +1,45 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::New < MiGA::Cli::Action
+
+  def parse_cli
+    cli.parse do |opt|
+      cli.opt_object(opt, [:project, :project_type_req])
+      opt.on(
+        '-n', '--name STRING',
+        'Name of the project'
+        ){ |v| cli[:name] = v }
+      opt.on(
+        '-d', '--description STRING',
+        'Description of the project'
+        ){ |v| cli[:description] = v }
+      opt.on(
+        '-c', '--comments STRING',
+        'Comments on the project'
+        ){ |v| cli[:comments] = v }
+      opt.on(
+        '-m', '--metadata STRING',
+        'Metadata as key-value pairs separated by = and delimited by comma',
+        'Values are saved as strings except for booleans (true / false) or nil'
+        ){ |v| cli[:metadata] = v }
+    end
+  end
+
+  def perform
+    cli.ensure_type(MiGA::Project)
+    cli.ensure_par(project: '-P')
+    unless File.exist? "#{ENV["HOME"]}/.miga_rc" and
+          File.exist? "#{ENV["HOME"]}/.miga_daemon.json"
+      raise "You must initialize MiGA before creating the first project.\n" +
+        'Please use "miga init".'
+    end
+    cli.say "Creating project: #{cli[:project]}"
+    raise 'Project already exists, aborting.' if Project.exist? cli[:project]
+    p = Project.new(cli[:project], false)
+    p = cli.add_metadata(p)
+    p.save
+  end
+end

data/lib/miga/cli/action/next_step.rb

@@ -0,0 +1,27 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::NextStep < MiGA::Cli::Action
+
+  def parse_cli
+    cli.parse do |opt|
+      cli.opt_object(opt, [:project, :dataset_opt])
+    end
+  end
+
+  def perform
+    p = cli.load_project
+    n = nil
+    if cli[:dataset].nil?
+      n = p.next_distances(false)
+      n ||= p.next_inclade(false)
+    else
+      d = cli.load_dataset
+      n = d.next_preprocessing if d.is_active?
+    end
+    n ||= '?'
+    cli.puts n
+  end
+end

data/lib/miga/cli/action/plugins.rb

@@ -0,0 +1,28 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::Plugins < MiGA::Cli::Action
+
+  def parse_cli
+    cli.parse do |opt|
+      cli.opt_object(opt, [:project])
+      opt.on(
+        '--install PATH',
+        'Install the specified plugin in the project'
+        ){ |v| cli[:install] = v }
+      opt.on(
+        '--uninstall PATH',
+        'Uninstall the specified plugin from the project'
+        ){ |v| cli[:uninstall] = v }
+    end
+  end
+
+  def perform
+    p = cli.load_project
+    p.install_plugin(cli[:install]) unless cli[:install].nil?
+    p.uninstall_plugin(cli[:uninstall]) unless cli[:uninstall].nil?
+    p.plugins.each { |i| cli.puts i }
+  end
+end

data/lib/miga/cli/action/rm.rb

@@ -0,0 +1,25 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::Rm < MiGA::Cli::Action
+
+  def parse_cli
+    cli.defaults = {remove: false}
+    cli.parse do |opt|
+      cli.opt_object(opt)
+      opt.on(
+        '-r', '--remove',
+        'Also remove all associated files',
+        'By default, only unlinks from metadata'
+        ){ |v| cli[:remove] = v }
+    end
+  end
+
+  def perform
+    d = cli.load_dataset
+    cli.load_project.unlink_dataset(d.name)
+    d.remove! if cli[:remove]
+  end
+end

data/lib/miga/cli/action/run.rb

@@ -0,0 +1,39 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+require 'shellwords'
+
+class MiGA::Cli::Action::Run < MiGA::Cli::Action
+
+  def parse_cli
+    cli.defaults = {try_load: false, thr: 1}
+    cli.parse do |opt|
+      cli.opt_object(opt, [:project, :dataset_opt, :result])
+      opt.on(
+        '-t', '--threads INT', Integer,
+        "Threads to use in the local run (by default: #{cli[:thr]})."
+        ){ |v| cli[:thr] = v }
+    end
+  end
+
+  def perform
+    virtual_task = false
+    miga = MiGA.root_path
+    p = cli.load_project
+    cmd = ["PROJECT=#{p.path.shellescape}", 'RUNTYPE=bash',
+      "MIGA=#{miga.shellescape}", "CORES=#{cli[:thr]}"]
+
+    obj = cli.load_project_or_dataset
+    klass = obj.class
+    virtual_task = true if [:p, :d].include? cli[:result]
+    cmd << "DATASET=#{obj.name.shellescape}" if obj.is_a? MiGA::Dataset
+
+    if klass.RESULT_DIRS[cli[:result]].nil? and not virtual_task
+      raise "Unsupported #{klass.to_s.gsub(/.*::/, '')} result: #{cli[:result]}."
+    end
+    cmd << MiGA.script_path(cli[:result], miga: miga, project: p).shellescape
+    pid = spawn cmd.join(' ')
+    Process.wait pid
+  end
+end

data/lib/miga/cli/action/stats.rb

@@ -0,0 +1,140 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::Stats < MiGA::Cli::Action
+
+  def parse_cli
+    cli.defaults = {try_load: false}
+    cli.parse do |opt|
+      cli.opt_object(opt, [:project, :dataset_opt, :result])
+      opt.on(
+        '--key STRING',
+        'Return only the value of the requested key'
+        ){ |v| cli[:key] = v }
+      opt.on(
+        '--compute-and-save',
+        'Compute and saves the statistics'
+        ){ |v| cli[:compute] = v }
+      opt.on(
+        '--try-load',
+        'Check if stat exists instead of computing on --compute-and-save'
+        ){ |v| cli[:try_load] = v }
+    end
+  end
+
+  def perform
+    cli[:compute] = false if cli[:try_load] and
+      (not r[:stats].nil?) and (not r[:stats].empty?)
+    r = cli.load_result
+    if cli[:compute]
+      cli.say 'Computing statistics'
+      stats = {}
+      case cli[:result]
+      when :raw_reads
+        if r[:files][:pair1].nil?
+          s = MiGA.seqs_length(r.file_path(:single), :fastq, gc: true)
+          stats = {
+            reads: s[:n],
+            length_average: [s[:avg], 'bp'],
+            length_standard_deviation: [s[:sd], 'bp'],
+            g_c_content: [s[:gc], '%']}
+        else
+          s1 = MiGA.seqs_length(r.file_path(:pair1), :fastq, gc: true)
+          s2 = MiGA.seqs_length(r.file_path(:pair2), :fastq, gc: true)
+          stats = {
+            read_pairs: s1[:n],
+            forward_length_average: [s1[:avg], 'bp'],
+            forward_length_standard_deviation: [s1[:sd], 'bp'],
+            forward_g_c_content: [s1[:gc], '%'],
+            reverse_length_average: [s2[:avg], 'bp'],
+            reverse_length_standard_deviation: [s2[:sd], 'bp'],
+            reverse_g_c_content: [s2[:gc], '%']}
+        end
+      when :trimmed_fasta
+        f = r[:files][:coupled].nil? ? r.file_path(:single) : r.file_path(:coupled)
+        s = MiGA.seqs_length(f, :fasta, gc: true)
+        stats = {
+          reads: s[:n],
+          length_average: [s[:avg], 'bp'],
+          length_standard_deviation: [s[:sd], 'bp'],
+          g_c_content: [s[:gc], '%']}
+      when :assembly
+        s = MiGA.seqs_length(r.file_path(:largecontigs), :fasta,
+          n50: true, gc: true)
+        stats = {
+          contigs: s[:n],
+          n50: [s[:n50], 'bp'],
+          total_length: [s[:tot], 'bp'],
+          g_c_content: [s[:gc], '%']}
+      when :cds
+        s = MiGA.seqs_length(r.file_path(:proteins), :fasta)
+        stats = {
+          predicted_proteins: s[:n],
+          average_length: [s[:avg], 'aa']}
+        asm = cli.load_dataset.add_result(:assembly, false)
+        unless asm.nil? or asm[:stats][:total_length].nil?
+          stats[:coding_density] =
+            [300.0 * s[:tot] / asm[:stats][:total_length][0], '%']
+        end
+      when :essential_genes
+        d = cli.load_dataset
+        if d.is_multi?
+          stats = {median_copies: 0, mean_copies: 0}
+          File.open(r.file_path(:report), 'r') do |fh|
+            fh.each_line do |ln|
+              if /^! (Mean|Median) number of copies per model: (.*)\./.match(ln)
+                stats["#{$1.downcase}_copies".to_sym] = $2.to_f
+              end
+            end
+          end
+        else
+          # Fix estimate for Archaea
+          if not d.metadata[:tax].nil? &&
+                d.metadata[:tax].in?(Taxonomy.new('d:Archaea')) &&
+                r.file_path(:bac_report).nil?
+            scr = "#{MiGA.root_path}/utils/arch-ess-genes.rb"
+            rep = r.file_path(:report)
+            $stderr.print `ruby '#{scr}' '#{rep}' '#{rep}.archaea'`
+            r.add_file(:bac_report, "#{d.name}.ess/log")
+            r.add_file(:report, "#{d.name}.ess/log.archaea")
+          end
+          # Extract/compute quality values
+          stats = {completeness: [0.0, '%'], contamination: [0.0, '%']}
+          File.open(r.file_path(:report), 'r') do |fh|
+            fh.each_line do |ln|
+              if /^! (Completeness|Contamination): (.*)%/.match(ln)
+                stats[$1.downcase.to_sym][0] = $2.to_f
+              end
+            end
+          end
+          stats[:quality] = stats[:completeness][0] - stats[:contamination][0] * 5
+          d.metadata[:quality] = case stats[:quality]
+            when 80..100 ; :excellent
+            when 50..80  ; :high
+            when 20..50  ; :intermediate
+            else         ; :low
+          end
+          d.save
+        end
+      else
+        stats = nil
+      end
+      unless stats.nil?
+        r[:stats] = stats
+        r.save
+      end
+    end
+
+    if cli[:key].nil?
+      r[:stats].each do |k,v|
+        cli.puts "#{k==:g_c_content ? 'G+C content' : k.to_s.unmiga_name.capitalize}: #{
+          v.is_a?(Array) ? v.join(' ') : v}."
+      end
+    else
+      v = r[:stats][cli[:key].downcase.miga_name.to_sym]
+      puts v.is_a?(Array) ? v.first : v
+    end
+  end
+end

data/lib/miga/cli/action/summary.rb

@@ -0,0 +1,49 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::Summary < MiGA::Cli::Action
+
+  def parse_cli
+    cli.defaults = {units: false, tabular: false}
+    cli.parse do |opt|
+      cli.opt_object(opt, [:project, :dataset_opt])
+      cli.opt_filter_datasets(opt)
+      cli.opt_object(opt, [:result_dataset])
+      opt.on(
+        '--tab',
+        'Return a tab-delimited table'
+        ){ |v| cli[:tabular] = v }
+      opt.on(
+        '--key STRING',
+        'Return only the value of the requested key'
+        ){ |v| cli[:key_md] = v }
+      opt.on(
+        '--with-units',
+        'Include units in each cell'
+        ){ |v| cli[:units] = v }
+    end
+  end
+
+  def perform
+    cli.ensure_par(result: '-r')
+    ds = cli.load_and_filter_datasets
+    cli.say 'Loading results'
+    stats = ds.map do |d|
+      r = d.add_result(cli[:result].to_sym, false)
+      s = r.nil? ? {} : r[:stats]
+      s.tap{ |i| i[:dataset] = d.name }
+    end
+    keys = cli[:key_md].nil? ? stats.map(&:keys).flatten.uniq :
+      [:dataset, cli[:key_md].downcase.miga_name.to_sym]
+    keys.delete :dataset
+    keys.unshift :dataset
+
+    table = cli[:units] ?
+      stats.map{ |s| keys.
+        map{ |k| s[k].is_a?(Array) ? s[k].map(&:to_s).join('') : s[k] } } :
+      stats.map{ |s| keys.map{ |k| s[k].is_a?(Array) ? s[k].first : s[k] } }
+    cli.puts MiGA.tabulate(keys, table, cli[:tabular])
+  end
+end

data/lib/miga/cli/action/tax_dist.rb

@@ -0,0 +1,102 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+require 'miga/tax_index'
+require 'zlib'
+require 'tmpdir'
+
+class MiGA::Cli::Action::TaxDist < MiGA::Cli::Action
+
+  def parse_cli
+    cli.parse do |opt|
+      cli.opt_object(opt, [:project])
+      cli.opt_filter_datasets(opt)
+      opt.on(
+        '-i', '--index FILE',
+        'Pre-calculated tax-index (in tabular format) to be used',
+        'If passed, dataset filtering arguments are ignored'
+        ){ |v| cli[:index] = v }
+    end
+  end
+
+  def cannid(a, b)
+    (a > b ? [b, a] : [a, b]).join('-')
+  end
+
+  def perform
+    p = cli.load_project
+    metric = p.is_clade? ? 'ani' : 'aai'
+    res_n  = "#{metric}_distances"
+    cli.say "Reading distances: 1-#{metric.upcase}"
+    res = p.result(res_n)
+    raise "#{res_n} not yet calculated" if res.nil?
+    matrix = res.file_path(:matrix)
+    raise "#{res_n} has no matrix" if matrix.nil?
+    dist = {}
+    mfh = (matrix =~ /\.gz$/) ?
+      Zlib::GzipReader.open(matrix) : File.open(matrix, 'r')
+    mfh.each_line do |ln|
+      next if mfh.lineno == 1
+      row = ln.chomp.split("\t")
+      dist[cannid(row[1], row[2])] = [row[3], row[5], row[6], 0, ['root:biota']]
+      cli.advance("Ln: #{mfh.lineno}") if (mfh.lineno % 1_000) == 0
+    end
+    cli.say "  Lines: #{mfh.lineno}"
+    mfh.close
+
+    Dir.mktmpdir do |dir|
+      if cli[:index].nil?
+        ds = cli.load_and_filter_datasets
+        ds.keep_if { |d| !d.metadata[:tax].nil? }
+
+        cli.say 'Indexing taxonomy'
+        tax_index = TaxIndex.new
+        ds.each { |d| tax_index << d }
+        tab = File.expand_path('index.tab', dir)
+        File.open(tab, 'w') { |fh| fh.print tax_index.to_tab }
+      else
+        tab = cli[:index]
+      end
+
+      cli.say 'Traversing taxonomy'
+      rank_i = 0
+      Taxonomy.KNOWN_RANKS.each do |rank|
+        cli.say "o #{rank}: "
+        rank_n = 0
+        rank_i += 1
+        in_rank = nil
+        ds_name = []
+        File.open(tab, 'r') do |fh|
+          fh.each_line do |ln|
+            if ln =~ /^ {#{(rank_i-1)*2}}\S+:\S+:/
+              in_rank = nil
+              ds_name = []
+            elsif ln =~ /^ {#{rank_i*2}}(#{rank}:(\S+)):/
+              in_rank = $2 == '?' ? nil : $1
+              ds_name = []
+            elsif ln =~ /^ *# (\S+)/ and not in_rank.nil?
+              ds_i = $1
+              ds_name << ds_i
+              ds_name.each do |ds_j|
+                k = cannid(ds_i, ds_j)
+                next if dist[k].nil?
+                rank_n += 1
+                dist[k][3] = rank_i
+                dist[k][4].unshift in_rank
+              end
+            end
+          end
+        end
+        cli.say "#{rank_n} pairs of datasets"
+      end
+    end
+
+    cli.say 'Generating report'
+    dist.keys.each do |k|
+      dist[k][5] = dist[k][4].reverse.join(' ')
+      dist[k][4] = dist[k][4].first
+      puts (k.split('-') + dist[k]).join("\t")
+    end
+  end
+end

data/lib/miga/cli/action/tax_index.rb

@@ -0,0 +1,47 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+require 'miga/tax_index'
+
+class MiGA::Cli::Action::TaxIndex < MiGA::Cli::Action
+
+  def parse_cli
+    cli.defaults = {format: :json}
+    cli.parse do |opt|
+      cli.opt_object(opt, [:project])
+      opt.on(
+        '-i', '--index PATH',
+        '(Mandatory) File to create with the index'
+        ){ |v| cli[:index] = v }
+      opt.on(
+        '-f', '--format STRING',
+        "Format of the index file, by default: #{cli[:format]}",
+        'Supported: json, tab.'
+        ){ |v| cli[:format] = v.downcase.to_sym }
+      cli.opt_filter_datasets(opt)
+    end
+  end
+
+  def perform
+    cli.ensure_par(index: '-i')
+    ds = cli.load_and_filter_datasets
+    ds.keep_if {|d| !d.metadata[:tax].nil? }
+
+    cli.say 'Indexing taxonomy'
+    tax_index = MiGA::TaxIndex.new
+    ds.each { |d| tax_index << d }
+
+    cli.say 'Saving index'
+    File.open(cli[:index], 'w') do |fh|
+      case cli[:format]
+      when :json
+        fh.print tax_index.to_json
+      when :tab
+        fh.print tax_index.to_tab
+      else
+        raise "Unsupported output format: #{cli[:format]}"
+      end
+    end
+  end
+end

data/lib/miga/cli/action/tax_set.rb

@@ -0,0 +1,59 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::TaxSet < MiGA::Cli::Action
+
+  def parse_cli
+    cli.parse do |opt|
+      cli.opt_object(opt, [:project, :dataset_opt])
+      opt.on(
+        '-s', '--tax-string STRING',
+        'String corresponding to the taxonomy of the dataset',
+        'A space-delimited set of \'rank:name\' pairs'
+        ){ |v| cli[:taxstring] = v }
+      opt.on('-t', '--tax-file PATH',
+        '(Mandatory unless -D and -s are provided)',
+        'Tab-delimited file containing datasets taxonomy',
+        'Each row corresponds to a datasets and each column to a rank',
+        'The first row must be a header with the rank names,',
+        'and the first column must contain dataset names'
+        ){ |v| cli[:taxfile] = v }
+    end
+  end
+
+  def perform
+    p = cli.load_project
+    if !cli[:taxfile].nil?
+      cli.say 'Reading tax-file and registering taxonomy'
+      tfh = File.open(cli[:taxfile], 'r')
+      header = nil
+      tfh.each_line do |ln|
+        next if ln =~ /^\s*?$/
+        r = ln.chomp.split(/\t/, -1)
+        dn = r.shift
+        if header.nil?
+          header = r
+          next
+        end
+        d = p.dataset(dn)
+        if d.nil?
+          warn "Impossible to find dataset at line #{$.}: #{dn}. Ignoring..."
+          next
+        end
+        d.metadata[:tax] = Taxonomy.new(r, header)
+        d.save
+        cli.say "o #{d.name} registered"
+      end
+      tfh.close
+    else
+      cli.ensure_par({dataset: '-D', taxstring: '-s'},
+        '%<flag>s is mandatory unless -t is provided')
+      cli.say 'Registering taxonomy'
+      d = cli.load_dataset
+      d.metadata[:tax] = Taxonomy.new(cli[:taxstring])
+      d.save
+    end
+  end
+end

data/lib/miga/cli/action/tax_test.rb

@@ -0,0 +1,77 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+require 'miga/tax_dist'
+
+class MiGA::Cli::Action::TaxTest < MiGA::Cli::Action
+
+  def parse_cli
+    cli.defaults = {test: 'both', ref_project: false}
+    cli.parse do |opt|
+      cli.opt_object(opt, [:project, :dataset])
+      opt.on(
+        '--ref-project',
+        'Use the taxonomy from the reference project, not the current project'
+        ){ |v| cli[:ref_project] = v }
+      opt.on(
+        '-t', '--test STRING',
+        'Test to perform. Supported values: intax, novel, both'
+        ){ |v| cli[:test] = v.downcase }
+    end
+  end
+
+  def perform
+    d = cli.load_dataset
+    cli.say 'Finding closest relative'
+    cr = d.closest_relatives(1, cli[:ref_project])
+    if cr.nil? or cr.empty?
+      raise 'Action not supported for the project or dataset' if cr.nil?
+      raise 'No close relatives found'
+    else
+      cli.say 'Querying probability distributions'
+      cr = cr[0]
+      cli.puts "Closest relative: #{cr[0]} with AAI: #{cr[1]}."
+      if self[:ref_project]
+        if (ref = p.metadata[:ref_project]).nil?
+          raise '--ref-project requested but no reference project has been set'
+        end
+        if (q = MiGA::Project.load(ref)).nil?
+          raise '--ref-project requested but reference project doesn\'t exist'
+        end
+        cr_d = q.dataset(cr[0])
+      else
+        cr_d = p.dataset(cr[0])
+      end
+      tax = cr_d.metadata[:tax] unless cr_d.nil?
+      tax ||= {}
+
+      if %w[intax both].include? cli[:test]
+        # Intax
+        r = TaxDist.aai_pvalues(cr[1], :intax).map do |k,v|
+          sig = ''
+          [0.5, 0.1, 0.05, 0.01].each{ |i| sig << '*' if v<i }
+          [Taxonomy.LONG_RANKS[k], (tax[k] || '?'), v, sig]
+        end
+        cli.puts ''
+        cli.puts 'Taxonomic classification'
+        cli.puts MiGA.tabulate(%w[Rank Taxonomy P-value Signif.], r)
+      end
+
+      if %w[novel both].include? cli[:test]
+        # Novel
+        r = TaxDist.aai_pvalues(cr[1], :novel).map do |k,v|
+          sig = ''
+          [0.5, 0.1, 0.05, 0.01].each{ |i| sig << '*' if v<i }
+          [Taxonomy.LONG_RANKS[k], v, sig]
+        end
+        cli.puts ''
+        cli.puts 'Taxonomic novelty'
+        cli.puts MiGA.tabulate(%w[Rank P-value Signif.], r)
+      end
+
+      cli.puts ''
+      cli.puts 'Significance at p-value below: *0.5, **0.1, ***0.05, ****0.01.'
+    end
+  end
+end