miga-base 0.4.1.0 → 0.4.2.0

data/lib/miga/cli/action/doctor.rb

@@ -0,0 +1,210 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+require 'sqlite3'
+
+class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
+
+  def parse_cli
+    @@OPERATIONS.keys.each { |i| cli.defaults = {i => true} }
+    cli.parse do |opt|
+      operation_n = Hash[@@OPERATIONS.map{ |k,v| [v[0], k] }]
+      cli.opt_object(opt, [:project])
+      opt.on(
+        '--ignore TASK1,TASK2', Array,
+        'Do not perform the task(s) listed. Available tasks are:',
+        * @@OPERATIONS.values.map{ |v| "~ #{v[0]}: #{v[1]}" }
+        ){ |v| v.map{ |i| cli[operation_n[i]] = false } }
+      opt.on(
+        '--only TASK',
+        'Perform only the specified task (see --ignore)'
+        ) do |v|
+          op_k = @@OPERATIONS.find { |_, i| i[0] == v.downcase }.first
+          @@OPERATIONS.keys.each{ |i| cli[i] = false }
+          cli[op_k] = true
+        end
+    end
+  end
+
+  def check_sqlite3_database(db_file, metric)
+    begin
+      SQLite3::Database.new(db_file) do |conn|
+        conn.execute("select count(*) from #{metric}").first
+      end
+    rescue SQLite3::SQLException
+      yield
+    end
+  end
+
+  def perform
+    p = cli.load_project
+    @@OPERATIONS.keys.each do |k|
+      send("check_#{k}", cli) if cli[k]
+    end
+  end
+
+  @@OPERATIONS = {
+    db: ['databases', 'Check database files integrity'],
+    dist: ['distances', 'Check distance summary tables'],
+    files: ['files', 'Check for outdated files'],
+    ess: ['essential-genes', 'Check for unarchived essential genes'],
+    mts: ['mytaxa-scan', 'Check for unarchived MyTaxa scan'],
+    start: ['start', 'Check for lingering .start files'],
+    tax: ['taxonomy', 'Check for taxonomy consistency (not yet implemented)']
+  }
+  class << self
+    def OPERATIONS
+      @@OPERATIONS
+    end
+  end
+
+  def check_db(cli)
+    cli.say 'Checking databases integrity'
+    cli.load_project.each_dataset do |d|
+      [:distances, :taxonomy].each do |r_key|
+        r = d.result(r_key) or next
+        {haai_db: :aai, aai_db: :aai, ani_db: :ani}.each do |db_key, metric|
+          db_file = r.file_path(db_key) or next
+          check_sqlite3_database(db_file, metric) do
+            cli.say(
+              "  > Removing #{db_key} #{r_key} table for #{d.name}")
+            [db_file, r.path(:done), r.path].each do |f|
+              File.unlink(f) if File.exist? f
+            end # each |f|
+          end # check_sqlite3_database
+        end # each |db_key, metric|
+      end # each |r_key|
+    end # each |d|
+  end
+
+  def check_dist(cli)
+    p = cli.load_project
+    [:ani, :aai].each do |dist|
+      res = p.result("#{dist}_distances")
+      next if res.nil?
+      cli.say "Checking #{dist} table for consistent datasets"
+      notok = {}
+      fix = {}
+      Zlib::GzipReader.open(res.file_path(:matrix)) do |fh|
+        lineno = 0
+        fh.each_line do |ln|
+          next if (lineno+=1)==1
+          r = ln.split("\t")
+          if [1,2].map{ |i| p.dataset(r[i]).nil? }.any?
+            [1,2].each do |i|
+              if p.dataset(r[i]).nil?
+                notok[r[i]] = true
+              else
+                fix[r[i]] = true
+              end
+            end
+          end
+        end
+      end
+
+      cli.say("- Fixing #{fix.size} datasets") unless fix.empty?
+      fix.keys.each do |d_n|
+        cli.say "  > Fixing #{d_n}."
+        p.dataset(d_n).cleanup_distances!
+      end
+
+      unless notok.empty?
+        cli.say '- Unregistered datasets detected: '
+        if notok.size <= 5
+          notok.keys.each { |i| cli.say "  > #{i}" }
+        else
+          cli.say "  > #{notok.size}, including #{notok.keys.first}"
+        end
+        cli.say '- Removing tables, recompute'
+        res.remove!
+      end
+    end
+  end
+
+  def check_files(cli)
+    cli.say 'Looking for outdated files in results'
+    p = cli.load_project
+    p.each_dataset do |d|
+      d.each_result do |r_k, r|
+        ok = true
+        r.each_file do |_f_sym, _f_rel, f_abs|
+          unless File.exist? f_abs
+            ok = false
+            break
+          end
+        end
+        unless ok
+          cli.say "  > Registering again #{d.name}:#{r_k}"
+          d.add_result(r_k, true, force: true)
+        end
+      end
+    end
+  end
+
+  def check_ess(cli)
+    cli.say 'Looking for unarchived essential genes'
+    cli.load_project.each_dataset do |d|
+      res = d.result(:essential_genes)
+      next if res.nil?
+      dir = res.file_path(:collection)
+      if dir.nil?
+        cli.say "  > Removing #{d.name}:essential_genes"
+        res.remove!
+        next
+      end
+      unless Dir["#{dir}/*.faa"].empty?
+        cli.say "  > Fixing #{d.name}"
+        cmdo = `cd '#{dir}' && tar -zcf proteins.tar.gz *.faa && rm *.faa`.chomp
+        warn(cmdo) unless cmdo.empty?
+      end
+    end
+  end
+
+  def check_mts(cli)
+    cli.say 'Looking for unarchived MyTaxa Scan runs'
+    cli.load_project.each_dataset do |d|
+      res = d.result(:mytaxa_scan)
+      next if res.nil?
+      dir = res.file_path(:regions)
+      fix = false
+      unless dir.nil?
+        if Dir.exist? dir
+          cmdo = `cd '#{dir}/..' \
+                && tar -zcf '#{d.name}.reg.tar.gz' '#{d.name}.reg' \
+                && rm -r '#{d.name}.reg'`.chomp
+          warn(cmdo) unless cmdo.empty?
+        end
+        fix = true
+      end
+      %w[blast mytaxain wintax gene_ids region_ids].each do |ext|
+        file = res.file_path(ext.to_sym)
+        unless file.nil?
+          FileUtils.rm(file) if File.exist? file
+          fix = true
+        end
+      end
+      if fix
+        cli.say "  > Fixing #{d.name}"
+        d.add_result(:mytaxa_scan, true, force: true)
+      end
+    end
+  end
+
+  def check_start(cli)
+    cli.say 'Looking for legacy .start files lingering'
+    cli.load_project.each_dataset do |d|
+      d.each_result do |r_k, r|
+        if File.exist? r.path(:start)
+          cli.say "  > Registering again #{d.name}:#{r_k}"
+          r.save
+        end
+      end
+    end
+  end
+
+  def check_tax(cli)
+    #cli.say 'o Checking for taxonomy/distances consistency'
+    # TODO: Find 95%ANI clusters with entries from different species
+  end
+end

data/lib/miga/cli/action/edit.rb

@@ -0,0 +1,24 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::Edit < MiGA::Cli::Action
+
+  def parse_cli
+    cli.parse do |opt|
+      cli.opt_object(opt, [:project, :dataset_opt])
+      opt.on(
+        '-m', '--metadata STRING',
+        'Metadata as key-value pairs separated by = and delimited by comma',
+        'Values are saved as strings except for booleans (true / false) or nil'
+        ){ |v| cli[:metadata] = v }
+    end
+  end
+
+  def perform
+    obj = cli.load_project_or_dataset
+    cli.add_metadata(obj)
+    obj.save
+  end
+end

data/lib/miga/cli/action/files.rb

@@ -0,0 +1,31 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::Files < MiGA::Cli::Action
+
+  def parse_cli
+    cli.defaults = {details: false, json: true}
+    cli.parse do |opt|
+      cli.opt_object(opt, [:project, :dataset_opt])
+      opt.on(
+        '-i', '--info',
+        'Print additional details for each file'
+        ){ |v| cli[:details] = v }
+      opt.on('--[no-]json',
+        'Include (or not) JSON files containing results metadata',
+        'JSON files are included by default'
+        ){ |v| cli[:json] = v }
+    end
+  end
+
+  def perform
+    cli.load_project_or_dataset.each_result do |sym, res|
+      cli.puts "#{ "#{sym}\tjson\t" if cli[:details] }#{res.path}" if cli[:json]
+      res.each_file do |k,f|
+        cli.puts "#{ "#{sym}\t#{k}\t" if cli[:details] }#{res.dir}/#{f}"
+      end
+    end
+  end
+end

data/lib/miga/cli/action/find.rb

@@ -0,0 +1,48 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::Find < MiGA::Cli::Action
+
+  def parse_cli
+    cli.defaults = {add: false, ref: true}
+    cli.parse do |opt|
+      cli.opt_object(opt, [:project, :dataset_type])
+      opt.on(
+        '-a', '--add',
+        'Register the datasets found',
+        'By default, only lists them (dry run)'
+        ){ |v| cli[:add] = v }
+      opt.on(
+        '-q', '--query',
+        'Register datasets as query'
+        ){ |v| cli[:ref] = !v }
+      opt.on(
+        '-u', '--user STRING',
+        'Owner of the dataset.'
+        ){ |v| cli[:user] = v }
+      opt.on(
+        '-m', '--metadata STRING',
+        'Metadata as key-value pairs separated by = and delimited by comma',
+        'Values are saved as strings except for booleans (true / false) or nil'
+        ){ |v| cli[:metadata] = v }
+    end
+  end
+
+  def perform
+    p = cli.load_project
+    ud = p.unregistered_datasets
+    ud.each do |dn|
+      cli.puts dn
+      if cli[:add]
+        cli.say "Registering: #{dn}"
+        d = Dataset.new(p, dn, cli[:ref])
+        d = add_metadata(d)
+        p.add_dataset(dn)
+        res = d.first_preprocessing(true)
+        cli.say "- #{res}"
+      end
+    end
+  end
+end

data/lib/miga/cli/action/generic.rb

@@ -0,0 +1,44 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::Generic < MiGA::Cli::Action
+
+  def parse_cli
+    cli.opt_common = false
+    cli.parse do |opt|
+      descriptions = cli.class.TASK_DESC.keep_if { |k,v| k != :generic }
+      opt.separator MiGA::MiGA.tabulate(
+        [:action, :description], descriptions).join("\n")
+      opt.separator ''
+      opt.separator 'generic options:'
+      opt.on(
+        '-h', '--help',
+        'Display this screen'
+        ){ puts opt ; exit }
+      opt.on(
+        '-v', '--version',
+        'Show MiGA version'
+        ){ puts MiGA::MiGA.VERSION ; exit }
+      opt.on(
+        '-V', '--long-version',
+        'Show complete MiGA version'
+        ){ |v| puts MiGA::MiGA.LONG_VERSION ; exit }
+      opt.on(
+        '-C', '--citation',
+        'How to cite MiGA'
+        ) {|v| puts MiGA::MiGA.CITATION ; exit }
+    end
+  end
+
+  def perform
+  end
+
+  def complete
+  end
+
+  def name
+    '{action}'
+  end
+end

data/lib/miga/cli/action/get.rb

@@ -0,0 +1,132 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+require 'miga/remote_dataset'
+
+class MiGA::Cli::Action::Get < MiGA::Cli::Action
+
+  def parse_cli
+    cli.defaults = {query: false, universe: :ncbi, db: :nuccore,
+      get_md: false, only_md: false}
+    cli.parse do |opt|
+      cli.opt_object(opt, [:project, :dataset, :dataset_type])
+      opt.on(
+        '-I', '--ids ID1,ID2,...', Array,
+        '(Mandatory unless -F) IDs in the remote database separated by commas'
+        ){ |v| cli[:ids] = v }
+      opt.on(
+        '-U', '--universe STRING',
+        "Universe of the remote database. By default: #{cli[:universe]}"
+        ){ |v| cli[:universe] = v.to_sym }
+      opt.on(
+        '--db STRING',
+        "Name of the remote database. By default: #{cli[:db]}"
+        ){ |v| cli[:db] = v.to_sym }
+      opt.on(
+        '-F', '--file PATH',
+        'Tab-delimited file (with header) listing the datasets to download',
+        'The long form of most options are supported as header (without --)',
+        'including: dataset, ids, universe, db, metadata',
+        'For flags without value (like query) use true/false',
+        'Unsupported values are: project, file, verbose, help, and debug'
+        ){ |v| cli[:file] = v }
+      opt.on(
+        '-q', '--query',
+        'Register the dataset as a query, not a reference dataset'
+        ){ |v| cli[:query] = v }
+      opt.on('--ignore-dup',
+        'Ignore datasets that already exist'
+        ){ |v| cli[:ignore_dup] = v }
+      opt.on(
+        '-d', '--description STRING',
+        'Description of the dataset'
+        ){ |v| cli[:description] = v }
+      opt.on(
+        '-c', '--comments STRING',
+        'Comments on the dataset'
+        ){ |v| cli[:comments] = v }
+      opt.on(
+        '-m', '--metadata STRING',
+        'Metadata as key-value pairs separated by = and delimited by comma',
+        'Values are saved as strings except for booleans (true / false) or nil'
+        ){ |v| cli[:metadata] = v }
+      opt.on(
+        '--get-metadata',
+        'Only download and update metadata for existing datasets'
+        ){ |v| cli[:get_md] = v }
+      opt.on(
+        '--only-metadata',
+        'Create datasets without input data but retrieve all metadata'
+        ){ |v| cli[:only_md] = v }
+      opt.on(
+        '--api-key STRING',
+        'API key for the given universe'
+        ){ |v| cli[:api_key] = v }
+    end
+  end
+
+  def perform
+    glob = [cli]
+    unless cli[:file].nil?
+      glob = []
+      File.open(cli[:file], 'r') do |fh|
+        h = nil
+        fh.each do |ln|
+          r = ln.chomp.split(/\t/)
+          if h.nil?
+             h = r
+          else
+            argv_i = [self.name]
+            h.each_with_index do |field, k|
+              case field.downcase
+              when *%w[query ignore-dup get-metadata only-metadata]
+                argv_i << "--#{field.downcase}" if r[k].downcase == 'true'
+              when *%w[project file verbose help debug]
+                raise "Unsupported header: #{field}"
+              else
+                argv_i += ["--#{field.downcase}", r[k]]
+              end
+            end
+            sub_cli = MiGA::Cli.new(argv_i)
+            sub_cli.defaults = cli.data
+            sub_cli.action.parse_cli
+            glob << sub_cli
+          end
+        end
+      end
+    end
+
+    p = cli.load_project
+    glob.each do |sub_cli|
+      sub_cli.ensure_par(dataset: '-D', ids: '-I')
+      unless sub_cli[:api_key].nil?
+        ENV["#{sub_cli[:universe].to_s.upcase}_API_KEY"] = sub_cli[:api_key]
+      end
+
+      sub_cli.say "Dataset: #{sub_cli[:dataset]}"
+      if sub_cli[:ignore_dup] && !sub_cli[:get_md]
+        next if Dataset.exist?(p, sub_cli[:dataset])
+      end
+
+      sub_cli.say 'Locating remote dataset'
+      rd = RemoteDataset.new(sub_cli[:ids], sub_cli[:db], sub_cli[:universe])
+
+      if sub_cli[:get_md]
+        sub_cli.say 'Updating dataset'
+        d = p.dataset(sub_cli[:dataset])
+        next if d.nil?
+        md = sub_cli.add_metadata(d).metadata.data
+        rd.update_metadata(d, md)
+      else
+        sub_cli.say 'Creating dataset'
+        dummy_d = Dataset.new(p, sub_cli[:dataset])
+        md = sub_cli.add_metadata(dummy_d).metadata.data
+        md[:metadata_only] = true if cli[:only_md]
+        dummy_d.remove!
+        rd.save_to(p, sub_cli[:dataset], !sub_cli[:query], md)
+        p.add_dataset(sub_cli[:dataset])
+      end
+    end
+  end
+end