miga-base 0.3.0.0 → 0.3.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 141b39aed7636b3f2389837e5fd13348a8e11252
-  data.tar.gz: b4bcec9bc8a8fd8c15c670b6f0fae0925d36138e
+  metadata.gz: 20d84d5171a208f02d6278778c7967ff7d1ec3e5
+  data.tar.gz: 7f92d205fb8058af84b15c96d158049889efa0ca
 SHA512:
-  metadata.gz: c92ae5fb577d1e945d2a8be0f3d3dbe05aefd61094ee386f0bfb198237e83e487660419fa3cafb1cdc13aa25929ddd2022fab03530cac3ab4e927cb35481e82e
-  data.tar.gz: 6566cad346ff51d1e6b55c2e7bbaa3b5d766ae8292db0e829430237e2254dbb09a67d1ea63cdb9ea42365e47236b01b2af41e6e722aa153523bf39438d5e7ccb
+  metadata.gz: 3d78f1bf02243234a668e91cd2bafe608a7ca48fd41a11bcee77f6a7164d0222841a0b54ed8c3076f38d47268ff671ac128519d6433284f7fef8286d638c77b4
+  data.tar.gz: 21ae7b9c7514d542496197541a14ddc3d19dd424f97c04ec1a1913f34dadf93f4b25f76c7d54d2c3c79b4ccb67cf567bb552ea35c263dc62f0db4ab235757ffd

data/README.md

@@ -8,20 +8,33 @@
 # MiGA: Microbial Genomes Atlas
 
 **Important**: MiGA is under active development, and we currently cannot ensure
-any stability on the different interfaces. We'll be launching a Beta Testing
-program soon, with dedicated support for a small number of laboratories. If 
+any stability on the different interfaces. We're currently running a Beta
+Testing program with dedicated support for a small number of laboratories. If 
 you're interested, please [contact us][contact].
 
 For additional information on MiGA, visit:
+
+* [MiGA users list][mailing-list]:
+  Forum to discuss with other users and developers.
 * [MiGA manual][gitbook]: The definitive guide to MiGA.
 * [MiGA API docs][rubydoc]: Inner-workings of the `miga-base` gem.
 * [MiGA Web][miga-web]: MiGA on Rails!
-* [MiGA GUI][miga-gui]: MiGA on Shoes!
 
+# For the impatient
+
+If you're like us, you probably want to see sofware in action from the get go.
+You have two options:
+
+1. Get a peak on MiGA using [MiGA Online][miga-online].
+2. Install the [requirements](manual/part2/requirements.md) and follow the
+  [installation instructions](manual/part2/installation.md). Once you have MiGA
+  installed, you can [deploy some examples](manual/part4.md).
 
 # Authors
 
-Developed and maintained by [Luis M. Rodriguez-R][lrr].
+Developed and maintained by [Luis M. Rodriguez-R][lrr]. MiGA is the result of a
+collaboration between [Kostas Lab][kostas] at the Georgia Institute of
+Technology and [RDP][rdp] at Michigan State University.
 
 
 # License
@@ -29,8 +42,12 @@ Developed and maintained by [Luis M. Rodriguez-R][lrr].
 See [LICENSE](LICENSE).
 
 [lrr]: http://lmrodriguezr.github.io/
+[mailing-list]: https://groups.google.com/forum/#!forum/miga-users
 [gitbook]: https://miga.gitbooks.io/miga/content/
 [rubydoc]: http://www.rubydoc.info/github/bio-miga/miga
 [contact]: http://enve-omics.gatech.edu/node/7
 [miga-web]: https://github.com/bio-miga/miga-web
 [miga-gui]: https://github.com/bio-miga/miga-gui
+[miga-online]: http://microbial-genomes.org/
+[kostas]: http://enve-omics.gatech.edu/
+[rdp]: http://rdp.cme.msu.edu/

data/actions/init.rb

@@ -0,0 +1,258 @@
+#!/usr/bin/env ruby
+
+# @package MiGA
+# @license Artistic-2.0
+
+require "shellwords"
+
+o = {q:true, mytaxa:nil, config:File.expand_path(".miga_modules", ENV["HOME"]),
+  ask: false, auto:false, dtype: "bash"}
+opts = OptionParser.new do |opt|
+  opt_banner(opt)
+  opt.on("-c","--config PATH",
+    "Path to the Bash configuration file.",
+    "By default: #{o[:config]}."){ |v| o[:config] = v }
+  opt.on("--[no-]mytaxa",
+    "Should I try setting up MyTaxa? By default: interactive."
+    ){ |v| o[:mytaxa] = v }
+  opt.on("--daemon-type STRING",
+    "Type of daemon launcher, one of: bash, qsub, msub.",
+    "By default: #{o[:dtype]}."){ |v| o[:dtype]=v }
+  opt.on("--ask-all", "If set, asks for the location of all software.",
+    "By default, only the locations missing in PATH are requested"
+    ){ |v| o[:ask] = v }
+  opt.on("--auto", "If set, accepts all defaults as answers."
+    ){ |v| o[:auto] = v }
+  opt_common(opt, o)
+end.parse!
+$auto_answer = o[:auto]
+
+def ask_user(q, d=nil, ans=nil, force=false)
+  $stderr.print "#{q}#{" (#{ans.join(" / ")})" unless ans.nil?}" +
+    "#{"  [#{d}]" unless d.nil?} > "
+  if $auto_answer and not force
+    $stderr.puts ""
+  else
+    o = gets.chomp
+  end
+  o = d if o.nil? or o.empty?
+  unless ans.nil? or ans.include? o
+    $stderr.puts "Answer not recognized."
+    return ask_user(q, d, ans)
+  end
+  o
+end
+
+##=> Main <=
+
+miga = MiGA::MiGA.root_path
+$stderr.puts <<BANNER
+===[ Welcome to MiGA, the Microbial Genome Atlas ]===
+
+I'm the initialization script, and I'll sniff around your computer to
+make sure you have all the requirements for MiGA data processing.
+
+BANNER
+
+if ask_user(
+      "Would you like to see all the requirements before starting?",
+      "no", %w(yes no)) == "yes"
+  File.open(File.expand_path("utils/requirements.txt", miga), "r") do |fh|
+    fh.each_line{ |ln| $stderr.puts ln }
+  end
+end
+
+rc_path = File.expand_path(".miga_rc", ENV["HOME"])
+if File.exist? rc_path
+  if ask_user(
+        "I found a previous configuration. Do you want to continue?",
+        "yes", %w(yes no))=="no"
+    $stderr.puts "OK, see you soon!"
+    exit 0
+  end
+end
+rc_fh = File.open(rc_path, "w")
+rc_fh.puts <<BASH
+#!/bin/bash
+# `miga init` made this on #{Time.now}
+
+BASH
+
+# Check bash configuration file
+unless File.exist? o[:config]
+  o[:config] = ask_user(
+    "Is there a script I need to load at startup?", o[:config])
+end
+if File.exists? o[:config]
+  o[:config] = File.expand_path o[:config]
+  $stderr.puts "Found bash configuration script: #{o[:config]}."
+  rc_fh.puts "MIGA_STARTUP='#{o[:config]}'"
+  rc_fh.puts "source \"$MIGA_STARTUP\""
+end
+$stderr.puts ""
+
+# Check for software requirements
+$stderr.puts "Looking for requirements:"
+if o[:mytaxa].nil?
+  o[:mytaxa] = ask_user(
+        "Should I include MyTaxa modules?","yes",%w(yes no))=="yes"
+end
+rc_fh.puts "export MIGA_MYTAXA=\"no\"" unless o[:mytaxa]
+paths = {}
+File.open(File.expand_path("utils/requirements.txt", miga), "r") do |fh|
+  fh.each_line do |ln|
+    next if $. < 3
+    r = ln.chomp.split(/\t+/)
+    next if r[0] =~ /\(opt\)$/ and not o[:mytaxa]
+    $stderr.print "Testing #{r[0]}#{" (#{r[3]})" if r[3]}... "
+    path = nil
+    loop do
+      d_path = File.dirname(`which "#{r[1]}"`)
+      if o[:ask] or d_path=="."
+        path = ask_user("Where can I find it?", d_path, nil, true) 
+      else
+        path = d_path
+        $stderr.puts path
+      end
+      if File.executable? File.expand_path(r[1], path)
+        if d_path != path
+          rc_fh.puts "MIGA_PATH=\"#{path}:$MIGA_PATH\" # #{r[1]}"
+        end
+        break
+      end
+      $stderr.print "I cannot find #{r[1]}. "
+    end
+    paths[r[1]] = File.expand_path(r[1], path).shellescape
+  end
+end
+rc_fh.puts "export PATH=\"$MIGA_PATH$PATH\""
+$stderr.puts ""
+
+# Check for other files
+if o[:mytaxa]
+  $stderr.puts "Looking for MyTaxa databases:"
+  mt = File.dirname paths["MyTaxa"]
+  $stderr.print "Looking for scores... "
+  unless Dir.exist? File.expand_path("db", mt)
+    $stderr.puts "no.\nExecute 'python #{mt}/utils/download_db.py'."
+    exit 1
+  end
+  $stderr.puts "yes."
+  $stderr.print "Looking for diamond db... "
+  unless File.exist? File.expand_path("AllGenomes.faa.dmnd", mt)
+    $stderr.puts "no.\nDownload " +
+      "'http://enve-omics.ce.gatech.edu/data/public_mytaxa/" +
+      "AllGenomes.faa.dmnd' into #{mt}."
+    exit 1
+  end
+  $stderr.puts ""
+end
+
+# Check for R packages
+$stderr.puts "Looking for R packages:"
+%w(enveomics.R ape phangorn phytools cluster vegan).each do |pkg|
+  $stderr.print "Testing #{pkg}... "
+  `echo "library('#{pkg}')" | #{paths["R"].shellescape} --vanilla -q 2>&1`
+  if $?.success?
+    $stderr.puts "yes."
+  else
+    $stderr.puts "no, installing."
+    $stderr.print "" +
+      `echo "install.packages('#{pkg}', repos='http://cran.rstudio.com/')" \
+            | #{paths["R"].shellescape} --vanilla -q 2>&1`
+    `echo "library('#{pkg}')" | #{paths["R"].shellescape} --vanilla -q 2>&1`
+    raise "Unable to auto-install R package #{pkg}." unless $?.success?
+  end
+end
+$stderr.puts ""
+
+# Check for Ruby gems
+$stderr.puts "Looking for Ruby gems:"
+%w(rest-client sqlite3 daemons json).each do |pkg|
+  $stderr.print "Testing #{pkg}... "
+  `#{paths["ruby"].shellescape} -r "#{pkg}" -e "" 2>/dev/null`
+  if $?.success?
+    $stderr.puts "yes."
+  else
+    $stderr.puts "no, installing."
+    # This hackey mess is meant to ensure the test and installation are done
+    # on the configuration Ruby, not on the Ruby currently executing the init
+    # action
+    $stderr.print `#{paths["ruby"].shellescape} \
+        -r rubygems -r rubygems/gem_runner \
+        -e "Gem::GemRunner.new.run %w(install --user #{pkg})" 2>&1`
+    raise "Unable to auto-install Ruby gem #{pkg}." unless $?.success?
+  end
+end
+$stderr.puts ""
+
+# Configure daemon
+$stderr.puts "Default daemon configuration:"
+v = {created:Time.now.to_s, updated:Time.now.to_s}
+v[:type] = ask_user("Please select the type of daemon you want to setup",
+          o[:dtype], %w(bash qsub msub))
+case v[:type]
+  when "bash"
+    v[:latency] = ask_user("How long should I sleep? (in seconds)","30").to_i
+    v[:maxjobs] = ask_user("How many jobs can I launch at once?", "6").to_i
+    v[:ppn]     = ask_user("How many CPUs can I use per job?", "2").to_i
+    $stderr.puts "Setting up internal daemon defaults."
+    $stderr.puts "If you don't understand this just leave default values:"
+    v[:cmd]     = ask_user(
+      "How should I launch tasks?\n  %1$s: script path, %2$s: variables, " +
+      "%3$d: CPUs, %4$s: log file, %5$s: task name.\n",
+      "%2$s '%1$s' > '%4$s' 2>&1")
+    v[:var]     = ask_user(
+      "How should I pass variables?\n  %1$s: keys, %2$s: values.\n",
+      "%1$s=%2$s")
+    v[:sep]     = ask_user("What should I use to separate variables?", " ")
+    v[:alive]   = ask_user(
+      "How can I know that a process is still alive?\n  %1$s: PID, " +
+      "output should be 1 for running and 0 for non-running.\n",
+      "ps -p '%1$s'|tail -n+2|wc -l")
+  else # [qm]sub
+    queue       = ask_user("What queue should I use?", nil, nil, true)
+    v[:latency] = ask_user("How long should I sleep? (in seconds)", "150").to_i
+    v[:maxjobs] = ask_user("How many jobs can I launch at once?", "300").to_i
+    v[:ppn]     = ask_user("How many CPUs can I use per job?", "4").to_i
+    $stderr.puts "Setting up internal daemon defaults."
+    $stderr.puts "If you don't understand this just leave default values:"
+    v[:cmd]     = ask_user(
+      "How should I launch tasks?\n  %1$s: script path, %2$s: variables, " +
+      "%3$d: CPUs, %4$d: log file, %5$s: task name.\n",
+      "#{v[:type]} -q '#{queue}' -v '%2$s' -l nodes=1:ppn=%3$d %1$s " +
+      "-j oe -o '%4$s' -N '%5$s' -l mem=9g -l walltime=12:00:00 | grep .")
+    v[:var]     = ask_user(
+      "How should I pass variables?\n  %1$s: keys, %2$s: values.\n",
+      "%1$s=%2$s")
+    v[:sep]     = ask_user("What should I use to separate variables?", ",")
+    if v[:type] == "qsub"
+      v[:alive] = ask_user(
+        "How can I know that a process is still alive?\n  %1$s: job id, " +
+        "output should be 1 for running and 0 for non-running.\n",
+        "qstat -f '%1$s'|grep ' job_state ='|perl -pe 's/.*= //'|grep '[^C]'" +
+        "|tail -n1|wc -l|awk '{print $1}'")
+    else
+      v[:alive] = ask_user(
+        "How can I know that a process is still alive?\n  %1$s: job id, " +
+        "output should be 1 for running and 0 for non-running.\n",
+        "checkjob '%1$s'|grep '^State:'|perl -pe 's/.*: //'" +
+        "|grep 'Deferred\\|Hold\\|Idle\\|Starting\\|Running\\|Blocked'"+
+        "|tail -n1|wc -l|awk '{print $1}'")
+    end
+end
+File.open(File.expand_path(".miga_daemon.json", ENV["HOME"]), "w") do |fh|
+  fh.puts JSON.pretty_generate(v)
+end
+$stderr.puts ""
+
+rc_fh.puts <<FOOT
+
+MIGA_CONFIG_VERSION='#{MiGA::MiGA.VERSION}'
+MIGA_CONFIG_LONGVERSION='#{MiGA::MiGA.LONG_VERSION}'
+MIGA_CONFIG_DATE='#{Time.now}'
+
+FOOT
+
+$stderr.puts "Configuration complete. MiGA is ready to work!"
+$stderr.puts ""

data/actions/run_local.rb

@@ -16,14 +16,13 @@ opts = OptionParser.new do |opt|
 end.parse!
 
 ##=> Main <=
-opts.parse!
 opt_require(o, project:"-P", name:"-r")
 
 $stderr.puts "Loading project." unless o[:q]
 p = MiGA::Project.load(o[:project])
 raise "Impossible to load project: #{o[:project]}" if p.nil?
 
-miga = File.expand_path("../..", __FILE__)
+miga = MiGA::MiGA.root_path
 cmd = ["PROJECT=#{p.path.shellescape}", "RUNTYPE=bash",
   "MIGA=#{miga.shellescape}", "CORES=#{o[:thr]}"]
 if o[:dataset].nil?

data/actions/test_taxonomy.rb

@@ -28,7 +28,10 @@ ds = p.dataset(o[:dataset])
 $stderr.puts "Finding closest relative." unless o[:q]
 cr = ds.closest_relatives(1)
 
-unless cr.empty?
+if cr.nil? or cr.empty?
+  raise "This action is not supported for the project or dataset." if cr.nil?
+  raise "No close relatives found."
+else
   $stderr.puts "Querying probability distributions." unless o[:q]
   cr = cr[0]
   puts "Closest relative: #{cr[0]} with AAI: #{cr[1]}."

data/bin/miga

@@ -30,8 +30,10 @@ $task_desc = {
     "project.",
   run_local: "Executes locally one step analysis producing the given result.",
   # System
+  init: "Initialize MiGA to process new projects.",
   daemon: "Controls the daemon of a MiGA project.",
   date: "Returns the current date in standard MiGA format.",
+  console: "Opens an IRB console with MiGA (alias: c).",
   # Taxonomy
   add_taxonomy: "Registers taxonomic information for datasets.",
   test_taxonomy: "Returns test of taxonomic distributions for query datasets.",
@@ -137,9 +139,14 @@ when "-V", "--long-version"
   puts MiGA::MiGA.LONG_VERSION
 when "-C", "--citation"
   puts MiGA::MiGA.CITATION
+when "console", "c"
+  require "irb"
+  require "irb/completion"
+  ARGV.shift
+  IRB.start
 when *execs
   $task = ARGV.shift.to_sym
-  ARGV << "-h" if ARGV.empty? and not [:date].include? $task
+  ARGV << "-h" if ARGV.empty? and not [:date, :init].include? $task
   begin
     load File.expand_path("../actions/#{$task}.rb", File.dirname(__FILE__))
   rescue => err

data/lib/miga/dataset.rb

@@ -38,13 +38,13 @@ class MiGA::Dataset < MiGA::MiGA
   def self.KNOWN_TYPES ; @@KNOWN_TYPES end
   @@KNOWN_TYPES = {
     genome: {description: "The genome from an isolate.", multi: false},
-    metagenome: {description: "A metagenome (excluding viromes).",
-      multi: true},
-    virome: {description: "A viral metagenome.", multi: true},
     scgenome: {description: "A Single-cell Genome Amplification (SGA).",
       multi: false},
     popgenome: {description: "A population genome (including " +
-      "metagenomic bins).", :multi=>false}
+      "metagenomic bins).", :multi=>false},
+    metagenome: {description: "A metagenome (excluding viromes).",
+      multi: true},
+    virome: {description: "A viral metagenome.", multi: true}
   }
 
   ##

data/lib/miga/dataset_result.rb

@@ -142,10 +142,11 @@ module MiGA::DatasetResult
     # Add result type +:mytaxa+ at +base+ (no +_opts+ supported).
     def add_result_mytaxa(base, _opts)
       if is_multi?
-        return nil unless result_files_exist?(base, ".mytaxa")
+        return nil unless result_files_exist?(base, ".mytaxa") or
+          result_files_exist?(base, ".nomytaxa.txt")
         r = MiGA::Result.new("#{base}.json")
         add_files_to_ds_result(r, name, mytaxa:".mytaxa", blast:".blast",
-          mytaxain:".mytaxain")
+          mytaxain:".mytaxain", nomytaxa:".nomytaxa.txt")
       else
         MiGA::Result.new("#{base}.json")
       end
@@ -156,11 +157,13 @@ module MiGA::DatasetResult
     def add_result_mytaxa_scan(base, _opts)
       if is_nonmulti?
         return nil unless
-          result_files_exist?(base, %w[.pdf .wintax .mytaxa .reg])
+          result_files_exist?(base, %w[.pdf .wintax .mytaxa .reg]) or
+          result_files_exist?(base, ".nomytaxa.txt")
         r = MiGA::Result.new("#{base}.json")
         add_files_to_ds_result(r, name, mytaxa:".mytaxa", wintax:".wintax",
           blast:".blast", mytaxain:".mytaxain", report:".pdf", regions:".reg",
-          gene_ids:".wintax.genes", region_ids:".wintax.regions")
+          gene_ids:".wintax.genes", region_ids:".wintax.regions",
+          nomytaxa:".nomytaxa.txt")
       else
         MiGA::Result.new("#{base}.json")
       end

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.3, 0, 0]
+  VERSION = [0.3, 0, 1]
   
   ##
   # Nickname for the current major.minor version.
@@ -18,7 +18,7 @@ module MiGA
   
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2017, 5, 29)
+  VERSION_DATE = Date.new(2017, 6, 6)
   
   ##
   # Reference of MiGA.

data/scripts/_distances_noref_nomulti.bash

@@ -58,7 +58,9 @@ fi
 
 CLASSIF="."
 [[ -e "$DATASET.$METRIC-medoids.tsv" ]] && rm "$DATASET.$METRIC-medoids.tsv"
-while [[ -e "$CLADES/$CLASSIF/miga-project.medoids" ]] ; do
+[[ -s "$CLADES/$CLASSIF/miga-project.medoids" ]] || \
+  touch "$DATASET.$METRIC-medoids.tsv" "${DATASET}.${METRIC}.db"
+while [[ -s "$CLADES/$CLASSIF/miga-project.medoids" ]] ; do
   MAX_VAL=0
   VAL_MED=""
   VAL_CLS=""