miga-base 0.3.0.2 → 0.3.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 128dc675d7890680368567d6d20d4fc37f746852
-  data.tar.gz: f58156b654b1463670d48c613c552e55b14d1b18
+  metadata.gz: f402fe3633ac475e3c3d753d28131b78b5cab562
+  data.tar.gz: 5df9dd884c904d613ab34205da6f7d75f031237a
 SHA512:
-  metadata.gz: 9431dc3c5ce79996bdaf77d9a2bffe06cb32645a95ebbe397164de0706825c50c8da54fdd0574b4ba4123226cbbd3dd7a322a9a5d87d8dd40ced955c618f59ae
-  data.tar.gz: 24e4ba089124116c4a12b67df9be0930c11a60af8d70dac3a2525061ffddf195e20b4f130d94de391a474af7782222066f2a41552de5f078234af7f3d03f0972
+  metadata.gz: f6013b57eab9606b6b4130f92e06afb549b46392862e3b67c1ed5698d9a4c8eda8ca765047d4e548b49841723f6c67985a3635d28619fde85a5055fd481aa10b
+  data.tar.gz: bbe04db4291ca2c58cf22232b82740af5072f217bc66ac254a92d0783d74e26d762d4276dad80f9e8ca640d5d5f80b54c47b5d1ca5fa1ec4d85beb90ac057962

data/actions/{project_info.rb → about.rb}

@@ -10,7 +10,7 @@ OptionParser.new do |opt|
   opt.on("-p", "--processing",
     "Print information on processing advance."){ |v| o[:processing]=v }
   opt.on("-m", "--metadata STRING",
-    "Print name and metadata field only. If set, ignores -i."
+    "Print name and metadata field only."
     ){ |v| o[:datum]=v }
   opt_common(opt, o)
 end.parse!
@@ -24,7 +24,8 @@ p = MiGA::Project.load(o[:project])
 raise "Impossible to load project: #{o[:project]}" if p.nil?
 
 if not o[:datum].nil?
-  puts (p.metadata[ o[:datum] ] || "?")
+  v = p.metadata[ o[:datum] ]
+  puts v.nil? ? "?" : v
 elsif o[:processing]
   keys = MiGA::Project.DISTANCE_TASKS + MiGA::Project.INCLADE_TASKS
   puts MiGA::MiGA.tabulate([:task, :status], keys.map do |k|

data/actions/init.rb

@@ -109,7 +109,7 @@ File.open(File.expand_path("utils/requirements.txt", miga), "r") do |fh|
     loop do
       d_path = File.dirname(`which "#{r[1]}"`)
       if o[:ask] or d_path=="."
-        path = ask_user("Where can I find it?", d_path, nil, true) 
+        path = ask_user("Where can I find it?", d_path, nil, true)
       else
         path = d_path
         $stderr.puts path
@@ -205,11 +205,13 @@ case v[:type]
     v[:var]     = ask_user(
       "How should I pass variables?\n  %1$s: keys, %2$s: values.\n",
       "%1$s=%2$s")
-    v[:sep]     = ask_user("What should I use to separate variables?", " ")
+    v[:varsep]  = ask_user("What should I use to separate variables?", " ")
     v[:alive]   = ask_user(
       "How can I know that a process is still alive?\n  %1$s: PID, " +
       "output should be 1 for running and 0 for non-running.\n",
       "ps -p '%1$s'|tail -n+2|wc -l")
+    v[:kill]    = ask_user(
+      "How should I terminate tasks?\n  %s: process ID.", "kill -9 '%s'")
   else # [qm]sub
     queue       = ask_user("What queue should I use?", nil, nil, true)
     v[:latency] = ask_user("How long should I sleep? (in seconds)", "150").to_i
@@ -225,13 +227,15 @@ case v[:type]
     v[:var]     = ask_user(
       "How should I pass variables?\n  %1$s: keys, %2$s: values.\n",
       "%1$s=%2$s")
-    v[:sep]     = ask_user("What should I use to separate variables?", ",")
+    v[:varsep]  = ask_user("What should I use to separate variables?", ",")
     if v[:type] == "qsub"
       v[:alive] = ask_user(
         "How can I know that a process is still alive?\n  %1$s: job id, " +
         "output should be 1 for running and 0 for non-running.\n",
         "qstat -f '%1$s'|grep ' job_state ='|perl -pe 's/.*= //'|grep '[^C]'" +
         "|tail -n1|wc -l|awk '{print $1}'")
+    v[:kill]    = ask_user(
+      "How should I terminate tasks?\n  %s: process ID.", "qdel '%s'")
     else
       v[:alive] = ask_user(
         "How can I know that a process is still alive?\n  %1$s: job id, " +
@@ -239,6 +243,8 @@ case v[:type]
         "checkjob '%1$s'|grep '^State:'|perl -pe 's/.*: //'" +
         "|grep 'Deferred\\|Hold\\|Idle\\|Starting\\|Running\\|Blocked'"+
         "|tail -n1|wc -l|awk '{print $1}'")
+    v[:kill]    = ask_user(
+      "How should I terminate tasks?\n  %s: process ID.", "canceljob '%s'")
     end
 end
 File.open(File.expand_path(".miga_daemon.json", ENV["HOME"]), "w") do |fh|

data/bin/miga

@@ -12,35 +12,65 @@ require "miga"
 
 $task_desc = {
   # Projects
-  create_project: "Creates an empty MiGA project.",
-  project_info: "Displays information about a MiGA project.",
+  new: "Creates an empty MiGA project.",
+  about: "Displays information about a MiGA project.",
   plugins: "Lists or (un)installs plugins in a MiGA project.",
   # Datasets
-  create_dataset: "Creates an empty dataset in a pre-existing MiGA project.",
-  download_dataset: "Creates an empty dataset in a pre-existing MiGA project.",
-  unlink_dataset: "Removes a dataset from an MiGA project.",
-  find_datasets: "Finds unregistered datasets based on result files.",
-  import_datasets: "Link datasets (including results) from one project to "+
-    "another.",
-  list_datasets: "Lists all registered datasets in an MiGA project.",
+  add: "Creates an empty dataset in a pre-existing MiGA project.",
+  get: "Creates an empty dataset in a pre-existing MiGA project.",
+  rm: "Removes a dataset from an MiGA project.",
+  find: "Finds unregistered datasets based on result files.",
+  ln: "Link datasets (including results) from one project to another.",
+  ls: "Lists all registered datasets in an MiGA project.",
   # Results
   add_result: "Registers a result.",
-  result_stats: "Extracts statistics for the given result.",
-  list_files: "Lists all registered files from the results of a dataset or a "+
-    "project.",
-  run_local: "Executes locally one step analysis producing the given result.",
+  stats: "Extracts statistics for the given result.",
+  files: "Lists all registered files from the results of a dataset or project.",
+  run: "Executes locally one step analysis producing the given result.",
   # System
   init: "Initialize MiGA to process new projects.",
   daemon: "Controls the daemon of a MiGA project.",
   date: "Returns the current date in standard MiGA format.",
-  console: "Opens an IRB console with MiGA (alias: c).",
+  console: "Opens an IRB console with MiGA.",
   # Taxonomy
-  add_taxonomy: "Registers taxonomic information for datasets.",
-  test_taxonomy: "Returns test of taxonomic distributions for query datasets.",
-  index_taxonomy: "Creates a taxonomy-indexed list of the datasets.",
-  tax_distributions: "Estimates distributions of distance by taxonomy.",
+  tax_set: "Registers taxonomic information for datasets.",
+  tax_test: "Returns test of taxonomic distributions for query datasets.",
+  tax_index: "Creates a taxonomy-indexed list of the datasets.",
+  tax_dist: "Estimates distributions of distance by taxonomy.",
 }
 
+$task_alias = {
+  # Projects
+  create_project: :new,
+  project_info: :about,
+  # Datasets
+  create_dataset: :add,
+  download_dataset: :get,
+  unlink_dataset: :rm,
+  find_datasets: :find,
+  import_datasets: :ln,
+  list_datasets: :ls,
+  # Results
+  result_stats: :stats,
+  list_files: :files,
+  run_local: :run,
+  # System
+  c: :console,
+  # Taxonomy
+  add_taxonomy: :tax_set,
+  test_taxonomy: :tax_test,
+  index_taxonomy: :tax_index,
+  tax_distributions: :tax_dist,
+}
+
+$task_alias.each do |nick, task|
+  $task_desc[task] = (
+    ($task_desc[task] =~ /\(alias: .*\)\./) ?
+        $task_desc[task].sub(/\)\.$/, ", #{nick}).") :
+        $task_desc[task].sub(/\.$/, " (alias: #{nick}).")
+  )
+end
+
 ##=> Functions <=
 
 # OptParse banner
@@ -142,7 +172,7 @@ def add_metadata(o, obj)
       when "nil";   v = nil
     end
     obj.metadata[k] = v
-  end
+  end unless o[:metadata].nil?
   [:type, :name, :user, :description, :comments].each do |k|
     obj.metadata[k] = o[k] unless o[k].nil?
   end
@@ -153,14 +183,17 @@ end
 
 execs = $task_desc.keys.map{ |k| k.to_s }
 
-case ARGV[0]
+ARGV[0] = $task_alias[ARGV[0].to_sym] unless
+  ARGV[0].nil? or $task_alias[ARGV[0].to_sym].nil?
+
+case ARGV[0].to_s
 when "-v", "--version"
   puts MiGA::MiGA.VERSION
 when "-V", "--long-version"
   puts MiGA::MiGA.LONG_VERSION
 when "-C", "--citation"
   puts MiGA::MiGA.CITATION
-when "console", "c"
+when "console"
   require "irb"
   require "irb/completion"
   ARGV.shift

data/lib/miga/daemon.rb

@@ -17,6 +17,10 @@ class MiGA::Daemon < MiGA::MiGA
     return nil unless File.exist? f
     DateTime.parse(File.read(f))
   end
+
+  # Shutdown all spawned daemons before exit.
+  $_MIGA_DAEMON_LAIR = []
+  END { $_MIGA_DAEMON_LAIR.each{ |d| d.terminate } }
   
   # MiGA::Project in which the daemon is running.
   attr_reader :project
@@ -33,6 +37,7 @@ class MiGA::Daemon < MiGA::MiGA
   # Initialize an unactive daemon for the MiGA::Project +project+. See #daemon
   # to wake the daemon.
   def initialize(project)
+    $_MIGA_DAEMON_LAIR << self
     @project = project
     @runopts = JSON.parse(
       File.read(File.expand_path("daemon/daemon.json", project.path)),
@@ -68,6 +73,13 @@ class MiGA::Daemon < MiGA::MiGA
       raise "Daemon's #{k} cannot be set to zero." if !force and v==0
       @runopts[k] = v
     end
+    if k==:kill and v.nil?
+      case @runopts[:type].to_s
+        when "bash" ; return "kill -9 '%s'"
+        when "qsub" ; return "qdel '%s'"
+        else        ; return "canceljob '%s'"
+      end
+    end
     @runopts[k]
   end
 
@@ -117,13 +129,22 @@ class MiGA::Daemon < MiGA::MiGA
   end
 
   ##
-  # Tell the world that you're alive
+  # Tell the world that you're alive.
   def declare_alive
     f = File.open(File.expand_path("daemon/alive", project.path), "w")
     f.print Time.now.to_s
     f.close
   end
 
+  ##
+  # Report status in a JSON file.
+  def report_status
+    f = File.open(File.expand_path("daemon/status.json", project.path), "w")
+    f.print JSON.pretty_generate(
+      jobs_running:@jobs_running, jobs_to_run:@jobs_to_run)
+    f.close
+  end
+  
   ##
   # Traverse datasets
   def check_datasets
@@ -240,8 +261,10 @@ class MiGA::Daemon < MiGA::MiGA
       @loop_i = 0
       purge!
     end
+    report_status
     sleep(latency)
     if shutdown_when_done? and jobs_running.size+jobs_to_run.size == 0
+      say "Nothing else to do, shutting down."
       return false
     end
     true
@@ -253,6 +276,20 @@ class MiGA::Daemon < MiGA::MiGA
     print "[#{Time.new.inspect}] ", *opts, "\n"
   end
 
+  ##
+  # Terminates a daemon.
+  def terminate
+    say "Terminating daemon..."
+    report_status
+    k = runopts(:kill)
+    @jobs_running.each do |i|
+      `#{k % i[:pid]}`
+      puts "Terminating pid:#{i[:pid]} for #{i[:task_name]}"
+    end
+    f = File.expand_path("daemon/alive", project.path)
+    File.unlink(f) if File.exist? f
+  end
+
   private
     
     def launch_job(job)

data/lib/miga/dataset_result.rb

@@ -44,9 +44,11 @@ module MiGA::DatasetResult
       r = MiGA::Result.new("#{base}.json")
       if result_files_exist?(base, ".2.clipped.fastq")
         r = add_files_to_ds_result(r, name,
-          pair1:".1.clipped.fastq", pair2:".2.clipped.fastq")
+          pair1:".1.clipped.fastq", pair2:".2.clipped.fastq",
+          single:".1.clipped.single.fastq")
+      else
+        r = add_files_to_ds_result(r, name, single:".1.clipped.fastq")
       end
-      r.add_file(:single, "#{name}.1.clipped.single.fastq")
       r.add_file(:trimming_sumary, "#{name}.1.fastq.trimmed.summary.txt")
       add_result(:raw_reads) #-> Post gunzip
       r

data/lib/miga/project.rb

@@ -380,6 +380,8 @@ class MiGA::Project < MiGA::MiGA
   # List plugins installed in the project.
   def plugins ; metadata[:plugins] ||= [] ; end
 
+  ##
+  # Loads the plugins installed in the project.
   def load_plugins
     plugins.each { |pl| require File.expand_path("lib-plugin.rb", pl) }
   end

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.3, 0, 2]
+  VERSION = [0.3, 0, 5]
   
   ##
   # Nickname for the current major.minor version.
@@ -18,7 +18,7 @@ module MiGA
   
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2017, 6, 14)
+  VERSION_DATE = Date.new(2017, 7, 8)
   
   ##
   # Reference of MiGA.

data/scripts/ogs.bash

@@ -31,7 +31,8 @@ if [[ ! -s miga-project.ogs ]] ; then
 
   # Estimate OGs and Clean RBMs
   ogs.mcl.rb -o miga-project.ogs -d miga-project.rbm -t "$CORES"
-  rm -rf miga-project.rbm
+  [[ $(miga about -P "$PROJECT" -m clean_ogs) == "false" ]] \
+    || rm -rf miga-project.rbm
 fi
 
 # Calculate Statistics

data/test/daemon_test.rb

@@ -10,7 +10,7 @@ class DaemonTest < Test::Unit::TestCase
     FileUtils.touch("#{ENV["MIGA_HOME"]}/.miga_rc")
     File.open("#{ENV["MIGA_HOME"]}/.miga_daemon.json", "w") do |fh|
       fh.puts '{"maxjobs":1,"ppn":1,"latency":2,"varsep":" ","var":"%s=%s",
-        "cmd":"%5$s","alive":"echo 1 # %s"}'
+        "cmd":"%5$s","alive":"echo 1 # %s","type":"bash"}'
     end
     $p1 = MiGA::Project.new(File.expand_path("project1", $tmp))
     $d1 = MiGA::Daemon.new($p1)
@@ -113,6 +113,12 @@ class DaemonTest < Test::Unit::TestCase
     assert_raise do
       $d1.runopts(:latency, "!")
     end
+    assert_equal("bash", $d1.runopts(:type))
+    assert_equal("kill -9 '%s'", $d1.runopts(:kill))
+    $d1.runopts(:type, "qsub")
+    assert_equal("qdel '%s'", $d1.runopts(:kill))
+    $d1.runopts(:type, "msub")
+    assert_equal("canceljob '%s'", $d1.runopts(:kill))
   end
 
   def test_say
@@ -122,4 +128,15 @@ class DaemonTest < Test::Unit::TestCase
     assert(out.string =~ /^\[.*\] Olm/)
   end
 
+  def test_terminate
+    d = $d1
+    d.declare_alive
+    assert_not_nil(d.last_alive)
+    out = capture_stdout do
+      d.terminate
+    end
+    assert(out.string =~ /Terminating daemon/)
+    assert_nil(d.last_alive)
+  end
+
 end

data/test/taxonomy_test.rb

@@ -31,6 +31,7 @@ class TaxonomyTest < Test::Unit::TestCase
     assert_equal("v3", tx[:g])
     tx << "s:v3_0"
     assert(tx.is_in? MiGA::Taxonomy.new("species:v3_0"))
+    assert_raise(RuntimeError) { tx << 123 }
   end
 
   def test_init_methods

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.3.0.2
+  version: 0.3.0.5
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-14 00:00:00.000000000 Z
+date: 2017-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -407,26 +407,26 @@ files:
 - utils/subclades-nj.R
 - utils/subclades.R
 - bin/miga
+- actions/about.rb
+- actions/add.rb
 - actions/add_result.rb
-- actions/add_taxonomy.rb
-- actions/create_dataset.rb
-- actions/create_project.rb
 - actions/daemon.rb
 - actions/date.rb
-- actions/download_dataset.rb
-- actions/find_datasets.rb
-- actions/import_datasets.rb
-- actions/index_taxonomy.rb
+- actions/files.rb
+- actions/find.rb
+- actions/get.rb
 - actions/init.rb
-- actions/list_datasets.rb
-- actions/list_files.rb
+- actions/ln.rb
+- actions/ls.rb
+- actions/new.rb
 - actions/plugins.rb
-- actions/project_info.rb
-- actions/result_stats.rb
-- actions/run_local.rb
-- actions/tax_distributions.rb
-- actions/test_taxonomy.rb
-- actions/unlink_dataset.rb
+- actions/rm.rb
+- actions/run.rb
+- actions/stats.rb
+- actions/tax_dist.rb
+- actions/tax_index.rb
+- actions/tax_set.rb
+- actions/tax_test.rb
 - Gemfile
 - Rakefile
 - README.md