miga-base 0.2.0.9 → 0.2.1.0

data/actions/index_taxonomy.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+
+# @package MiGA
+# @license Artistic-2.0
+
+require "miga/tax_index"
+
+o = {q:true, format: :json}
+OptionParser.new do |opt|
+  opt_banner(opt)
+  opt_object(opt, o, [:project])
+  opt.on("-i", "--index PATH",
+    "(Mandatory) File to create with the index."){ |v| o[:index]=v }
+  opt.on("-f", "--format STRING",
+    "Format of the index file. By default: #{o[:format]}. Supported: " +
+    "json, tab."){ |v| o[:format]=v.to_sym }
+  opt_filter_datasets(opt, o)
+  opt_common(opt, o)
+end.parse!
+
+##=> Main <=
+opt_require(o, project:"-P", index:"-i")
+
+$stderr.puts "Loading project." unless o[:q]
+p = MiGA::Project.load(o[:project])
+raise "Impossible to load project: #{o[:project]}" if p.nil?
+
+$stderr.puts "Loading datasets." unless o[:q]
+ds = p.datasets
+ds.select!{|d| not d.metadata[:tax].nil? }
+ds = filter_datasets!(ds, o)
+
+$stderr.puts "Indexing taxonomy." unless o[:q]
+tax_index = MiGA::TaxIndex.new
+ds.each { |d| tax_index << d }
+
+$stderr.puts "Saving index." unless o[:q]
+fh = File.open(o[:index], "w")
+if o[:format]==:json
+  fh.print tax_index.to_json
+elsif o[:format]==:tab
+  fh.print tax_index.to_tab
+end
+fh.close
+
+$stderr.puts "Done." unless o[:q]

data/actions/list_datasets.rb

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+
+# @package MiGA
+# @license Artistic-2.0
+
+o = {q:true, info:false, processing:false}
+OptionParser.new do |opt|
+  opt_banner(opt)
+  opt_object(opt, o, [:project, :dataset_opt])
+  opt_filter_datasets(opt, o)
+  opt.on("-i", "--info",
+    "Print additional information on each dataset."){ |v| o[:info]=v }
+  opt.on("-p", "--processing",
+    "Print information on processing advance."){ |v| o[:processing]=v }
+  opt.on("-m", "--metadata STRING",
+    "Print name and metadata field only. If set, ignores -i."
+    ){ |v| o[:datum]=v }
+  opt_common(opt, o)
+end.parse!
+
+##=> Main <=
+opt_require(o, project:"-P")
+
+$stderr.puts "Loading project." unless o[:q]
+p = MiGA::Project.load(o[:project])
+raise "Impossible to load project: #{o[:project]}" if p.nil?
+
+$stderr.puts "Listing datasets." unless o[:q]
+if o[:dataset].nil?
+  ds = p.datasets
+elsif MiGA::Dataset.exist? p, o[:dataset]
+  ds = [p.dataset(o[:dataset])]
+else
+  ds = []
+end
+ds = filter_datasets!(ds, o)
+
+if not o[:datum].nil?
+  ds.each{|d| puts "#{d.name}\t#{d.metadata[ o[:datum] ] || "?"}"}
+elsif o[:info]
+  puts MiGA::MiGA.tabulate(MiGA::Dataset.INFO_FIELDS, ds.map{ |d| d.info })
+elsif o[:processing]
+  comp = ["undef","done","queued"]
+  puts MiGA::MiGA.tabulate([:name] + MiGA::Dataset.PREPROCESSING_TASKS,
+    ds.map{ |d| [d.name] + d.profile_advance.map{ |i| comp[i] } })
+else
+  ds.each{|d| puts d.name}
+end
+
+$stderr.puts "Done." unless o[:q]

data/actions/list_files.rb

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+
+# @package MiGA
+# @license Artistic-2.0
+
+o = {q:true, details:false, json:true}
+OptionParser.new do |opt|
+  opt_banner(opt)
+  opt_object(opt, o, [:project, :dataset_opt])
+  opt.on("-i", "--info",
+    "If set, it prints additional details for each file."
+    ){ |v| o[:details]=v }
+  opt.on("--[no-]json",
+    "If set to no, excludes json files containing results metadata."
+    ){ |v| o[:json]=v }
+  opt_common(opt, o)
+end.parse!
+
+##=> Main <=
+opt_require(o, project:"-P")
+
+$stderr.puts "Loading project." unless o[:q]
+p = MiGA::Project.load(o[:project])
+raise "Impossible to load project: #{o[:project]}" if p.nil?
+
+if o[:dataset].nil?
+  results = p.results
+else
+  $stderr.puts "Loading dataset." unless o[:q]
+  ds = p.dataset(o[:dataset])
+  raise "Impossible to load dataset: #{o[:dataset]}" if ds.nil?
+  results = ds.results
+end
+
+$stderr.puts "Listing files." unless o[:q]
+results.each do |result|
+  puts "#{ "#{result.path}\t\t" if o[:details] }#{result.path}" if o[:json]
+  result.each_file do |k,f|
+    puts "#{ "#{result.path}\t#{k}\t" if o[:details] }#{result.dir}/#{f}"
+  end
+end
+
+$stderr.puts "Done." unless o[:q]

data/actions/project_info.rb

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+
+# @package MiGA
+# @license Artistic-2.0
+
+o = {q:true, info:false, processing:false}
+OptionParser.new do |opt|
+  opt_banner(opt)
+  opt_object(opt, o, [:project])
+  opt.on("-p", "--processing",
+    "Print information on processing advance."){ |v| o[:processing]=v }
+  opt.on("-m", "--metadata STRING",
+    "Print name and metadata field only. If set, ignores -i."
+    ){ |v| o[:datum]=v }
+  opt_common(opt, o)
+end.parse!
+
+
+##=> Main <=
+opt_require(o, project:"-P")
+
+$stderr.puts "Loading project." unless o[:q]
+p = MiGA::Project.load(o[:project])
+raise "Impossible to load project: #{o[:project]}" if p.nil?
+
+if not o[:datum].nil?
+  puts (p.metadata[ o[:datum] ] || "?")
+elsif o[:processing]
+  keys = MiGA::Project.DISTANCE_TASKS + MiGA::Project.INCLADE_TASKS
+  puts MiGA::MiGA.tabulate([:task, :status], keys.map do |k|
+    [k, p.add_result(k, false).nil? ? "queued" : "done"]
+  end)
+else
+  puts MiGA::MiGA.tabulate([:key, :value], p.metadata.data.keys.map do |k|
+    v = p.metadata[k]
+    [k, k==:datasets ? v.size : v]
+  end)
+end
+
+$stderr.puts "Done." unless o[:q]

data/actions/unlink_dataset.rb

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+# @package MiGA
+# @license Artistic-2.0
+
+o = {q:true, remove:false}
+OptionParser.new do |opt|
+  opt_banner(opt)
+  opt_object(opt, o)
+  opt.on("-r", "--remove", "Also remove all associated files.",
+    "By default, only unlinks from metadata."){ o[:remove]=true }
+  opt_common(opt, o)
+end.parse!
+
+##=> Main <=
+opt_require(o)
+
+$stderr.puts "Loading project." unless o[:q]
+p = MiGA::Project.load(o[:project])
+raise "Impossible to load project: #{o[:project]}" if p.nil?
+
+$stderr.puts "Unlinking dataset." unless o[:q]
+raise "Dataset doesn't exist, aborting." unless
+  MiGA::Dataset.exist?(p, o[:dataset])
+d = p.unlink_dataset(o[:dataset])
+d.remove! if o[:remove]
+
+$stderr.puts "Done." unless o[:q]

data/bin/miga

@@ -1,48 +1,144 @@
 #!/usr/bin/env ruby
-#
+
 # @package MiGA
-# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
-# @license artistic license 2.0
-# @update Oct-01-2015
-#
+# @license Artistic-2.0
 
 $:.push File.expand_path("../lib", File.dirname(__FILE__))
 
 require "optparse"
 require "miga"
 
-execs = Dir[File.expand_path("../actions/*",
-   File.dirname(__FILE__))].map{ |b| File.basename b }
-
-if %w{-v --version}.include? ARGV[0]
-   puts MiGA::MiGA.VERSION
-elsif %w{-V --long-version}.include? ARGV[0]
-   puts MiGA::MiGA.LONG_VERSION
-elsif %w{-C --citation}.include? ARGV[0]
-   puts MiGA::MiGA.CITATION
-elsif execs.include? ARGV[0]
-   task = ARGV.shift
-   ARGV << "-h" if ARGV.empty?
-   begin
-      load File.expand_path("../actions/" + task, File.dirname(__FILE__))
-   rescue => err
-      $stderr.puts "Exception: #{err}\n\n"
-      err.backtrace.each { |l| $stderr.puts l + "\n" }
-      err
-   end
+##=> Global variables <=
+
+$task_desc = {
+  add_result: "Registers a result.",
+  add_taxonomy: "Registers taxonomic information for datasets.",
+  create_dataset: "Creates an empty dataset in a pre-existing MiGA project.",
+  create_project: "Creates an empty MiGA project.",
+  daemon: "Controls the daemon of a MiGA project.",
+  date: "Returns the current date in standard MiGA format.",
+  download_dataset: "Creates an empty dataset in a pre-existing MiGA project.",
+  find_datasets: "Finds unregistered datasets based on result files.",
+  import_datasets: "Link datasets (including results) from one project to "+
+    "another.",
+  index_taxonomy: "Creates a taxonomy-indexed list of the datasets.",
+  list_datasets: "Lists all registered datasets in an MiGA project.",
+  list_files: "Lists all registered files from the results of a dataset or a "+
+    "project.",
+  project_info: "Displays information about a MiGA project.",
+  unlink_dataset: "Removes a dataset from an MiGA project."
+}
+
+##=> Functions <=
+
+# OptParse banner
+def opt_banner(opt)
+  opt.banner = <<BAN
+#{$task_desc[$task]}
+
+Usage: #{$0} #{$task} [options]
+BAN
+  opt.separator ""
+end
+
+# OptParse flags that determine the object to load
+def opt_object(opt, o, what=[:project, :dataset])
+  opt.on("-P", "--project PATH", "(Mandatory) Path to the project."
+    ){ |v| o[:project]=v } if what.include? :project
+  opt.on("-D", "--dataset PATH", "(Mandatory) Name of the dataset."
+    ){ |v| o[:dataset]=v } if what.include? :dataset
+  opt.on("-D", "--dataset PATH", "Name of the dataset."
+    ){ |v| o[:dataset]=v } if what.include? :dataset_opt
+  opt.on("-t", "--type STRING",
+    "Type of dataset. Recognized types include:",
+    *MiGA::Dataset.KNOWN_TYPES.map{ |k,v| "~ #{k}: #{v[:description]}" }
+    ){ |v| o[:type]=v.to_sym } if what.include? :dataset_type
+  opt.on("-t", "--type STRING",
+    "Type of dataset. Recognized types include:", 
+    *MiGA::Project.KNOWN_TYPES.map{ |k,v| "~ #{k}: #{v[:description]}"}
+    ){ |v| o[:type]=v.to_sym } if what.include? :project_type
+end
+
+# OptParse flags common to all actions.
+def opt_common(opt, o)
+  opt.on("-v", "--verbose",
+    "Print additional information to STDERR."){ o[:q]=false }
+  opt.on("-d", "--debug INT", "Print debugging information to STDERR.") do |v|
+    v.to_i>1 ? MiGA::MiGA.DEBUG_TRACE_ON : MiGA::MiGA.DEBUG_ON
+  end
+  opt.on("-h", "--help", "Display this screen.") do
+    puts opt
+    exit
+  end
+  opt.separator ""
+end
+
+# OptParse flags to filter lists of datasets.
+def opt_filter_datasets(opt, o, what=[:ref, :multi, :taxonomy])
+  opt.on("--[no-]ref",
+    "If set, uses only reference (or only non-reference) datasets."
+    ){ |v| o[:ref]=v } if what.include? :ref
+  opt.on("--[no-]multi",
+    "If set, uses only multi-species (or only single-species) datasets."
+    ){ |v| o[:multi]=v } if what.include? :multi
+  opt.on("-t", "--taxonomy RANK:TAXON", "Filter by taxonomy."
+    ){ |v| o[:taxonomy]=MiGA::Taxonomy.new v } if what.include? :taxonomy
+end
+
+def opt_require(o, req={project:"-P", dataset:"-D"})
+  req.each do |k,v|
+    raise "#{v} is mandatory: please provide #{k}." if o[k].nil?
+  end
+end
+
+# Filters datasets by keys set in +opt_filter_datasets+.
+def filter_datasets!(ds, o)
+  ds.select!{|d| d.is_ref? == o[:ref] } unless o[:ref].nil?
+  ds.select! do |d|
+    o[:multi] ? d.is_multi? : d.is_nonmulti?
+  end unless o[:multi].nil?
+  ds.select! do |d|
+    (not d.metadata[:tax].nil?) and d.metadata[:tax].is_in?(o[:taxonomy])
+  end unless o[:taxonomy].nil?
+  ds
+end
+
+##=> Main <=
+
+execs = $task_desc.keys.map{ |k| k.to_s }
+
+case ARGV[0]
+when "-v", "--version"
+  puts MiGA::MiGA.VERSION
+when "-V", "--long-version"
+  puts MiGA::MiGA.LONG_VERSION
+when "-C", "--citation"
+  puts MiGA::MiGA.CITATION
+when *execs
+  $task = ARGV.shift.to_sym
+  ARGV << "-h" if ARGV.empty? and not [:date].include? $task
+  begin
+    load File.expand_path("../actions/#{$task}.rb", File.dirname(__FILE__))
+  rescue => err
+    $stderr.puts "Exception: #{err}\n\n"
+    err.backtrace.each { |l| $stderr.puts l + "\n" }
+    err
+  end
 else
-   print <<HELP.gsub(/^   /,"")
-   Microbial Genomes Atlas.
+  print <<HELP
+
+Microbial Genomes Atlas.
+
+Usage: #{$0} {action} [options]
 
-   Usage: #{$0} {action} [options]
+#{ MiGA::MiGA.tabulate([:action, :description], $task_desc.to_a).join("\n")}
 
-   actions:#{ execs.map{ |e| "\n       #{e}"}.join }
+generic options:
+    -h, --help          Display this screen.
+    -v, --version       Show MiGA version.
+    -V, --long-version  Show complete MiGA version.
+    -C, --citation      How to cite MiGA.
 
-   generic options:
-       -h, --help		Display this screen.
-       -v, --version	Show MiGA version.
-       -V, --long-version	Show complete MiGA version.
-       -C, --citation	How to cite MiGA.
 HELP
 end
 

data/lib/miga/daemon.rb

@@ -26,6 +26,8 @@ class MiGA::Daemon < MiGA::MiGA
   attr_reader :jobs_to_run
   # Array of jobs currently running.
   attr_reader :jobs_running
+  # Integer indicating the current iteration.
+  attr_reader :loop_i
 
   ##
   # Initialize an unactive daemon for the MiGA::Project +project+. See #daemon
@@ -37,6 +39,7 @@ class MiGA::Daemon < MiGA::MiGA
         {:symbolize_names=>true})
     @jobs_to_run = []
     @jobs_running = []
+    @loop_i = -1
   end
 
   ##
@@ -55,13 +58,14 @@ class MiGA::Daemon < MiGA::MiGA
 
   ##
   # Set/get #options, where +k+ is the Symbol of the option and +v+ is the value
-  # (or nil to use as getter). Returns new value.
-  def runopts(k, v=nil)
+  # (or nil to use as getter). Skips consistency tests if +force+. Returns new
+  # value.
+  def runopts(k, v=nil, force=false)
     k = k.to_sym
     unless v.nil?
       v = v.to_i if [:latency, :maxjobs, :ppn].include? k
       raise "Daemon's #{k} cannot be set to zero." if
-        v.is_a? Integer and v==0
+        !force and v.is_a? Integer and v==0
       @runopts[k] = v
     end
     @runopts[k]
@@ -80,20 +84,20 @@ class MiGA::Daemon < MiGA::MiGA
   def ppn() runopts(:ppn) ; end
 
   ##
-  # Initializes the daemon.
-  def start() daemon("start") ; end
+  # Initializes the daemon with +opts+.
+  def start(opts=[]) daemon("start", opts) ; end
 
   ##
-  # Stops the daemon.
-  def stop() daemon("stop") ; end
+  # Stops the daemon with +opts+.
+  def stop(opts=[]) daemon("stop", opts) ; end
 
   ##
-  # Restarts the daemon.
-  def restart() daemon("restart") ; end
+  # Restarts the daemon with +opts+.
+  def restart(opts=[]) daemon("restart", opts) ; end
 
   ##
-  # Returns the status of the daemon.
-  def status() daemon("status") ; end
+  # Returns the status of the daemon with +opts+.
+  def status(opts=[]) daemon("status", opts) ; end
 
   ##
   # Launches the +task+ with options +opts+ (as command-line arguments).
@@ -102,26 +106,7 @@ class MiGA::Daemon < MiGA::MiGA
     options = default_options
     opts.unshift(task)
     options[:ARGV] = opts
-    Daemons.run_proc("MiGA:#{project.name}", options) do
-      say "-----------------------------------"
-      say "MiGA:#{project.name} launched."
-      say "-----------------------------------"
-      loop_i = 0
-      loop do
-        loop_i += 1
-        declare_alive
-        check_datasets
-        check_project
-        flush!
-        if loop_i==12
-          say "Housekeeping for sanity"
-          loop_i = 0
-          purge!
-          project.load
-        end
-        sleep(latency)
-      end
-    end
+    Daemons.run_proc("MiGA:#{project.name}", options) { loop { in_loop } }
   end
 
   ##
@@ -135,9 +120,14 @@ class MiGA::Daemon < MiGA::MiGA
   ##
   # Traverse datasets
   def check_datasets
-    project.each_dataset do |ds|
-      to_run = ds.next_preprocessing(true)
-      queue_job(to_run, ds) unless to_run.nil?
+    project.each_dataset do |n, ds|
+      if ds.nil?
+        say "Warning: Dataset #{n} listed but not loaded, reloading project."
+        project.load
+      else
+        to_run = ds.next_preprocessing(true)
+        queue_job(to_run, ds) unless to_run.nil?
+      end
     end
   end
 
@@ -145,6 +135,7 @@ class MiGA::Daemon < MiGA::MiGA
   # Check if all reference datasets are pre-processed. If yes, check the
   # project-level tasks
   def check_project
+    return if project.dataset_names.empty?
     if project.done_preprocessing?(false)
       to_run = project.next_distances(true)
       to_run = project.next_inclade(true) if to_run.nil?
@@ -230,6 +221,29 @@ class MiGA::Daemon < MiGA::MiGA
     end
   end
 
+  ##
+  # Run one loop step.
+  def in_loop
+    if loop_i == -1
+      say "-----------------------------------"
+      say "MiGA:#{project.name} launched."
+      say "-----------------------------------"
+      @loop_i = 0
+    end
+    @loop_i += 1
+    declare_alive
+    check_datasets
+    check_project
+    flush!
+    if loop_i==12
+      say "Housekeeping for sanity"
+      @loop_i = 0
+      purge!
+      project.load
+    end
+    sleep(latency)
+  end
+
   ##
   # Send a datestamped message to the log.
   def say(*opts)