miga-base 0.4.1.0 → 0.4.2.0

data/lib/miga/cli/action.rb

@@ -0,0 +1,66 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli'
+
+##
+# An action to be performed by the CLI. This is a generic class to be extended
+# by MiGA::Cli::Action::* classes. Do not attempt creating directly with +new+,
+# use instead the MiGA::Cli::Action.load interface.
+class MiGA::Cli::Action < MiGA::MiGA
+
+  class << self
+    def load(task, cli)
+      require "miga/cli/action/#{task}"
+      camel = task.to_s.gsub(/(?:_|^)(\S)/, &:upcase).delete('_')
+      klass = Object.const_get("MiGA::Cli::Action::#{camel}")
+      klass.new(cli)
+    end
+  end
+
+  attr_accessor :cli
+
+  def initialize(cli)
+    @cli = cli
+  end
+
+  ##
+  # Launch the sequence
+  def launch
+    empty_action if cli.argv.empty?
+    parse_cli
+    perform
+    complete
+  end
+
+  ##
+  # Parse the CLI object
+  def parse_cli
+    raise "Undefined interface for the command line of #{cli.task}"
+  end
+
+  ##
+  # Perform the action
+  def perform
+    raise "Undefined action for the command line of #{cli.task}"
+  end
+
+  ##
+  # Complete the action
+  def complete
+    cli.say 'Done'
+  end
+
+  ##
+  # Name of the action, as referred to by the CLI
+  def name
+    camel = self.class.to_s.gsub(/.*::/,'')
+    camel.gsub(/(\S)([A-Z])/,'\1_\2').downcase
+  end
+
+  ##
+  # What to do when cli.argv is empty
+  def empty_action
+    cli.argv << '-h'
+  end
+end

data/lib/miga/cli/base.rb

@@ -0,0 +1,90 @@
+# @package MiGA
+# @license Artistic-2.0
+
+module MiGA::Cli::Base
+
+  @@TASK_DESC = {
+    generic:  'MiGA: The Microbial Genomes Atlas',
+    # Projects
+    new:      'Creates an empty MiGA project',
+    about:    'Displays information about a MiGA project',
+    plugins:  'Lists or (un)installs plugins in a MiGA project',
+    doctor:   'Performs consistency checks on a MiGA project',
+    # Datasets
+    add:      'Creates a dataset in a MiGA project',
+    get:      'Downloads a dataset from public databases into a MiGA project',
+    ncbi_get: 'Downloads all genomes in a taxon from NCBI into a MiGA project',
+    rm:       'Removes a dataset from an MiGA project',
+    find:     'Finds unregistered datasets based on result files',
+    ln:       'Link datasets (including results) from one project to another',
+    ls:       'Lists all registered datasets in an MiGA project',
+    # Results
+    add_result: 'Registers a result',
+    stats:    'Extracts statistics for the given result',
+    files:    'Lists registered files from the results of a dataset or project',
+    run:      'Executes locally one step analysis producing the given result',
+    summary:  'Generates a summary table for the statistics of all datasets',
+    next_step: 'Returns the next task to run in a dataset or project',
+    # Objects (Datasets or Projects)
+    edit:     'Edits the metadata of a dataset or project',
+    # System
+    init:     'Initialize MiGA to process new projects',
+    daemon:   'Controls the daemon of a MiGA project',
+    date:     'Returns the current date in standard MiGA format',
+    console:  'Opens an IRB console with MiGA',
+    # Taxonomy
+    tax_set:  'Registers taxonomic information for datasets',
+    tax_test: 'Returns test of taxonomic distributions for query datasets',
+    tax_index: 'Creates a taxonomy-indexed list of the datasets',
+    tax_dist: 'Estimates distributions of distance by taxonomy',
+  }
+
+  @@TASK_ALIAS = {
+    # Projects
+    create_project: :new,
+    project_info: :about,
+    # Datasets
+    create_dataset: :add,
+    download_dataset: :get,
+    unlink_dataset: :rm,
+    find_datasets: :find,
+    import_datasets: :ln,
+    list_datasets: :ls,
+    # Results
+    result_stats: :stats,
+    list_files: :files,
+    run_local: :run,
+    sum_stats: :summary,
+    next_task: :next_step,
+    # Objects
+    update_metadata: :edit,
+    # System
+    c: :console,
+    # Taxonomy
+    add_taxonomy: :tax_set,
+    test_taxonomy: :tax_test,
+    index_taxonomy: :tax_index,
+    tax_distributions: :tax_dist,
+  }
+
+  @@TASK_ALIAS.each do |nick, task|
+    @@TASK_DESC[task] = ((@@TASK_DESC[task] =~ /\(alias: .*\)\./) ?
+        @@TASK_DESC[task].sub(/\)\.$/, ", #{nick}).") :
+        @@TASK_DESC[task].sub(/\.$/, " (alias: #{nick}).")
+    )
+  end
+
+  @@EXECS = @@TASK_DESC.keys
+
+end
+
+class MiGA::Cli < MiGA::MiGA
+
+  include MiGA::Cli::Base
+
+  class << self
+    def TASK_DESC; @@TASK_DESC end
+    def TASK_ALIAS; @@TASK_ALIAS end
+    def EXECS; @@EXECS end
+  end
+end

data/lib/miga/cli.rb

@@ -0,0 +1,426 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/project'
+require 'optparse'
+
+##
+# MiGA Command Line Interface API.
+class MiGA::Cli < MiGA::MiGA
+
+  require 'miga/cli/base'
+  require 'miga/cli/action'
+
+  ##
+  # Task to execute, a symbol
+  attr_accessor :task
+
+  ##
+  # The CLI parameters (except the task), and Array of String
+  attr_accessor :argv
+
+  ##
+  # Action to launch, an object inheriting from MiGA::Cli::Action
+  attr_accessor :action
+
+  ##
+  # If files are expected after the parameters, a boolean
+  attr_accessor :expect_files
+
+  ##
+  # Files passed after all other options, if +#expect_files = true+
+  attr_accessor :files
+
+  ##
+  # If an operation verb preceding all other arguments is to be expected
+  attr_accessor :expect_operation
+
+  ##
+  # Interactivity with the user is expected
+  attr_accessor :interactive
+
+  ##
+  # Operation preceding all other options, if +#expect_operation = true+
+  attr_accessor :operation
+
+  ##
+  # Include common options, a boolean (true by default)
+  attr_writer :opt_common
+
+  ##
+  # Default values as a Hash
+  attr_accessor :defaults
+
+  ##
+  # Parsed values as a Hash
+  attr_reader :data
+
+  def initialize(argv)
+    @data = {}
+    @defaults = {verbose: false, tabular: false}
+    @opt_common = true
+    @objects = {}
+    if argv[0].nil? or argv[0].to_s[0] == '-'
+      @task = :generic
+    else
+      @task = argv.shift.to_sym
+      @task = @@TASK_ALIAS[task] unless @@TASK_ALIAS[task].nil?
+    end
+    @argv = argv
+    reset_action
+  end
+
+  ##
+  # Send +par+ to $stdout, ensuring new line at the end
+  def puts(*par)
+    $stdout.puts(*par)
+  end
+
+  ##
+  # Send +par+ to $stdout as is
+  def print(*par)
+    $stdout.print(*par)
+  end
+
+  ##
+  # Display a table with headers +header+ and contents +values+, both Array
+  def table(header, values)
+    self.puts MiGA.tabulate(header, values, self[:tabular])
+  end
+
+  ##
+  # Send +par+ to $stderr (ensuring new line at the end), iff --verbose.
+  # Date/time each line.
+  def say(*par)
+    return unless self[:verbose]
+    par.map! { |i| "[#{Time.now}] #{i}" }
+    $stderr.puts(*par)
+  end
+
+  ##
+  # Reports the advance of a task at +step+ (String), the +n+ out of +total+
+  # The report goes to $stderr iff --verborse
+  def advance(step, n = 0, total = nil)
+    return unless self[:verbose]
+    adv = total.nil? ? '' : ('%.1f%% (%d/%d)' % [n/total, n, total])
+    $stderr.print("[%s] %s %s    \r" % [Time.now, step, adv])
+  end
+
+  ##
+  # Ask a question +question+ to the user (requires +#interactive = true+)
+  # The +default+ is used if the answer is empty
+  # The +answers+ are supported values, unless nil
+  # If --auto, all questions are anwered with +default+ unless +force+
+  def ask_user(question, default = nil, answers = nil, force = false)
+    ans = " (#{answers.join(' / ')})" unless answers.nil?
+    dft = " [#{default}]" unless default.nil?
+    print "#{question}#{ans}#{dft} > "
+    if self[:auto] && !force
+      puts ''
+    else
+      y = gets.chomp
+    end
+    y = default.to_s if y.nil? or y.empty?
+    unless answers.nil? or answers.map(&:to_s).include?(y)
+      warn "Answer not recognized: '#{y}'"
+      return ask_user(question, default, answers, force)
+    end
+    y
+  end
+
+  ##
+  # Set default values in the Hash +hsh+
+  def defaults=(hsh)
+    hsh.each{ |k,v| @defaults[k] = v }
+  end
+
+  ##
+  # Access parsed data
+  def [](k)
+    k = k.to_sym
+    @data[k].nil? ? @defaults[k] : @data[k]
+  end
+
+  ##
+  # Set parsed data
+  def []=(k, v)
+    @data[k.to_sym] = v
+  end
+
+  ##
+  # Redefine #action based on #task
+  def reset_action
+    @action = nil
+    if @@EXECS.include? task
+      @action = Action.load(task, self)
+    else
+      warn "No action set for #{task}"
+    end
+  end
+
+  ##
+  # Perform the task requested (see #task)
+  def launch
+    begin
+      raise "See `miga -h`" if action.nil?
+      action.launch
+    rescue => err
+      $stderr.puts "Exception: #{err}"
+      $stderr.puts ''
+      err.backtrace.each { |l| $stderr.puts "DEBUG: #{l}" }
+      err
+    end
+  end
+
+  ##
+  # Parse the #argv parameters
+  def parse(&fun)
+    if expect_operation
+      @operation = @argv.shift unless argv.first =~ /^-/
+    end
+    OptionParser.new do |opt|
+      banner(opt)
+      fun[opt]
+      opt_common(opt)
+    end.parse!(@argv)
+    if expect_files
+      @files = argv
+    end
+  end
+
+  ##
+  # Send MiGA's banner to OptionParser +opt+
+  def banner(opt)
+    usage = "Usage: miga #{action.name}"
+    usage += ' {operation}' if expect_operation
+    usage += ' [options]'
+    usage += ' {FILES...}' if expect_files
+    opt.banner = "\n#{task_description}\n\n#{usage}\n"
+    opt.separator ''
+  end
+
+  ##
+  # Common options at the end of most actions, passed to OptionParser +opt+
+  # No action is performed if +#opt_common = false+ is passed
+  # Executes only once, unless +#opt_common = true+ is passed between calls
+  def opt_common(opt)
+    return unless @opt_common
+    opt.on(
+      '--auto',
+      'Accept all defaults as answers'
+      ){ |v| cli[:auto] = v } if interactive
+    opt.on(
+      '-v', '--verbose',
+      'Print additional information to STDERR'
+      ){ |v| self[:verbose] = v }
+    opt.on(
+      '-d', '--debug INT', Integer,
+      'Print debugging information to STDERR (1: debug, 2: trace)'
+      ){ |v| (v > 1) ? MiGA.DEBUG_TRACE_ON : MiGA.DEBUG_ON }
+    opt.on(
+      '-h', '--help',
+      'Display this screen'
+      ){ puts opt ; exit }
+    opt.separator ''
+    self.opt_common = false
+  end
+
+  ##
+  # Options to load an object passed to OptionParser +opt+, as determined
+  # by +what+ an Array with any combination of:
+  # - :project To require a project
+  # - :dataset To require a dataset
+  # - :dataset_opt To allow (optionally) a dataset
+  # - :dataset_type To allow (optionally) a type of dataset
+  # - :dataset_type_req To require a type of dataset
+  # - :project_type To allow (optionally) a type of project
+  # - :project_type_req To require a type of project
+  # - :result To require a type of project or dataset result
+  # - :result_dataset To require a type of dataset result
+  # - :result_project To require a type of project result
+  def opt_object(opt, what = [:project, :dataset])
+    opt.on(
+      '-P', '--project PATH',
+      '(Mandatory) Path to the project'
+      ){ |v| self[:project] = v } if what.include? :project
+    opt.on(
+      '-D', '--dataset STRING',
+      (what.include?(:dataset) ? '(Mandatory) ' : '') + 'Name of the dataset'
+      ){ |v| self[:dataset] = v } if what.include? :dataset or
+        what.include? :dataset_opt
+    opt.on(
+      '-D', '--dataset STRING',
+      'Name of the dataset'
+      ){ |v| self[:dataset] = v } if what.include? :dataset_opt
+    opt.on(
+      '-t', '--type STRING',
+      (what.include?(:dataset_type_req) ? '(Mandatory) ' : '') +
+      'Type of dataset. Recognized types include:',
+      *Dataset.KNOWN_TYPES.map{ |k,v| "~ #{k}: #{v[:description]}" }
+      ){ |v| self[:type] = v.downcase.to_sym } if what.include? :dataset_type or
+        what.include? :dataset_type_req
+    opt.on(
+      '-t', '--type STRING',
+      (what.include?(:project_type_req) ? '(Mandatory) ' : '') +
+      'Type of project. Recognized types include:',
+      *Project.KNOWN_TYPES.map{ |k,v| "~ #{k}: #{v[:description]}"}
+      ){ |v| self[:type] = v.downcase.to_sym } if what.include? :project_type or
+        what.include? :project_type_req
+    opt.on(
+      '-r', '--result STRING',
+      '(Mandatory) Name of the result',
+      'Recognized names for dataset-specific results include:',
+      *Dataset.RESULT_DIRS.keys.map{|n| " ~ #{n}"},
+      'Recognized names for project-wide results include:',
+      *Project.RESULT_DIRS.keys.map{|n| " ~ #{n}"}
+      ){ |v| self[:result] = v.downcase.to_sym } if what.include? :result
+    opt.on(
+      '-r', '--result STRING',
+      '(Mandatory) Name of the result, one of:',
+      *Dataset.RESULT_DIRS.keys.map{|n| " ~ #{n}"}
+      ){ |v| self[:result] = v.downcase.to_sym } if what.include? :result_dataset
+    opt.on(
+      '-r', '--result STRING',
+      '(Mandatory) Name of the result, one of:',
+      *Project.RESULT_DIRS.keys.map{|n| " ~ #{n}"}
+      ){ |v| self[:result] = v.downcase.to_sym } if what.include? :result_project
+  end
+
+  ##
+  # Options to filter a list of datasets passed to OptionParser +opt+,
+  # as determined by +what+ an Array with any combination of:
+  # - :ref To filter by reference (--ref) or query (--no-ref)
+  # - :multi To filter by multiple (--multi) or single (--no-multi) species
+  # - :active To filter by active (--active) or inactive (--no-active)
+  # - :taxonomy To filter by taxonomy (--taxonomy)
+  # The "k-th" filter (--dataset-k) is always included
+  def opt_filter_datasets(opt, what = [:ref, :multi, :active, :taxonomy])
+    opt.on(
+      '--[no-]ref',
+      'Use only reference (or only non-reference) datasets'
+      ){ |v| self[:ref] = v } if what.include? :ref
+    opt.on(
+      '--[no-]multi',
+      'Use only multi-species (or only single-species) datasets'
+      ){ |v| self[:multi] = v } if what.include? :multi
+    opt.on(
+      '--[no-]active',
+      'Use only active (or inactive) datasets'
+      ){ |v| self[:active] = v } if what.include? :active
+    opt.on(
+      '-t', '--taxonomy RANK:TAXON',
+      'Filter by taxonomy'
+      ){ |v| self[:taxonomy] = Taxonomy.new(v) } if what.include? :taxonomy
+    opt.on(
+      '--dataset-k INTEGER', Integer,
+      'Use only the k-th dataset in the list'
+      ){ |v| self[:dataset_k] = v }
+  end
+
+  ##
+  # Ensure that these parameters have been passed to the CLI, as defined by
+  # +par+, a Hash with object names as keys and parameter flag as values.
+  # If missing, raise an error with message +msg+
+  def ensure_par(req, msg = '%<name>s is mandatory: please provide %<flag>s')
+    req.each do |k,v|
+      raise (msg % {name: k, flag: v}) if self[k].nil?
+    end
+  end
+
+  ##
+  # Ensure that "type" is passed and valid for the given +klass+
+  def ensure_type(klass)
+    ensure_par(type: '-t')
+    if klass.KNOWN_TYPES[self[:type]].nil?
+      raise "Unrecognized type: #{self[:type]}"
+    end
+  end
+
+  ##
+  # Get the project defined in the CLI by parameter +name+ and +flag+
+  def load_project(name = :project, flag = '-P')
+    return @objects[name] unless @objects[name].nil?
+    ensure_par(name => flag)
+    say "Loading project: #{self[name]}"
+    @objects[name] = Project.load(self[name])
+    raise "Cannot load project: #{self[name]}" if @objects[name].nil?
+    @objects[name]
+  end
+
+  ##
+  # Load the dataset defined in the CLI
+  # If +silent=true+, it allows failures silently
+  def load_dataset(silent = false)
+    return @objects[:dataset] unless @objects[:dataset].nil?
+    ensure_par(dataset: '-D')
+    @objects[:dataset] = load_project.dataset(self[:dataset])
+    if !silent && @objects[:dataset].nil?
+      raise "Cannot load dataset: #{self[:dataset]}"
+    end
+    return @objects[:dataset]
+  end
+
+  ##
+  # Load an a project or (if defined) a dataset
+  def load_project_or_dataset
+    self[:dataset].nil? ? load_project : load_dataset
+  end
+
+  ##
+  # Load and filter a list of datasets as requested in the CLI
+  # If +silent=true+, it allows failures silently
+  def load_and_filter_datasets(silent = false)
+    return @objects[:filtered_datasets] unless @objects[:filtered_datasets].nil?
+    say 'Listing datasets'
+    ds = self[:dataset].nil? ?
+      load_project.datasets : [load_dataset(silent)].compact
+    ds.select! { |d| d.is_ref? == self[:ref] } unless self[:ref].nil?
+    ds.select! { |d| d.is_active? == self[:active] } unless self[:active].nil?
+    ds.select! do |d|
+      self[:multi] ? d.is_multi? : d.is_nonmulti?
+    end unless self[:multi].nil?
+    ds.select! do |d|
+      (not d.metadata[:tax].nil?) && d.metadata[:tax].in?(self[:taxonomy])
+    end unless self[:taxonomy].nil?
+    ds = ds.values_at(self[:dataset_k]-1) unless self[:dataset_k].nil?
+    @objects[:filtered_datasets] = ds
+  end
+
+  def load_result
+    return @objects[:result] unless @objects[:result].nil?
+    ensure_par(result: '-r')
+    obj = load_project_or_dataset
+    if obj.class.RESULT_DIRS[self[:result]].nil?
+      raise "Unsupported result for #{obj.class.to_s.gsub(/.*::/,'')}: #{self[:result]}"
+    end
+    r = obj.add_result(self[:result], false)
+    raise "Cannot load result: #{self[:result]}" if r.nil?
+    @objects[:result] = r
+  end
+
+  def add_metadata(obj, cli = self)
+    cli[:metadata].split(',').each do |pair|
+      (k,v) = pair.split('=')
+      case v
+        when 'true';  v = true
+        when 'false'; v = false
+        when 'nil';   v = nil
+      end
+      if k == '_step'
+        obj.metadata["_try_#{v}"] ||= 0
+        obj.metadata["_try_#{v}"]  += 1
+      end
+      obj.metadata[k] = v
+    end unless cli[:metadata].nil?
+    [:type, :name, :user, :description, :comments].each do |k|
+      obj.metadata[k] = cli[k] unless cli[k].nil?
+    end
+    obj
+  end
+
+  ##
+  # Task description
+  def task_description
+    @@TASK_DESC[task]
+  end
+end

data/lib/miga/project/result.rb

@@ -9,7 +9,7 @@ require 'miga/project/base'
 module MiGA::Project::Result
 
   include MiGA::Project::Base
-  
+
   ##
   # Get result identified by Symbol +name+, returns MiGA::Result.
   def result(name)
@@ -23,7 +23,15 @@ module MiGA::Project::Result
   def results
     @@RESULT_DIRS.keys.map{ |k| result(k) }.reject{ |r| r.nil? }
   end
-  
+
+  ##
+  # For each result executes the 2-ary +blk+ block: key symbol and MiGA::Result.
+  def each_result(&blk)
+    @@RESULT_DIRS.keys.each do |k|
+      blk.call(k, result(k)) unless result(k).nil?
+    end
+  end
+
   ##
   # Add the result identified by Symbol +name+, and return MiGA::Result. Save
   # the result if +save+. The +opts+ hash controls result creation (if
@@ -45,12 +53,12 @@ module MiGA::Project::Result
     r.save unless r.nil?
     r
   end
-  
+
   ##
   # Get the next distances task, saving intermediate results if +save+. Returns
   # a Symbol.
   def next_distances(save = true) ; next_task(@@DISTANCE_TASKS, save) ; end
-  
+
   ##
   # Get the next inclade task, saving intermediate results if +save+. Returns a
   # Symbol.
@@ -72,8 +80,8 @@ module MiGA::Project::Result
       end
     end
   end
-  
-  
+
+
   private
 
     ##

data/lib/miga/remote_dataset.rb

@@ -16,7 +16,7 @@ class MiGA::RemoteDataset < MiGA::MiGA
       search_doc = MiGA::Json.parse(
         download(:ncbi_search, :assembly, acc, :json),
         symbolize: false, contents: true)
-      search_doc['esearchresult']['idlist'].first
+      (search_doc['esearchresult']['idlist'] || []).first
     end
   end
 

data/lib/miga/tax_index.rb

@@ -6,9 +6,9 @@ require "miga/taxonomy"
 ##
 # Indexing methods based on taxonomy.
 class MiGA::TaxIndex < MiGA::MiGA
-  
+
   # Instance-level
-  
+
   ##
   # Datasets in the index.
   attr_reader :datasets
@@ -28,6 +28,7 @@ class MiGA::TaxIndex < MiGA::MiGA
     return nil if dataset.metadata[:tax].nil?
     taxon = @root
     MiGA::Taxonomy.KNOWN_RANKS.each do |rank|
+      next if rank == :ns
       taxon = taxon.add_child(rank, dataset.metadata[:tax][rank])
     end
     taxon.add_dataset dataset
@@ -69,9 +70,9 @@ end
 ##
 # Helper class for MiGA::TaxIndex.
 class MiGA::TaxIndexTaxon < MiGA::MiGA
-  
+
   # Instance-level
-  
+
   ##
   # Rank of the taxon.
   attr_reader :rank