miga-base 0.4.1.0 → 0.4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3c339de89529e19ad52d1d3522fbe25206045d3db3b030e4ee8c7ddf7d3c1db
-  data.tar.gz: d220804d9a9bc722496f271135da3c4415a77c84d40e49db5d39d015e13d9865
+  metadata.gz: 8d83a4e140647fdcfdd062c0f503e8ad5fefacef870fe5c8c3528e4edeb1e44f
+  data.tar.gz: f8248690dde4d6441658ab6f75181c4786ca7ad6d8d26fa83f195ebdef801c26
 SHA512:
-  metadata.gz: 27b5d98711e579c077eda211387457578e70183a30459dc6c6e917b5fc20cdbc730d649dfd58b0a2477ea019e0577c1c4993fc6d29eaee9bbdeb77a753ddb107
-  data.tar.gz: 8a391e9090bdf8dbf555fcdd456c1ae7c909a69b1f59cf5eac039d6f18ffbd0b9caf04bdcad406485f914c1c14681105d086e2b36f82039102df4d6b29baace0
+  metadata.gz: 36bd4f0122601175a09f3e61ef81618f8db323b41acda67c927bc9578b86ca1d9da4d714a0a02b91541a320cccbbb5b396da7ebe03c6b7f03b5ccd0880ea5953
+  data.tar.gz: 4874ebeabbebdcd0661ec8b7de879f075fd9847c921319192f3272861ed9f41c5a9954000ef60542164240d2dee8a01a2c752256d32323dc7ded0a5b50b2f099

data/bin/miga

@@ -4,249 +4,7 @@
 # @license Artistic-2.0
 
 $:.push File.expand_path('../../lib', __FILE__)
-
-require 'optparse'
 require 'miga'
+require 'miga/cli'
 
-##=> Global variables <=
-
-$task_desc = {
-  # Projects
-  new:      'Creates an empty MiGA project',
-  about:    'Displays information about a MiGA project',
-  plugins:  'Lists or (un)installs plugins in a MiGA project',
-  doctor:   'Performs consistency checks on a MiGA project',
-  # Datasets
-  add:      'Creates a dataset in a MiGA project',
-  get:      'Downloads a dataset from public databases into a MiGA project',
-  ncbi_get: 'Downloads all genomes in a taxon from NCBI into a MiGA project',
-  rm:       'Removes a dataset from an MiGA project',
-  find:     'Finds unregistered datasets based on result files',
-  ln:       'Link datasets (including results) from one project to another',
-  ls:       'Lists all registered datasets in an MiGA project',
-  # Results
-  add_result: 'Registers a result',
-  stats:    'Extracts statistics for the given result',
-  files:    'Lists registered files from the results of a dataset or project',
-  run:      'Executes locally one step analysis producing the given result',
-  summary:  'Generates a summary table for the statistics of all datasets',
-  next_step: 'Returns the next task to run in a dataset or project',
-  # Objects (Datasets or Projects)
-  edit:   'Edits the metadata of a dataset or project',
-  # System
-  init:     'Initialize MiGA to process new projects',
-  daemon:   'Controls the daemon of a MiGA project',
-  date:     'Returns the current date in standard MiGA format',
-  console:  'Opens an IRB console with MiGA',
-  # Taxonomy
-  tax_set:  'Registers taxonomic information for datasets',
-  tax_test: 'Returns test of taxonomic distributions for query datasets',
-  tax_index: 'Creates a taxonomy-indexed list of the datasets',
-  tax_dist: 'Estimates distributions of distance by taxonomy',
-}
-
-$task_alias = {
-  # Projects
-  create_project: :new,
-  project_info: :about,
-  # Datasets
-  create_dataset: :add,
-  download_dataset: :get,
-  unlink_dataset: :rm,
-  find_datasets: :find,
-  import_datasets: :ln,
-  list_datasets: :ls,
-  # Results
-  result_stats: :stats,
-  list_files: :files,
-  run_local: :run,
-  sum_stats: :summary,
-  next_task: :next_step,
-  # Objects
-  update_metadata: :edit,
-  # System
-  c: :console,
-  # Taxonomy
-  add_taxonomy: :tax_set,
-  test_taxonomy: :tax_test,
-  index_taxonomy: :tax_index,
-  tax_distributions: :tax_dist,
-}
-
-$task_alias.each do |nick, task|
-  $task_desc[task] = (
-    ($task_desc[task] =~ /\(alias: .*\)\./) ?
-        $task_desc[task].sub(/\)\.$/, ", #{nick}).") :
-        $task_desc[task].sub(/\.$/, " (alias: #{nick}).")
-  )
-end
-
-##=> Functions <=
-
-# OptParse banner
-def opt_banner(opt, files = false)
-  opt.banner = <<BAN
-#{$task_desc[$task]}
-
-Usage: #{$0} #{$task} [options]#{ ' FILES...' if files}
-BAN
-  opt.separator ""
-end
-
-# OptParse flags that determine the object to load
-def opt_object(opt, o, what = [:project, :dataset])
-  opt.on('-P', '--project PATH', '(Mandatory) Path to the project.'
-    ){ |v| o[:project]=v } if what.include? :project
-  opt.on('-D', '--dataset STRING',
-    (what.include?(:dataset) ? '(Mandatory) ' : '') +
-    'Name of the dataset.'){ |v| o[:dataset]=v } if what.include? :dataset or
-      what.include? :dataset_opt
-  opt.on('-D', '--dataset STRING', 'Name of the dataset.'
-    ){ |v| o[:dataset]=v } if what.include? :dataset_opt
-  opt.on('-t', '--type STRING',
-    (what.include?(:dataset_type_req) ? '(Mandatory) ' : '')+
-    'Type of dataset. Recognized types include:',
-    *MiGA::Dataset.KNOWN_TYPES.map{ |k,v| "~ #{k}: #{v[:description]}" }
-    ){ |v| o[:type]=v.to_sym } if what.include? :dataset_type or
-      what.include? :dataset_type_req
-  opt.on('-t', '--type STRING',
-    (what.include?(:project_type_req) ? '(Mandatory) ' : '') +
-    'Type of project. Recognized types include:',
-    *MiGA::Project.KNOWN_TYPES.map{ |k,v| "~ #{k}: #{v[:description]}"}
-    ){ |v| o[:type]=v.to_sym } if what.include? :project_type or
-      what.include? :project_type_req
-  opt.on('-r', '--result STRING',
-    '(Mandatory) Name of the result.',
-    'Recognized names for dataset-specific results include:',
-    *MiGA::Dataset.RESULT_DIRS.keys.map{|n| " ~ #{n}"},
-    'Recognized names for project-wide results include:',
-    *MiGA::Project.RESULT_DIRS.keys.map{|n| " ~ #{n}"}
-    ){ |v| o[:name]=v.downcase.to_sym } if what.include? :result
-  opt.on('-r', '--result STRING',
-    '(Mandatory) Name of the result. One of:',
-    *MiGA::Dataset.RESULT_DIRS.keys.map{|n| " ~ #{n}"}
-    ){ |v| o[:name]=v.downcase.to_sym } if what.include? :result_dataset
-end
-
-# OptParse flags common to all actions.
-def opt_common(opt, o)
-  opt.on('-v', '--verbose',
-    'Print additional information to STDERR.'){ o[:q]=false }
-  opt.on('-d', '--debug INT',
-    'Print debugging information to STDERR (1: debug, 2: trace).') do |v|
-    v.to_i>1 ? MiGA::MiGA.DEBUG_TRACE_ON : MiGA::MiGA.DEBUG_ON
-  end
-  opt.on('-h', '--help', 'Display this screen.') do
-    puts opt
-    exit
-  end
-  opt.separator ''
-end
-
-# OptParse flags to filter lists of datasets.
-def opt_filter_datasets(opt, o, what = [:ref, :multi, :active, :taxonomy])
-  opt.on('--[no-]ref',
-    'If set, uses only reference (or only non-reference) datasets.'
-    ){ |v| o[:ref]=v } if what.include? :ref
-  opt.on('--[no-]multi',
-    'If set, uses only multi-species (or only single-species) datasets.'
-    ){ |v| o[:multi]=v } if what.include? :multi
-  opt.on('--[no-]active',
-    'If set, uses only active (or inactive) datasets.'
-    ){ |v| o[:active]=v } if what.include? :active
-  opt.on('-t', '--taxonomy RANK:TAXON', 'Filter by taxonomy.'
-    ){ |v| o[:taxonomy]=MiGA::Taxonomy.new v } if what.include? :taxonomy
-  opt.on('-k', '--key INTEGER',
-    'Returns only the k-th dataset in the list.'){ |v| o[:key]=v.to_i }
-end
-
-def opt_require(o, req = {project: '-P', dataset: '-D'})
-  req.each do |k,v|
-    raise "#{v} is mandatory: please provide #{k}." if o[k].nil?
-  end
-end
-
-def opt_require_type(o, klass)
-  opt_require(o, type: '-t')
-  raise "Unrecognized type: #{o[:type]}." if klass.KNOWN_TYPES[o[:type]].nil?
-end
-
-# Filters datasets by keys set in +opt_filter_datasets+.
-def filter_datasets!(ds, o)
-  ds.select! { |d| d.is_ref? == o[:ref] } unless o[:ref].nil?
-  ds.select! { |d| d.is_active? == o[:active] } unless o[:active].nil?
-  ds.select! do |d|
-    o[:multi] ? d.is_multi? : d.is_nonmulti?
-  end unless o[:multi].nil?
-  ds.select! do |d|
-    (not d.metadata[:tax].nil?) and d.metadata[:tax].is_in?(o[:taxonomy])
-  end unless o[:taxonomy].nil?
-  o[:key].nil? ? ds : ds.values_at(o[:key]-1)
-end
-
-def add_metadata(o, obj)
-  o[:metadata].split(',').each do |pair|
-    (k,v) = pair.split('=')
-    case v
-      when 'true';  v = true
-      when 'false'; v = false
-      when 'nil';   v = nil
-    end
-    if k == '_step'
-      obj.metadata["_try_#{v}"] ||= 0
-      obj.metadata["_try_#{v}"]  += 1
-    end
-    obj.metadata[k] = v
-  end unless o[:metadata].nil?
-  [:type, :name, :user, :description, :comments].each do |k|
-    obj.metadata[k] = o[k] unless o[k].nil?
-  end
-  obj
-end
-
-##=> Main <=
-
-execs = $task_desc.keys.map{ |k| k.to_s }
-
-ARGV[0] = $task_alias[ARGV[0].to_sym] unless
-  ARGV[0].nil? or $task_alias[ARGV[0].to_sym].nil?
-
-case ARGV[0].to_s
-when '-v', '--version'
-  puts MiGA::MiGA.VERSION
-when '-V', '--long-version'
-  puts MiGA::MiGA.LONG_VERSION
-when '-C', '--citation'
-  puts MiGA::MiGA.CITATION
-when 'console'
-  require 'irb'
-  require 'irb/completion'
-  ARGV.shift
-  IRB.start
-when *execs
-  $task = ARGV.shift.to_sym
-  ARGV << '-h' if ARGV.empty? and not [:date, :init].include? $task
-  begin
-    load File.expand_path("../actions/#{$task}.rb", File.dirname(__FILE__))
-  rescue => err
-    $stderr.puts "Exception: #{err}\n\n"
-    err.backtrace.each { |l| $stderr.puts l + "\n" }
-    err
-  end
-else
-  print <<HELP
-
-Microbial Genomes Atlas.
-
-Usage: #{$0} {action} [options]
-
-#{ MiGA::MiGA.tabulate([:action, :description], $task_desc.to_a).join("\n") }
-
-generic options:
-    -h, --help          Display this screen.
-    -v, --version       Show MiGA version.
-    -V, --long-version  Show complete MiGA version.
-    -C, --citation      How to cite MiGA.
-
-HELP
-end
+MiGA::Cli.new(ARGV).launch

data/lib/miga/cli/action/about.rb

@@ -0,0 +1,44 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::About < MiGA::Cli::Action
+
+  def parse_cli
+    cli.defaults = {info: false, processing: false, tabular: false}
+    cli.parse do |opt|
+      cli.opt_object(opt, [:project])
+      opt.on(
+        '-p', '--processing',
+        'Print information on processing advance'
+        ){ |v| cli[:processing] = v }
+      opt.on(
+        '-m', '--metadata STRING',
+        'Print name and metadata field only'
+        ){ |v| cli[:datum] = v }
+      opt.on(
+        '--tab',
+        'Return a tab-delimited table'
+        ){ |v| cli[:tabular] = v }
+    end
+  end
+
+  def perform
+    p = cli.load_project
+    if not cli[:datum].nil?
+      v = p.metadata[cli[:datum]]
+      cli.puts v.nil? ? '?' : v
+    elsif cli[:processing]
+      keys = Project.DISTANCE_TASKS + Project.INCLADE_TASKS
+      cli.puts MiGA.tabulate([:task, :status], keys.map do |k|
+        [k, p.add_result(k, false).nil? ? 'queued' : 'done']
+      end, cli[:tabular])
+    else
+      cli.puts MiGA.tabulate([:key, :value], p.metadata.data.keys.map do |k|
+        v = p.metadata[k]
+        [k, k==:datasets ? v.size : v]
+      end, cli[:tabular])
+    end
+  end
+end

data/lib/miga/cli/action/add.rb

@@ -0,0 +1,139 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::Add < MiGA::Cli::Action
+
+  def parse_cli
+    cli.expect_files = true
+    cli.defaults = {ref: true, ignore_dups: false,
+      regexp: /^(?:.*\/)?(.+?)(?:\..*(?:[12]|Reads|Contigs))?(?i:\.f[nastq]+)?$/}
+    cli.parse do |opt|
+      opt.separator 'You can create multiple datasets with a single command; ' \
+        'simply pass all the files at the end: {FILES...}'
+      opt.separator 'If -D is passed, only one dataset will be added. ' \
+        'Otherwise, dataset names will be determined by the file paths (see -R)'
+      opt.separator ''
+      cli.opt_object(opt, [:project, :dataset_opt, :dataset_type_req])
+      opt.on(
+        '-q', '--query',
+        'Register the dataset as a query, not a reference dataset'
+        ){ |v| cli[:ref] = !v }
+      opt.on(
+        '-d', '--description STRING',
+        'Description of the dataset'
+        ){ |v| cli[:description] = v }
+      opt.on('-c', '--comments STRING',
+        'Comments on the dataset'
+        ){ |v| cli[:comments] = v }
+      opt.on('-m', '--metadata STRING',
+        'Metadata as key-value pairs separated by = and delimited by comma',
+        'Values are saved as strings except for booleans (true / false) or nil'
+        ){ |v| cli[:metadata] = v }
+      opt.on(
+        '-R', '--name-regexp REGEXP', Regexp,
+        'Regular expression indicating how to extract the name from the path',
+        "By default: '#{cli[:regexp]}'"
+        ){ |v| cli[:regexp] = v }
+      opt.on(
+        '-i', '--input-type STRING',
+        'Type of input data, one of the following:',
+        *self.class.INPUT_TYPES.map{ |k,v| "~ #{k}: #{v[0]}." }
+        ){ |v| cli[:input_type] = v.downcase.to_sym }
+      opt.on(
+        '--ignore-dups',
+        'Continue with a warning if a dataset already exists'
+        ){ |v| cli[:ignore_dups] = v }
+    end
+  end
+
+  def perform
+    p = cli.load_project
+    files = cli.files
+    file_type = nil
+    if files.empty?
+      cli.ensure_par({dataset: '-D'},
+        'dataset is mandatory (-D) unless files are provided')
+      cli.ensure_type(Dataset)
+      files = [nil]
+    else
+      raise 'Please specify input type (-i).' if cli[:input_type].nil?
+      file_type = self.class.INPUT_TYPES[cli[:input_type]]
+      raise "Unrecognized input type: #{cli[:input_type]}." if file_type.nil?
+      raise 'Some files are duplicated, files must be unique.' if
+        files.size != files.uniq.size
+      if cli[:input_type].to_s =~ /_paired$/
+        raise 'Odd number of files incompatible with input type.' if files.size.odd?
+        files = Hash[*files].to_a
+      else
+        files = files.map{ |i| [i] }
+      end
+      raise 'The dataset name (-D) can only be specified with one input file.' if
+        files.size > 1 && !cli[:dataset].nil?
+    end
+
+    cli.say 'Creating datasets:'
+    files.each do |file|
+      name = cli[:dataset]
+      if name.nil?
+        ref_file = file.is_a?(Array) ? file.first : file
+        m = cli[:regexp].match(ref_file)
+        raise "Cannot extract name from file: #{ref_file}" if m.nil? or m[1].nil?
+        name = m[1].miga_name
+      end
+      if Dataset.exist?(p, name)
+        msg = "Dataset already exists: #{name}."
+        cli[:ignore_dups] ? (warn(msg); next) : raise(msg)
+      end
+
+      cli.say "o #{name}"
+      d = Dataset.new(p, name, cli[:ref])
+      raise "Unexpected: Couldn't create dataset: #{name}." if d.nil?
+
+      unless file.nil?
+        r_dir = Dataset.RESULT_DIRS[ file_type[1] ]
+        r_path = File.expand_path("data/#{r_dir}/#{d.name}", p.path)
+        file_type[2].each_with_index do |ext, i|
+          gz = file[i] =~ /\.gz/ ? '.gz' : ''
+          FileUtils.cp(file[i], "#{r_path}#{ext}#{gz}")
+          cli.say "  file: #{file[i]}"
+        end
+        File.open("#{r_path}.done", 'w') { |f| f.print Time.now.to_s }
+      end
+
+      d = cli.add_metadata(d)
+      d.save
+      p.add_dataset(name)
+      res = d.first_preprocessing(true)
+      cli.say "  result: #{res}"
+    end
+  end
+
+  @@INPUT_TYPES = {
+    raw_reads_single:
+      ['Single raw reads in a single FastQ file',
+        :raw_reads, %w[.1.fastq]],
+    raw_reads_paired:
+      ['Paired raw reads in two FastQ files',
+        :raw_reads, %w[.1.fastq .2.fastq]],
+    trimmed_reads_single:
+      ['Single trimmed reads in a single FastA file',
+        :trimmed_fasta, %w[.SingleReads.fa]],
+    trimmed_reads_paired:
+      ['Paired trimmed reads in two FastA files',
+        :trimmed_fasta, %w[.1.fasta .2.fasta]],
+    trimmed_reads_interleaved:
+      ['Paired trimmed reads in a single FastA file',
+        :trimmed_fasta, %w[.CoupledReads.fa]],
+    assembly:
+      ['Assembled contigs or scaffolds in FastA format',
+        :assembly, %w[.LargeContigs.fna]]
+  }
+
+  class << self
+    def INPUT_TYPES
+      @@INPUT_TYPES
+    end
+  end
+end

data/lib/miga/cli/action/add_result.rb

@@ -0,0 +1,26 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::AddResult < MiGA::Cli::Action
+
+  def parse_cli
+    cli.defaults = {force: false}
+    cli.parse do |opt|
+      cli.opt_object(opt, [:project, :dataset_opt, :result])
+      opt.on(
+        '-f', '--force',
+        'Force re-indexing of the result even if it\'s already registered'
+        ){ |v| cli[:force] = v }
+    end
+  end
+
+  def perform
+    cli.ensure_par(result: '-r')
+    obj = cli.load_project_or_dataset
+    cli.say "Registering result: #{cli[:result]}"
+    r = obj.add_result(cli[:result], true, force: cli[:force])
+    raise "Cannot add result, incomplete expected files." if r.nil?
+  end
+end

data/lib/miga/cli/action/console.rb

@@ -0,0 +1,19 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::Console < MiGA::Cli::Action
+
+  def parse_cli
+  end
+
+  def perform
+    require 'irb'
+    require 'irb/completion'
+    IRB.start
+  end
+
+  def empty_action
+  end
+end

data/lib/miga/cli/action/daemon.rb

@@ -0,0 +1,74 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+require 'miga/daemon'
+
+class MiGA::Cli::Action::Daemon < MiGA::Cli::Action
+
+  def parse_cli
+    cli.defaults = {daemon_opts: []}
+    cli.expect_operation = true
+    cli.parse do |opt|
+      opt.separator 'Available operations:'
+      { start:   'Start an instance of the application.',
+        stop:    'Start an instance of the application.',
+        restart: 'Stop all instances and restart them afterwards.',
+        reload:  'Send a SIGHUP to all instances of the application.',
+        run:     'Start the application and stay on top.',
+        zap:     'Set the application to a stopped state.',
+        status:  'Show status (PID) of application instances.'
+      }.each{ |k,v| opt.separator sprintf '    %*s%s', -33, k, v }
+      opt.separator ''
+
+      opt.separator 'MiGA options:'
+      cli.opt_object(opt, [:project])
+      opt.on(
+        '--shutdown-when-done',
+        'Exit the daemon when all processing is done',
+        'Otherwise, it will stay idle awaiting for new data (default)'
+        ){ |v| cli[:shutdown_when_done] = v }
+      opt.on(
+        '--latency INT',
+        'Number of seconds the daemon will be sleeping'
+        ){ |v| cli[:latency]=v.to_i }
+      opt.on(
+        '--max-jobs INT',
+        'Maximum number of jobs to use simultaneously'
+        ){ |v| cli[:maxjobs]=v.to_i }
+      opt.on(
+        '--ppn INT',
+        'Maximum number of cores to use in a single job'
+        ){ |v| cli[:ppn]=v.to_i }
+      cli.opt_common(opt)
+
+      opt.separator 'Daemon options:'
+      opt.on(
+        '-t', '--ontop',
+        'Stay on top (does not daemonize)'
+        ){ cli[:daemon_opts] << '-t' }
+      opt.on(
+        '-f', '--force',
+        'Force operation'
+        ){ cli[:daemon_opts] << '-f' }
+      opt.on(
+        '-n', '--no_wait',
+        'Do not wait for processes to stop'
+        ){ cli[:daemon_opts] << '-n' }
+      opt.on(
+        '--shush',
+        'Silence the daemon'
+        ){ cli[:daemon_opts] << '--shush' }
+      opt.separator ''
+    end
+  end
+
+  def perform
+    p = cli.load_project
+    d = MiGA::Daemon.new(p)
+    [:latency, :maxjobs, :ppn, :shutdown_when_done].each do |k|
+      d.runopts(k, cli[k]) unless cli[k].nil?
+    end
+    d.daemon(cli.operation, cli[:daemon_opts])
+  end
+end

data/lib/miga/cli/action/date.rb

@@ -0,0 +1,18 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::Date < MiGA::Cli::Action
+
+  def parse_cli
+    cli.parse { |_| }
+  end
+
+  def perform
+    puts Time.now.to_s
+  end
+
+  def empty_action
+  end
+end