miga-base 0.4.3.0 → 0.5.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f547a13f78444c6d3f7149bb111d3a4444af885ef085fef913c9516fac706215
-  data.tar.gz: 0ae132567bd8e677bfc7e0bafaaeb28138499b1e720302e47f711364f790f020
+  metadata.gz: e370d282f1b28480765e1b91fcb7d8921d12baa31d22db1318975a1c2a79e19a
+  data.tar.gz: e7fb3941fd3381e0e9696a2c577aeb157657335e56434e7c6d6650be7ba45e98
 SHA512:
-  metadata.gz: f174c56c991a3055d347f0450aca196e8c6f4813bb21d18f0bcaca528df40fe76e4ea497ff36d58886817d26fbcb2281459a5866260040de975022e572f6869b
-  data.tar.gz: ff4777abf6dab9bd2800e86533840817a77d01b3724ed51d9a77d0dac4ec1506ca407420b615f5e16fd3548af5b58fd56a3e922a448f2d75052d814cdb504ae3
+  metadata.gz: 4642a212e1b4021e211fd144b515ff49e9ddb7a9b2292430553307a7ae165e4d8d5e6fd8426757f15ea6e70f4c3efbb055e0439497172cc1f91186d522c82635
+  data.tar.gz: 8d5d3ded3c03e56505572102110a4bca4b84d06b2e73bcf208856610a4cd6e60092ce6d54d47dcef2c8acf85f1cce5f8461097e8699344df6738ed8493215112

data/README.md

@@ -23,7 +23,7 @@ For additional information on MiGA, visit:
 If you're like us, you probably want to see sofware in action from the get go.
 You have two options:
 
-1. Get a peak on MiGA using [MiGA Online][miga-online].
+1. Get a peek on MiGA using [MiGA Online][miga-online].
 2. Install the [requirements](manual/part2/requirements.md) and follow the
   [installation instructions](manual/part2/installation.md). Once you have MiGA
   installed, you can [deploy some examples](manual/part4.md).

data/lib/miga/cli.rb

@@ -9,7 +9,11 @@ require 'optparse'
 class MiGA::Cli < MiGA::MiGA
 
   require 'miga/cli/base'
+  require 'miga/cli/opt_helper'
+  require 'miga/cli/objects_helper'
   require 'miga/cli/action'
+  include MiGA::Cli::OptHelper
+  include MiGA::Cli::ObjectsHelper
 
   ##
   # Task to execute, a symbol
@@ -71,41 +75,68 @@ class MiGA::Cli < MiGA::MiGA
   end
 
   ##
-  # Send +par+ to $stdout, ensuring new line at the end
+  # Print +par+, ensuring new line at the end.
+  # If the first parameter is +IO+, the output is sent there,
+  # otherwise it's sent to +$stdout+
   def puts(*par)
-    $stdout.puts(*par)
+    io = par.first.is_a?(IO) ? par.shift : $stdout
+    io.puts(*par)
   end
 
   ##
-  # Send +par+ to $stdout as is
+  # Print +par+.
+  # If the first parameter is +IO+, the output is sent there,
+  # otherwise it's sent to +$stdout+
   def print(*par)
-    $stdout.print(*par)
+    io = par.first.is_a?(IO) ? par.shift : $stdout
+    io.print(*par)
   end
 
   ##
-  # Display a table with headers +header+ and contents +values+, both Array
-  def table(header, values)
-    self.puts MiGA.tabulate(header, values, self[:tabular])
+  # Display a table with headers +header+ and contents +values+, both Array.
+  # The output is printed to +io+
+  def table(header, values, io = $stdout)
+    self.puts(io, MiGA.tabulate(header, values, self[:tabular]))
   end
 
   ##
-  # Send +par+ to $stderr (ensuring new line at the end), iff --verbose.
+  # Print +par+ ensuring new line at the end, iff --verbose.
   # Date/time each line.
+  # If the first parameter is +IO+, the output is sent there,
+  # otherwise it's sent to +$stderr+
   def say(*par)
     return unless self[:verbose]
     par.map! { |i| "[#{Time.now}] #{i}" }
-    $stderr.puts(*par)
+    io = par.first.is_a?(IO) ? par.shift : $stderr
+    io.puts(*par)
   end
 
   ##
-  # Reports the advance of a task at +step+ (String), the +n+ out of +total+
+  # Reports the advance of a task at +step+ (String), the +n+ out of +total+.
+  # The advance is reported in powers of 1,024 if +bin+ is true, or powers of
+  # 1,000 otherwise.
   # The report goes to $stderr iff --verborse
-  def advance(step, n = 0, total = nil)
+  def advance(step, n = 0, total = nil, bin = true)
     return unless self[:verbose]
-    adv = total.nil? ? '' : ('%.1f%% (%d/%d)' % [n/total, n, total])
+    adv = total.nil? ? '' :
+      ('%.1f%% (%s/%s)' % [100 * n / total,
+        num_suffix(n, bin), num_suffix(total, bin)])
     $stderr.print("[%s] %s %s    \r" % [Time.now, step, adv])
   end
 
+  def num_suffix(n, bin = false)
+    p = ''
+    {T: 4, G: 3, M: 2, K: 1}.each do |k,x|
+      v = (bin ? 1024 : 1e3) ** x
+      if n > v
+        n = '%.1f' % (n / v)
+        p = k
+        break
+      end
+    end
+    "#{n}#{p}"
+  end
+
   ##
   # Ask a question +question+ to the user (requires +#interactive = true+)
   # The +default+ is used if the answer is empty
@@ -188,135 +219,6 @@ class MiGA::Cli < MiGA::MiGA
     end
   end
 
-  ##
-  # Send MiGA's banner to OptionParser +opt+
-  def banner(opt)
-    usage = "Usage: miga #{action.name}"
-    usage += ' {operation}' if expect_operation
-    usage += ' [options]'
-    usage += ' {FILES...}' if expect_files
-    opt.banner = "\n#{task_description}\n\n#{usage}\n"
-    opt.separator ''
-  end
-
-  ##
-  # Common options at the end of most actions, passed to OptionParser +opt+
-  # No action is performed if +#opt_common = false+ is passed
-  # Executes only once, unless +#opt_common = true+ is passed between calls
-  def opt_common(opt)
-    return unless @opt_common
-    opt.on(
-      '--auto',
-      'Accept all defaults as answers'
-      ){ |v| cli[:auto] = v } if interactive
-    opt.on(
-      '-v', '--verbose',
-      'Print additional information to STDERR'
-      ){ |v| self[:verbose] = v }
-    opt.on(
-      '-d', '--debug INT', Integer,
-      'Print debugging information to STDERR (1: debug, 2: trace)'
-      ){ |v| (v > 1) ? MiGA.DEBUG_TRACE_ON : MiGA.DEBUG_ON }
-    opt.on(
-      '-h', '--help',
-      'Display this screen'
-      ){ puts opt ; exit }
-    opt.separator ''
-    self.opt_common = false
-  end
-
-  ##
-  # Options to load an object passed to OptionParser +opt+, as determined
-  # by +what+ an Array with any combination of:
-  # - :project To require a project
-  # - :dataset To require a dataset
-  # - :dataset_opt To allow (optionally) a dataset
-  # - :dataset_type To allow (optionally) a type of dataset
-  # - :dataset_type_req To require a type of dataset
-  # - :project_type To allow (optionally) a type of project
-  # - :project_type_req To require a type of project
-  # - :result To require a type of project or dataset result
-  # - :result_dataset To require a type of dataset result
-  # - :result_project To require a type of project result
-  def opt_object(opt, what = [:project, :dataset])
-    opt.on(
-      '-P', '--project PATH',
-      '(Mandatory) Path to the project'
-      ){ |v| self[:project] = v } if what.include? :project
-    opt.on(
-      '-D', '--dataset STRING',
-      (what.include?(:dataset) ? '(Mandatory) ' : '') + 'Name of the dataset'
-      ){ |v| self[:dataset] = v } if what.include? :dataset or
-        what.include? :dataset_opt
-    opt.on(
-      '-D', '--dataset STRING',
-      'Name of the dataset'
-      ){ |v| self[:dataset] = v } if what.include? :dataset_opt
-    opt.on(
-      '-t', '--type STRING',
-      (what.include?(:dataset_type_req) ? '(Mandatory) ' : '') +
-      'Type of dataset. Recognized types include:',
-      *Dataset.KNOWN_TYPES.map{ |k,v| "~ #{k}: #{v[:description]}" }
-      ){ |v| self[:type] = v.downcase.to_sym } if what.include? :dataset_type or
-        what.include? :dataset_type_req
-    opt.on(
-      '-t', '--type STRING',
-      (what.include?(:project_type_req) ? '(Mandatory) ' : '') +
-      'Type of project. Recognized types include:',
-      *Project.KNOWN_TYPES.map{ |k,v| "~ #{k}: #{v[:description]}"}
-      ){ |v| self[:type] = v.downcase.to_sym } if what.include? :project_type or
-        what.include? :project_type_req
-    opt.on(
-      '-r', '--result STRING',
-      '(Mandatory) Name of the result',
-      'Recognized names for dataset-specific results include:',
-      *Dataset.RESULT_DIRS.keys.map{|n| " ~ #{n}"},
-      'Recognized names for project-wide results include:',
-      *Project.RESULT_DIRS.keys.map{|n| " ~ #{n}"}
-      ){ |v| self[:result] = v.downcase.to_sym } if what.include? :result
-    opt.on(
-      '-r', '--result STRING',
-      '(Mandatory) Name of the result, one of:',
-      *Dataset.RESULT_DIRS.keys.map{|n| " ~ #{n}"}
-      ){ |v| self[:result] = v.downcase.to_sym } if what.include? :result_dataset
-    opt.on(
-      '-r', '--result STRING',
-      '(Mandatory) Name of the result, one of:',
-      *Project.RESULT_DIRS.keys.map{|n| " ~ #{n}"}
-      ){ |v| self[:result] = v.downcase.to_sym } if what.include? :result_project
-  end
-
-  ##
-  # Options to filter a list of datasets passed to OptionParser +opt+,
-  # as determined by +what+ an Array with any combination of:
-  # - :ref To filter by reference (--ref) or query (--no-ref)
-  # - :multi To filter by multiple (--multi) or single (--no-multi) species
-  # - :active To filter by active (--active) or inactive (--no-active)
-  # - :taxonomy To filter by taxonomy (--taxonomy)
-  # The "k-th" filter (--dataset-k) is always included
-  def opt_filter_datasets(opt, what = [:ref, :multi, :active, :taxonomy])
-    opt.on(
-      '--[no-]ref',
-      'Use only reference (or only non-reference) datasets'
-      ){ |v| self[:ref] = v } if what.include? :ref
-    opt.on(
-      '--[no-]multi',
-      'Use only multi-species (or only single-species) datasets'
-      ){ |v| self[:multi] = v } if what.include? :multi
-    opt.on(
-      '--[no-]active',
-      'Use only active (or inactive) datasets'
-      ){ |v| self[:active] = v } if what.include? :active
-    opt.on(
-      '-t', '--taxonomy RANK:TAXON',
-      'Filter by taxonomy'
-      ){ |v| self[:taxonomy] = Taxonomy.new(v) } if what.include? :taxonomy
-    opt.on(
-      '--dataset-k INTEGER', Integer,
-      'Use only the k-th dataset in the list'
-      ){ |v| self[:dataset_k] = v }
-  end
-
   ##
   # Ensure that these parameters have been passed to the CLI, as defined by
   # +par+, a Hash with object names as keys and parameter flag as values.
@@ -336,88 +238,6 @@ class MiGA::Cli < MiGA::MiGA
     end
   end
 
-  ##
-  # Get the project defined in the CLI by parameter +name+ and +flag+
-  def load_project(name = :project, flag = '-P')
-    return @objects[name] unless @objects[name].nil?
-    ensure_par(name => flag)
-    say "Loading project: #{self[name]}"
-    @objects[name] = Project.load(self[name])
-    raise "Cannot load project: #{self[name]}" if @objects[name].nil?
-    @objects[name]
-  end
-
-  ##
-  # Load the dataset defined in the CLI
-  # If +silent=true+, it allows failures silently
-  def load_dataset(silent = false)
-    return @objects[:dataset] unless @objects[:dataset].nil?
-    ensure_par(dataset: '-D')
-    @objects[:dataset] = load_project.dataset(self[:dataset])
-    if !silent && @objects[:dataset].nil?
-      raise "Cannot load dataset: #{self[:dataset]}"
-    end
-    return @objects[:dataset]
-  end
-
-  ##
-  # Load an a project or (if defined) a dataset
-  def load_project_or_dataset
-    self[:dataset].nil? ? load_project : load_dataset
-  end
-
-  ##
-  # Load and filter a list of datasets as requested in the CLI
-  # If +silent=true+, it allows failures silently
-  def load_and_filter_datasets(silent = false)
-    return @objects[:filtered_datasets] unless @objects[:filtered_datasets].nil?
-    say 'Listing datasets'
-    ds = self[:dataset].nil? ?
-      load_project.datasets : [load_dataset(silent)].compact
-    ds.select! { |d| d.is_ref? == self[:ref] } unless self[:ref].nil?
-    ds.select! { |d| d.is_active? == self[:active] } unless self[:active].nil?
-    ds.select! do |d|
-      self[:multi] ? d.is_multi? : d.is_nonmulti?
-    end unless self[:multi].nil?
-    ds.select! do |d|
-      (not d.metadata[:tax].nil?) && d.metadata[:tax].in?(self[:taxonomy])
-    end unless self[:taxonomy].nil?
-    ds = ds.values_at(self[:dataset_k]-1) unless self[:dataset_k].nil?
-    @objects[:filtered_datasets] = ds
-  end
-
-  def load_result
-    return @objects[:result] unless @objects[:result].nil?
-    ensure_par(result: '-r')
-    obj = load_project_or_dataset
-    if obj.class.RESULT_DIRS[self[:result]].nil?
-      raise "Unsupported result for #{obj.class.to_s.gsub(/.*::/,'')}: #{self[:result]}"
-    end
-    r = obj.add_result(self[:result], false)
-    raise "Cannot load result: #{self[:result]}" if r.nil?
-    @objects[:result] = r
-  end
-
-  def add_metadata(obj, cli = self)
-    cli[:metadata].split(',').each do |pair|
-      (k,v) = pair.split('=')
-      case v
-        when 'true';  v = true
-        when 'false'; v = false
-        when 'nil';   v = nil
-      end
-      if k == '_step'
-        obj.metadata["_try_#{v}"] ||= 0
-        obj.metadata["_try_#{v}"]  += 1
-      end
-      obj.metadata[k] = v
-    end unless cli[:metadata].nil?
-    [:type, :name, :user, :description, :comments].each do |k|
-      obj.metadata[k] = cli[k] unless cli[k].nil?
-    end
-    obj
-  end
-
   ##
   # Task description
   def task_description

data/lib/miga/cli/action/add.rb

@@ -7,8 +7,10 @@ class MiGA::Cli::Action::Add < MiGA::Cli::Action
 
   def parse_cli
     cli.expect_files = true
-    cli.defaults = {ref: true, ignore_dups: false,
-      regexp: /^(?:.*\/)?(.+?)(?:\..*(?:[12]|Reads|Contigs))?(?i:\.f[nastq]+)?$/}
+    cli.defaults = {
+      ref: true, ignore_dups: false,
+      regexp: MiGA::Cli.FILE_REGEXP
+    }
     cli.parse do |opt|
       opt.separator 'You can create multiple datasets with a single command; ' \
         'simply pass all the files at the end: {FILES...}'
@@ -19,92 +21,53 @@ class MiGA::Cli::Action::Add < MiGA::Cli::Action
       opt.on(
         '-q', '--query',
         'Register the dataset as a query, not a reference dataset'
-        ){ |v| cli[:ref] = !v }
+      ) { |v| cli[:ref] = !v }
       opt.on(
         '-d', '--description STRING',
         'Description of the dataset'
-        ){ |v| cli[:description] = v }
-      opt.on('-c', '--comments STRING',
+      ) { |v| cli[:description] = v }
+      opt.on(
+        '-c', '--comments STRING',
         'Comments on the dataset'
-        ){ |v| cli[:comments] = v }
-      opt.on('-m', '--metadata STRING',
+      ) { |v| cli[:comments] = v }
+      opt.on(
+        '-m', '--metadata STRING',
         'Metadata as key-value pairs separated by = and delimited by comma',
         'Values are saved as strings except for booleans (true / false) or nil'
-        ){ |v| cli[:metadata] = v }
+      ) { |v| cli[:metadata] = v }
       opt.on(
         '-R', '--name-regexp REGEXP', Regexp,
         'Regular expression indicating how to extract the name from the path',
         "By default: '#{cli[:regexp]}'"
-        ){ |v| cli[:regexp] = v }
+      ) { |v| cli[:regexp] = v }
+      opt.on(
+        '--prefix STRING',
+        'Prefix to all the dataset names'
+      ) { |v| cli[:prefix] = v }
       opt.on(
         '-i', '--input-type STRING',
         'Type of input data, one of the following:',
-        *self.class.INPUT_TYPES.map{ |k,v| "~ #{k}: #{v[0]}." }
-        ){ |v| cli[:input_type] = v.downcase.to_sym }
+        *self.class.INPUT_TYPES.map{ |k,v| "~ #{k}: #{v[0]}" }
+      ) { |v| cli[:input_type] = v.downcase.to_sym }
       opt.on(
         '--ignore-dups',
         'Continue with a warning if a dataset already exists'
-        ){ |v| cli[:ignore_dups] = v }
+      ) { |v| cli[:ignore_dups] = v }
     end
   end
 
   def perform
     p = cli.load_project
-    files = cli.files
-    file_type = nil
-    if files.empty?
-      cli.ensure_par({dataset: '-D'},
-        'dataset is mandatory (-D) unless files are provided')
-      cli.ensure_type(Dataset)
-      files = [nil]
-    else
-      raise 'Please specify input type (-i).' if cli[:input_type].nil?
-      file_type = self.class.INPUT_TYPES[cli[:input_type]]
-      raise "Unrecognized input type: #{cli[:input_type]}." if file_type.nil?
-      raise 'Some files are duplicated, files must be unique.' if
-        files.size != files.uniq.size
-      if cli[:input_type].to_s =~ /_paired$/
-        raise 'Odd number of files incompatible with input type.' if files.size.odd?
-        files = Hash[*files].to_a
-      else
-        files = files.map{ |i| [i] }
-      end
-      raise 'The dataset name (-D) can only be specified with one input file.' if
-        files.size > 1 && !cli[:dataset].nil?
-    end
+    files, file_type = get_files_and_type
 
     cli.say 'Creating datasets:'
     files.each do |file|
-      name = cli[:dataset]
-      if name.nil?
-        ref_file = file.is_a?(Array) ? file.first : file
-        m = cli[:regexp].match(ref_file)
-        raise "Cannot extract name from file: #{ref_file}" if m.nil? or m[1].nil?
-        name = m[1].miga_name
-      end
-      if Dataset.exist?(p, name)
-        msg = "Dataset already exists: #{name}."
-        cli[:ignore_dups] ? (warn(msg); next) : raise(msg)
-      end
-
-      cli.say "o #{name}"
-      d = Dataset.new(p, name, cli[:ref])
-      raise "Unexpected: Couldn't create dataset: #{name}." if d.nil?
-
-      unless file.nil?
-        r_dir = Dataset.RESULT_DIRS[ file_type[1] ]
-        r_path = File.expand_path("data/#{r_dir}/#{d.name}", p.path)
-        file_type[2].each_with_index do |ext, i|
-          gz = file[i] =~ /\.gz/ ? '.gz' : ''
-          FileUtils.cp(file[i], "#{r_path}#{ext}#{gz}")
-          cli.say "  file: #{file[i]}"
-        end
-        File.open("#{r_path}.done", 'w') { |f| f.print Time.now.to_s }
-      end
-
+      d = create_dataset(file, p)
+      next if d.nil?
+      copy_file_to_project(file, file_type, d, p)
       d = cli.add_metadata(d)
       d.save
-      p.add_dataset(name)
+      p.add_dataset(d.name)
       res = d.first_preprocessing(true)
       cli.say "  result: #{res}"
     end
@@ -136,4 +99,70 @@ class MiGA::Cli::Action::Add < MiGA::Cli::Action
       @@INPUT_TYPES
     end
   end
+
+  private
+
+  def get_files_and_type
+    files = cli.files
+    file_type = nil
+    if files.empty?
+      cli.ensure_par({dataset: '-D'},
+        'dataset is mandatory (-D) unless files are provided')
+      cli.ensure_type(Dataset)
+      files = [nil]
+    else
+      raise 'Please specify input type (-i).' if cli[:input_type].nil?
+      file_type = self.class.INPUT_TYPES[cli[:input_type]]
+      raise "Unrecognized input type: #{cli[:input_type]}." if file_type.nil?
+      raise 'Some files are duplicated, files must be unique.' if
+        files.size != files.uniq.size
+      if cli[:input_type].to_s =~ /_paired$/
+        if files.size.odd?
+          raise 'Odd number of files incompatible with input type.'
+        end
+        files = Hash[*files].to_a
+      else
+        files = files.map{ |i| [i] }
+      end
+      if files.size > 1 && !cli[:dataset].nil?
+        raise 'The dataset name (-D) can only be specified with one input file.'
+      end
+    end
+    [files, file_type]
+  end
+
+  def create_dataset(file, p)
+    name = cli[:dataset]
+    if name.nil?
+      ref_file = file.is_a?(Array) ? file.first : file
+      m = cli[:regexp].match(ref_file)
+      raise "Cannot extract name from file: #{ref_file}" if m.nil? or m[1].nil?
+      name = cli[:prefix].to_s + m[1].miga_name
+    end
+    if Dataset.exist?(p, name)
+      msg = "Dataset already exists: #{name}."
+      if cli[:ignore_dups]
+        warn(msg)
+        return nil
+      else
+        raise(msg)
+      end
+    end
+    cli.say "o #{name}"
+    d = Dataset.new(p, name, cli[:ref])
+    raise "Unexpected: Couldn't create dataset: #{name}." if d.nil?
+    d
+  end
+
+  def copy_file_to_project(file, file_type, d, p)
+    return if file.nil?
+    r_dir = Dataset.RESULT_DIRS[ file_type[1] ]
+    r_path = File.expand_path("data/#{r_dir}/#{d.name}", p.path)
+    file_type[2].each_with_index do |ext, i|
+      gz = file[i] =~ /\.gz/ ? '.gz' : ''
+      FileUtils.cp(file[i], "#{r_path}#{ext}#{gz}")
+      cli.say "  file: #{file[i]}"
+    end
+    File.open("#{r_path}.done", 'w') { |f| f.print Time.now.to_s }
+  end
 end