miga-base 0.2.0.9 → 0.2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ae747bf6d7acc800354df0bb0ca0f39d7495cd95
-  data.tar.gz: 066b442431e8976c848cc6be009a21d5566ab6d9
+  metadata.gz: f914d4f05c7c1cb97b6036fa1af3bb5c6e95c341
+  data.tar.gz: 3812c256d78cbe8125ee5aa14a1785c3de57918f
 SHA512:
-  metadata.gz: 56cdbe0f216acab5ddf14cd672e92fd282d161ff03b7a037a37f178cdd14082b7958f80fdbd0c4ad8418b3c21a6fd705457cedd0adbd4929225a297dab2a1003
-  data.tar.gz: 85e068491a8c34194ff2bb9b9c721a97b830c1bc607c588b9043a026400cd39f757dd499471e03555c1ebfe4acea7fc9d6269ae6b50e54ebf40eda67b56eebec
+  metadata.gz: cdc6692724bb6b7c39a313c35eaabe51405d104d8af465618983a83601435af66ca1f7eca0a3e9fbc19b24ce13cbde8f9962beaf643ad1f9e3970bb10067f671
+  data.tar.gz: 45e19af2b3e14e225c9f6a89193b0c400fe8ac685296314b4320885a563ab2b3c4472563a45f98c5a7d15edd74573cf1f9176a59905a2764a809d6bc3850b90d

data/Rakefile

@@ -5,6 +5,9 @@ SOURCES = FileList["lib/**/*.rb"]
 desc "Default Task"
 task :default => "test:all"
 
+desc "Default tests"
+task :test => "test:all"
+
 namespace :test do
   desc "All tests"
   Rake::TestTask.new(:all) do |t|

data/actions/add_result.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+# @package MiGA
+# @license Artistic-2.0
+
+o = {q:true}
+opts = OptionParser.new do |opt|
+  opt_banner(opt)
+  opt_object(opt, o, [:project, :dataset_opt])
+  opt.on("-r", "--result STRING",
+    "(Mandatory) Name of the result to add.",
+    "Recognized names for dataset-specific results include:",
+    *MiGA::Dataset.RESULT_DIRS.keys.map{|n| " ~ #{n}"},
+    "Recognized names for project-wide results include:",
+    *MiGA::Project.RESULT_DIRS.keys.map{|n| " ~ #{n}"}){ |v| o[:name]=v }
+  opt_common(opt, o)
+end.parse!
+
+##=> Main <=
+opts.parse!
+opt_require(o, project:"-P", name:"-r")
+
+$stderr.puts "Loading project." unless o[:q]
+p = MiGA::Project.load(o[:project])
+raise "Impossible to load project: #{o[:project]}" if p.nil?
+
+$stderr.puts "Registering result." unless o[:q]
+if o[:dataset].nil?
+  r = p.add_result o[:name].to_sym
+else
+  d = p.dataset(o[:dataset])
+  r = d.add_result o[:name].to_sym
+end
+
+raise "Cannot add result, incomplete expected files." if r.nil?
+
+$stderr.puts "Done." unless o[:q]

data/actions/add_taxonomy.rb

@@ -0,0 +1,63 @@
+#!/usr/bin/env ruby
+
+# @package MiGA
+# @license Artistic-2.0
+
+o = {q:true}
+OptionParser.new do |opt|
+  opt_banner(opt)
+  opt_object(opt, o, [:project, :dataset_opt])
+  opt.on("-s", "--tax-string STRING",
+    "String corresponding to the taxonomy of the dataset.",
+    "The MiGA format of string taxonomy is a space-delimited",
+    "set of 'rank:name' pairs."){ |v| o[:taxstring]=v }
+  opt.on("-t", "--tax-file PATH",
+    "(Mandatory unless -D and -s are provided) Tab-delimited file containing",
+    "datasets taxonomy.  Each row corresponds to a datasets and each column",
+    "corresponds to a rank.  The first row must be a header with the rank ",
+    "names, and the first column must contain dataset names."
+    ){ |v| o[:taxfile]=v }
+  opt_common(opt, o)
+end.parse!
+
+##=> Main <=
+opt_require(o, project:"-P")
+raise "-D is mandatory unless -t is provided." if
+  o[:dataset].nil? and o[:taxfile].nil?
+raise "-s is mandatory unless -t is provided." if
+  o[:taxstring].nil? and o[:taxfile].nil?
+
+$stderr.puts "Loading project." unless o[:q]
+p = MiGA::Project.load(o[:project])
+raise "Impossible to load project: #{o[:project]}" if p.nil?
+
+if not o[:taxfile].nil?
+  $stderr.puts "Reading tax-file and registering taxonomy." unless o[:q]
+  tfh = File.open(o[:taxfile], "r")
+  header = nil
+  tfh.each_line do |ln|
+    next if ln =~ /^\s*?$/
+    r = ln.chomp.split(/\t/, -1)
+    dn = r.shift
+    if header.nil?
+      header = r
+      next
+    end
+    d = p.dataset dn
+    if d.nil?
+      warn "Impossible to find dataset at line #{$.}: #{dn}. Ignoring..."
+      next
+    end
+    d.metadata[:tax] = MiGA::Taxonomy.new(r, header)
+    d.save
+    $stderr.puts " #{d.name} registered." unless o[:q]
+  end
+  tfh.close
+else
+  $stderr.puts "Registering taxonomy." unless o[:q]
+  d = p.dataset o[:dataset]
+  d.metadata[:tax] = MiGA::Taxonomy.new(o[:taxstring])
+  d.save
+end
+
+$stderr.puts "Done." unless o[:q]

data/actions/create_dataset.rb

@@ -0,0 +1,49 @@
+#!/usr/bin/env ruby
+
+# @package MiGA
+# @license Artistic-2.0
+
+o = {q:true, ref:true, update:false}
+OptionParser.new do |opt|
+  opt_banner(opt)
+  opt_object(opt, o, [:project, :dataset, :dataset_type])
+  opt.on("-q", "--query",
+    "If set, the dataset is registered as a query, not a reference dataset."
+    ){ |v| o[:ref]=!v }
+  opt.on("-d", "--description STRING",
+    "Description of the dataset."){ |v| o[:description]=v }
+  opt.on("-u", "--user STRING",
+    "Owner of the dataset."){ |v| o[:user]=v }
+  opt.on("-c", "--comments STRING",
+    "Comments on the dataset."){ |v| o[:comments]=v }
+  opt.on("--update",
+    "Updates the dataset if it already exists."){ o[:update]=true }
+  opt_common(opt, o)
+end.parse!
+
+##=> Main <=
+opt_require(o)
+
+$stderr.puts "Loading project." unless o[:q]
+p = MiGA::Project.load(o[:project])
+raise "Impossible to load project: #{o[:project]}" if p.nil?
+
+raise "Dataset already exists, aborting." unless
+  o[:update] or not MiGA::Dataset.exist?(p, o[:dataset])
+$stderr.puts "Loading dataset." unless o[:q]
+d = o[:update] ? p.dataset(o[:dataset]) :
+  MiGA::Dataset.new(p, o[:dataset], o[:ref], {})
+raise "Dataset does not exist." if d.nil?
+[:type, :description, :user, :comments].each do |k|
+  d.metadata[k]=o[k] unless o[k].nil?
+end
+
+if o[:update]
+  d.save
+else
+  p.add_dataset(o[:dataset])
+end
+res = d.first_preprocessing(true)
+$stderr.puts "- #{res}" unless o[:q]
+
+$stderr.puts "Done." unless o[:q]

data/actions/create_project.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+
+# @package MiGA
+# @license Artistic-2.0
+
+o = {q:true, update:false}
+OptionParser.new do |opt|
+  opt_banner(opt)
+  opt_object(opt, o, [:project, :project_type])
+  opt.on("-n", "--name STRING",
+    "Name of the project."){ |v| o[:name]=v }
+  opt.on("-d", "--description STRING",
+    "Description of the project."){ |v| o[:description]=v }
+  opt.on("-u", "--user STRING", "Owner of the project."){ |v| o[:user]=v }
+  opt.on("-c", "--comments STRING",
+    "Comments on the project."){ |v| o[:comments]=v }
+  opt.on("--update",
+    "Updates the project if it already exists."){ o[:update]=true }
+  opt_common(opt, o)
+end.parse!
+
+##=> Main <=
+opt_require(o, project:"-P")
+
+unless File.exist? "#{ENV["HOME"]}/.miga_rc" and
+      File.exist? "#{ENV["HOME"]}/.miga_daemon.json"
+  puts "You must initialize MiGA before creating the first project.\n" +
+    "Do you want to initialize MiGA now? (yes / no)"
+  `'#{File.dirname(__FILE__)}/../scripts/init.bash'` if
+    $stdin.gets.chomp == 'yes'
+end
+
+$stderr.puts "Creating project." unless o[:q]
+raise "Project already exists, aborting." unless
+  o[:update] or not MiGA::Project.exist? o[:project]
+p = MiGA::Project.new(o[:project], o[:update])
+# The following check is redundant with MiGA::Project#create,
+# but allows upgrading projects from (very) early code versions
+o[:name] = File.basename(p.path) if
+  o[:update] and o[:name].nil?
+[:name, :description, :user, :comments, :type].each do |k|
+  p.metadata[k] = o[k] unless o[k].nil?
+end
+p.save
+
+$stderr.puts "Done." unless o[:q]

data/actions/daemon.rb

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+
+# @package MiGA
+# @license Artistic-2.0
+
+require "miga/daemon"
+
+task = ARGV.shift unless ["-h","--help"].include? ARGV.first
+ARGV << "-h" if ARGV.empty?
+o = {q:true, daemon_opts:[]}
+OptionParser.new do |opt|
+  opt_banner(opt)
+  opt.separator "task:"
+  { start:  "Start an instance of the application.",
+    stop:    "Start an instance of the application.",
+    restart: "Stop all instances and restart them afterwards.",
+    reload:  "Send a SIGHUP to all instances of the application.",
+    run:     "Start the application and stay on top.",
+    zap:     "Set the application to a stopped state.",
+    status:  "Show status (PID) of application instances."
+  }.each{ |k,v| opt.separator sprintf "    %*s%s", -33, k, v }
+  opt.separator ""
+  opt.separator "MiGA options:"
+  opt_object(opt, o, [:project])
+  opt.on("--latency INT",
+    "Number of seconds the daemon will be sleeping."
+    ){ |v| o[:latency]=v.to_i }
+  opt.on("--max-jobs INT",
+    "Maximum number of jobs to use simultaneously."){ |v| o[:maxjobs]=v.to_i }
+  opt.on("--ppn INT",
+    "Maximum number of cores to use in a single job."){ |v| o[:ppn]=v.to_i }
+  opt_common(opt, o)
+  opt.separator "Daemon options:"
+  opt.on("-t", "--ontop",
+    "Stay on top (does not daemonize)"){ o[:daemon_opts] << '-t' }
+  opt.on("-f", "--force", "Force operation"){ o[:daemon_opts] << '-f' }
+  opt.on("-n", "--no_wait",
+    "Do not wait for processes to stop"){ o[:daemon_opts] << '-n' }
+end.parse!
+
+##=> Main <=
+opt_require(o, project:"-P")
+
+raise "Project doesn't exist, aborting." unless MiGA::Project.exist? o[:project]
+p = MiGA::Project.new(o[:project])
+d = MiGA::Daemon.new(p)
+[:latency, :maxjobs, :ppn].each do |k|
+  d.runopts(k, o[k]) unless o[k].nil?
+end
+d.daemon(task, o[:daemon_opts])

data/actions/date.rb

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+# @package MiGA
+# @license Artistic-2.0
+
+o = {}
+opts = OptionParser.new do |opt|
+  opt_banner(opt)
+  opt_common(opt, o)
+end.parse!
+
+##=> Main <=
+opts.parse!
+puts Time.now.to_s

data/actions/{download_dataset → download_dataset.rb}

@@ -7,16 +7,8 @@ require "miga/remote_dataset"
 
 o = {q:true, query:false, universe: :ebi, db: :embl}
 OptionParser.new do |opt|
-  opt.banner = <<BAN
-Creates an empty dataset in a pre-existing MiGA project.
-
-Usage: #{$0} #{File.basename(__FILE__)} [options]
-BAN
-  opt.separator ""
-  opt.on("-P", "--project PATH",
-    "(Mandatory) Path to the project to use."){ |v| o[:project]=v }
-  opt.on("-D", "--dataset STRING",
-    "(Mandatory unless -F) ID of the dataset to create."){ |v| o[:dataset]=v }
+  opt_banner(opt)
+  opt_object(opt, o, [:project, :dataset, :dataset_type])
   opt.on("-I", "--ids ID1,ID2,...",
     "(Mandatory unless -F) IDs in the remote database separated by commas."
     ){ |v| o[:ids]=v }
@@ -31,10 +23,6 @@ BAN
     "The long form of all the options are supported as header (without the --)",
     "including dataset, ids, universe, and db. For query use true/false values."
     ){ |v| o[:file] = v }
-  opt.on("-t", "--type STRING",
-    "Type of dataset. Recognized types include:",
-    *MiGA::Dataset.KNOWN_TYPES.map{ |k,v| "~ #{k}: #{v[:description]}" }
-    ){ |v| o[:type]=v.to_sym }
   opt.on("-q", "--query",
     "If set, the dataset is registered as a query, not a reference dataset."
     ){ |v| o[:query]=v }
@@ -46,20 +34,11 @@ BAN
     "Owner of the dataset."){ |v| o[:user]=v }
   opt.on("-c", "--comments STRING",
     "Comments on the dataset."){ |v| o[:comments]=v }
-  opt.on("-v", "--verbose",
-    "Print additional information to STDERR."){ o[:q]=false }
-  opt.on("-d", "--debug INT", "Print debugging information to STDERR.") do |v|
-    v.to_i>1 ? MiGA::MiGA.DEBUG_TRACE_ON : MiGA::MiGA.DEBUG_ON
-  end
-  opt.on("-h", "--help", "Display this screen.") do
-    puts opt
-    exit
-  end
-  opt.separator ""
+  opt_common(opt, o)
 end.parse!
 
 
-### MAIN
+##=> Main <=
 glob = [o]
 unless o[:file].nil?
   glob = []
@@ -81,9 +60,7 @@ unless o[:file].nil?
 end
 
 glob.each do |o_i|
-  raise "-P is mandatory." if o_i[:project].nil?
-  raise "-D is mandatory." if o_i[:dataset].nil?
-  raise "-I is mandatory." if o_i[:ids].nil?
+  opt_require(o_i, project:"-P", dataset:"-D", ids:"-I")
 
   $stderr.puts "Dataset: #{o_i[:dataset]}" unless o_i[:q]
   $stderr.puts "Loading project." unless o_i[:q]

data/actions/find_datasets.rb

@@ -0,0 +1,41 @@
+#!/usr/bin/env ruby
+
+# @package MiGA
+# @license Artistic-2.0
+
+o = {q:true, add:false, ref:true}
+OptionParser.new do |opt|
+  opt_banner(opt)
+  opt_object(opt, o, [:project, :dataset_type])
+  opt.on("-a", "--add",
+    "Register the datasets found. By default, only lists them (dry run)."
+    ){ |v| o[:add]=v }
+  opt.on("-q", "--query",
+    "If set, all datasets are registered as query datasets."
+    ){ |v| o[:ref]=!v }
+  opt.on("-u", "--user STRING", "Owner of the dataset."){ |v| o[:user]=v }
+  opt_common(opt, o)
+end.parse!
+
+##=> Main <=
+opt_require(o, project:"-P")
+
+$stderr.puts "Loading project." unless o[:q]
+p = MiGA::Project.load(o[:project])
+raise "Impossible to load project: #{o[:project]}" if p.nil?
+
+$stderr.puts "Finding datasets." unless o[:q]
+ud = p.unregistered_datasets
+ud.each do |dn|
+  puts dn
+  if o[:add]
+    md = {}
+    [:type, :user].each{ |k| md[k]=o[k] unless o[k].nil? }
+    d = MiGA::Dataset.new(p, dn, o[:ref], md)
+    p.add_dataset(dn)
+    res = d.first_preprocessing(true)
+    puts "- #{res}" unless o[:q]
+  end
+end
+
+$stderr.puts "Done." unless o[:q]

data/actions/import_datasets.rb

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+# @package MiGA
+# @license Artistic-2.0
+
+o = {q:true, info:false, force:false, method: :hardlink }
+OptionParser.new do |opt|
+  opt_banner(opt)
+  opt_object(opt, o, [:project, :dataset_opt])
+  opt.on("-Q", "--project-target PATH",
+    "(Mandatory) Path to the project where to link the dataset."
+    ){ |v| o[:project2]=v }
+  opt.on("-f", "--force",
+    "Forces linking, even if dataset's preprocessing is incomplete."
+    ){ |v| o[:force]=v }
+  opt.on("-s", "--symlink",
+    "Creates symlinks instead of the default hard links."
+    ){ o[:method] = :symlink }
+  opt.on("-c", "--copy",
+    "Creates copies instead of the default hard links."){ o[:method] = :copy }
+  opt_filter_datasets(opt, o)
+  opt_common(opt, o)
+end.parse!
+
+##=> Main <=
+opt_require(o, project:"-P", project2:"-Q")
+
+$stderr.puts "Loading project." unless o[:q]
+p = MiGA::Project.load(o[:project])
+raise "Impossible to load project: #{o[:project]}" if p.nil?
+q = MiGA::Project.load(o[:project2])
+raise "Impossible to load project: #{o[:project2]}" if q.nil?
+
+$stderr.puts "Listing dataset." unless o[:q]
+if o[:dataset].nil?
+  ds = p.datasets
+else
+  ds = [p.dataset(o[:dataset])]
+end
+ds = filter_datasets!(ds, o)
+ds.each do |d|
+  next unless o[:force] or d.done_preprocessing?
+  puts d.name
+  q.import_dataset(d, o[:method])
+end
+
+$stderr.puts "Done." unless o[:q]