miga-base 0.6.0.0 → 0.6.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 13b2ec821a69376491a0a3a31da7ccf717d8aafd1dc7753acaec36f3b7aa42c4
-  data.tar.gz: f6d513971e2c06c3f299bc5bf680682c533b380ff4340443f789446a6835b54f
+  metadata.gz: 1b7ff73c17fe13dbe5952bb607981bba291cf29d34c372157281339061036157
+  data.tar.gz: d5af6ded80c32042dbc3463fab085f2bc5c58ea9fc8755224baeb8d08f527ff1
 SHA512:
-  metadata.gz: 5a4de9af15f63372d98b64cf1604e64fc576c3809f435dc543acb98cab9779155998e5f1b382af856e5d4760faba0d9f606f614654afdad10659b764c9c6a17f
-  data.tar.gz: bd01962ae2c1f75ec3768d2515bab711191c4a2b256996c51a69bcbdce5f87b06215816a6368fad0e399d69c09ec7992f97b9868e8670f3866bc1fdcb29d6c41
+  metadata.gz: b178fdfbe3ee23519fa7a6c23d15e6a9f393c1bea610124fa1ccf8245edbb3b95bb8ae9b46382eae3ca2e2664b1dd536fdd7a7c2b34c23cba35fb4e82c8a4123
+  data.tar.gz: 96e1ff434584528ae8dcf0b6435cc1dbe74a312eb9a1a1dbab86d7464dfe50eb38bced778caf0ef46446c11864495af4b8902ab76ac9700716f5d82ddf8370d9

data/bin/miga

@@ -3,7 +3,7 @@
 # @package MiGA
 # @license Artistic-2.0
 
-$:.push File.expand_path('../../lib', __FILE__)
+$LOAD_PATH.push File.expand_path('../../lib', __FILE__)
 require 'miga'
 require 'miga/cli'
 

data/lib/miga/cli/action/archive.rb

@@ -0,0 +1,99 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+require 'rubygems/package'
+
+class MiGA::Cli::Action::Archive < MiGA::Cli::Action
+
+  def parse_cli
+    cli.parse do |opt|
+      opt.on(
+        '-o', '--tarball PATH',
+        '(Mandatory) Path to the archive to be created ending in .tar.gz'
+      ) { |v| cli[:tarball] = v }
+      opt.on(
+        '-f', '--folder STRING',
+        'Name of the output folder. By default: name of the source project'
+      ) { |v| cli[:folder] = v }
+      cli.opt_object(opt, [:project, :dataset_opt])
+      cli.opt_filter_datasets(opt)
+    end
+  end
+
+  def perform
+    cli.ensure_par(tarball: '-o')
+    unless cli[:tarball] =~ /\.tar\.gz$/
+      raise 'The tarball path (-o) must have .tar.gz extension'
+    end
+    cli[:folder] ||= cli.load_project.name
+    ds = cli.load_and_filter_datasets
+
+    open_tarball do |tar|
+      # Datasets
+      cli.say 'Archiving datasets'
+      each_file_listed(ds) do |rel_path, abs_path|
+        add_file_to_tar(tar, rel_path, abs_path)
+      end
+
+      # Project
+      cli.say 'Archiving project'
+      pmd = cli.load_project.metadata.dup
+      pmd[:datasets] = ds.map(&:name)
+      add_string_to_tar(tar, 'miga.project.json', pmd.to_json)
+      add_string_to_tar(tar, 'daemon/daemon.json', '{}')
+    end
+  end
+
+  private
+
+  def open_tarball(&blk)
+    File.open(cli[:tarball], 'wb') do |fh|
+      Zlib::GzipWriter.wrap(fh) do |gz|
+        Gem::Package::TarWriter.new(gz) do |tar|
+          blk.call(tar)
+        end
+      end
+    end
+  end
+
+  def each_file_listed(datasets, &blk)
+    datasets.each_with_index do |ds, k|
+      cli.advance('Datasets:', k + 1, datasets.size, false)
+      # Metadata
+      blk.call(
+        File.join('metadata', File.basename(ds.metadata.path)),
+        ds.metadata.path
+      )
+      # Results
+      ds.each_result do |sym, res|
+        res.each_file do |_sym, rel_path, abs_path|
+          blk.call(File.join(res.relative_dir, rel_path), abs_path)
+        end
+        blk.call(res.relative_path, res.path) # <- JSON
+      end
+    end
+    cli.say
+  end
+
+  def add_file_to_tar(tar, rel_path, abs_path)
+    if File.directory? abs_path
+      Dir["#{abs_path}/*"].each do |f|
+        add_file_to_tar(tar, File.join(rel_path, File.basename(f)), f)
+      end
+    else
+      in_tar = File.join(cli[:folder], rel_path)
+      tar.add_file_simple(in_tar, 0666, File.size(abs_path)) do |ofh|
+        File.open(abs_path, 'rb') do |ifh|
+          ofh.write(ifh.read(1024)) until ifh.eof?
+        end
+      end
+    end
+  end
+
+  def add_string_to_tar(tar, rel_path, string)
+    in_tar = File.join(cli[:folder], rel_path)
+    tar.add_file_simple(in_tar, 0666, string.size) { |fh| fh.write(string) }
+  end
+
+end

data/lib/miga/cli/base.rb

@@ -24,6 +24,7 @@ module MiGA::Cli::Base
     find:     'Finds unregistered datasets based on result files',
     ln:       'Link datasets (including results) from one project to another',
     ls:       'Lists all registered datasets in an MiGA project',
+    archive:  'Generates a tar-ball with all files from select datasets',
     # Results
     add_result: 'Registers a result',
     stats:    'Extracts statistics for the given result',

data/lib/miga/cli.rb

@@ -119,7 +119,7 @@ class MiGA::Cli < MiGA::MiGA
   def advance(step, n = 0, total = nil, bin = true)
     return unless self[:verbose]
     adv = total.nil? ? (n == 0 ? '' : num_suffix(n, bin)) :
-      ('%.1f%% (%s/%s)' % [100 * n / total,
+      ('%.1f%% (%s/%s)' % [100.0 * n / total,
         num_suffix(n, bin), num_suffix(total, bin)])
     $stderr.print("[%s] %s %s    \r" % [Time.now, step, adv])
   end

data/lib/miga/dataset/hooks.rb

@@ -39,6 +39,7 @@ module MiGA::Dataset::Hooks
     metadata.data.keys
       .select { |k| k.to_s =~ /^_try_/ }
       .each { |k| metadata[k] = nil }
+    metadata[:_step] = nil
     save
   end
 

data/lib/miga/metadata.rb

@@ -22,18 +22,18 @@ class MiGA::Metadata < MiGA::MiGA
   # Instance-level
 
   ##
-  # Path to the JSON file describing the metadata.
+  # Path to the JSON file describing the metadata
   attr_reader :path
 
   ##
-  # Initiate a MiGA::Metadata object with description in +path+. It will create
-  # it if it doesn't exist.
-  def initialize(path, defaults={})
+  # Initiate a MiGA::Metadata object with description in +path+.
+  # It will create it if it doesn't exist
+  def initialize(path, defaults = {})
     @data = nil
     @path = File.absolute_path(path)
     unless File.exist? path
       @data = {}
-      defaults.each_pair{ |k,v| self[k]=v }
+      defaults.each { |k,v| self[k] = v }
       create
     end
   end
@@ -57,10 +57,11 @@ class MiGA::Metadata < MiGA::MiGA
   def save
     MiGA.DEBUG "Metadata.save #{path}"
     self[:updated] = Time.now.to_s
-    json = MiGA::Json.generate(data)
+    json = to_json
     sleeper = 0.0
     slept = 0
     while File.exist?(lock_file)
+      MiGA::MiGA.DEBUG "Waiting for lock: #{lock_file}"
       sleeper += 0.1 if sleeper <= 10.0
       sleep(sleeper.to_i)
       slept += sleeper.to_i
@@ -70,7 +71,7 @@ class MiGA::Metadata < MiGA::MiGA
     ofh = File.open("#{path}.tmp", 'w')
     ofh.puts json
     ofh.close
-    raise "Lock-racing detected for #{path}." unless
+    raise "Lock-racing detected for #{path}" unless
       File.exist?("#{path}.tmp") and File.exist?(lock_file)
     File.rename("#{path}.tmp", path)
     File.unlink(lock_file)
@@ -122,4 +123,10 @@ class MiGA::Metadata < MiGA::MiGA
   # Iterate +blk+ for each data with 2 arguments key and value.
   def each(&blk) data.each{ |k,v| blk.call(k,v) } ; end
 
+  ##
+  # Show contents in JSON format as a String
+  def to_json
+    MiGA::Json.generate(data)
+  end
+
 end

data/lib/miga/result/source.rb

@@ -18,14 +18,21 @@ module MiGA::Result::Source
   ##
   # Detect the result key assigned to this result
   def key
-    @key ||= MiGA::Result.RESULT_DIRS.find { |k, v| v == relative_dir }.first
+    @key ||= MiGA::Result.RESULT_DIRS.find do |k, v|
+      "data/#{v}" == relative_dir
+    end.first
   end
 
   ##
-  # Path of the result containing the directory relative to the +data+ folder in
-  # the parent project
+  # Path of the result containing the directory relative to the parent project
   def relative_dir
-    @relative_dir ||= dir.sub("#{project_path}data/", '')
+    @relative_dir ||= dir.sub("#{project_path}/", '')
+  end
+
+  ##
+  # Path of the result's JSON definition relative to the parent project
+  def relative_path
+    @relative_path ||= path.sub("#{project_path}/", '')
   end
 
   ##
@@ -40,7 +47,7 @@ module MiGA::Result::Source
   # so the path referencing is identical to that of +self.path+ whenever they
   # need to be compared.
   def project_path
-    path[ 0 .. path.rindex('/data/') ]
+    path[ 0 .. path.rindex('/data/') - 1 ]
   end
 end
 

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.6, 0, 0]
+  VERSION = [0.6, 1, 0]
 
   ##
   # Nickname for the current major.minor version.
@@ -18,7 +18,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2020, 3, 7)
+  VERSION_DATE = Date.new(2020, 3, 10)
 
   ##
   # Reference of MiGA.

data/test/common_test.rb

@@ -1,7 +1,7 @@
-require "test_helper"
+require 'test_helper'
 
 class CommonTest < Test::Unit::TestCase
-  
+
   def setup
     #$jruby_tests = !ENV["JRUBY_TESTS"].nil?
   end
@@ -12,14 +12,14 @@ class CommonTest < Test::Unit::TestCase
     assert_respond_to(MiGA::MiGA, :DEBUG_OFF)
     MiGA::MiGA.DEBUG_ON
     err = capture_stderr do
-      MiGA::MiGA.DEBUG "Tralari"
+      MiGA::MiGA.DEBUG 'Tralari'
     end
     assert_equal("Tralari\n", err.string)
     MiGA::MiGA.DEBUG_OFF
     err = capture_stderr do
-      MiGA::MiGA.DEBUG "Tralara"
+      MiGA::MiGA.DEBUG 'Tralara'
     end
-    assert_equal("", err.string)
+    assert_equal('', err.string)
   ensure
     MiGA::MiGA.DEBUG_OFF
   end
@@ -31,12 +31,12 @@ class CommonTest < Test::Unit::TestCase
     #omit_if($jruby_tests, "JRuby doesn't like interceptions.")
     MiGA::MiGA.DEBUG_TRACE_ON
     err = capture_stderr do
-      MiGA::MiGA.DEBUG "Dandadi"
+      MiGA::MiGA.DEBUG 'Dandadi'
     end
     assert(err.string =~ /Dandadi\n    .*block in test_debug_trace/)
     MiGA::MiGA.DEBUG_TRACE_OFF
     err = capture_stderr do
-      MiGA::MiGA.DEBUG "Dandada"
+      MiGA::MiGA.DEBUG 'Dandada'
     end
     assert_equal("Dandada\n", err.string)
   ensure
@@ -46,37 +46,37 @@ class CommonTest < Test::Unit::TestCase
 
   def test_generic_transfer
     $tmp = Dir.mktmpdir
-    hello = File.expand_path("Hello", $tmp)
-    world = File.expand_path("World", $tmp)
+    hello = File.expand_path('Hello', $tmp)
+    world = File.expand_path('World', $tmp)
     assert_respond_to(File, :generic_transfer)
     FileUtils.touch(hello)
     File.generic_transfer(hello, world, :symlink)
-    assert_equal("link", File.ftype(world), "World should be a link.")
+    assert_equal('link', File.ftype(world), 'World should be a link.')
     File.generic_transfer(hello, world, :copy)
-    assert_equal("link", File.ftype(world), "World should still be a link.")
+    assert_equal('link', File.ftype(world), 'World should still be a link.')
     File.unlink world
     File.generic_transfer(hello, world, :hardlink)
-    assert_equal("file", File.ftype(world), "A hardlink should be a file.")
-    File.open(hello, "w"){ |fh| fh.print "!" }
-    File.open(world, "r"){ |fh| assert_equal("!", fh.gets) }
+    assert_equal('file', File.ftype(world), 'A hardlink should be a file.')
+    File.open(hello, 'w') { |fh| fh.print '!' }
+    File.open(world, 'r') { |fh| assert_equal('!', fh.gets) }
     File.unlink world
     File.generic_transfer(hello, world, :copy)
-    assert_equal("file", File.ftype(world), "A copy should be a file.")
+    assert_equal('file', File.ftype(world), 'A copy should be a file.')
     File.unlink world
     assert_raise do
       File.generic_transfer(hello, world, :monkey)
     end
-    assert(!File.exist?(world), "A monkey shouldn't create files.")
+    assert(!File.exist?(world), 'A monkey shouldn\'t create files.')
   ensure
     FileUtils.rm_rf $tmp
   end
 
   def test_tabulate
     tab = MiGA::MiGA.tabulate(%w[a b], [%w[123 45], %w[678 90]])
-    assert_equal("  a  b ", tab[0])
-    assert_equal("  -  - ", tab[1])
-    assert_equal("123  45", tab[2])
-    assert_equal("678  90", tab[3])
+    assert_equal('  a  b ', tab[0])
+    assert_equal('  -  - ', tab[1])
+    assert_equal('123  45', tab[2])
+    assert_equal('678  90', tab[3])
   end
 
   def test_miga_name

data/test/daemon_test.rb

@@ -1,29 +1,29 @@
-require "test_helper"
-require "miga/daemon"
+require 'test_helper'
+require 'miga/daemon'
 
 class DaemonTest < Test::Unit::TestCase
-  
+
   def setup
-    $jruby_tests = !ENV["JRUBY_TESTS"].nil?
+    $jruby_tests = !ENV['JRUBY_TESTS'].nil?
     $tmp = Dir.mktmpdir
-    ENV["MIGA_HOME"] = $tmp
-    FileUtils.touch("#{ENV["MIGA_HOME"]}/.miga_rc")
-    File.open("#{ENV["MIGA_HOME"]}/.miga_daemon.json", "w") do |fh|
+    ENV['MIGA_HOME'] = $tmp
+    FileUtils.touch(File.expand_path('.miga_rc', ENV['MIGA_HOME']))
+    daemon_json = File.expand_path('.miga_daemon.json', ENV['MIGA_HOME'])
+    File.open(daemon_json, 'w') do |fh|
       fh.puts '{"maxjobs":1,"ppn":1,"latency":2,"varsep":" ",
         "var":"{{key}}={{value}}","cmd":"{{task_name}}",
         "alive":"echo 1 # {{pid}}","type":"bash","format_version":1}'
     end
-    $p1 = MiGA::Project.new(File.expand_path("project1", $tmp))
+    $p1 = MiGA::Project.new(File.expand_path('project1', $tmp))
     $d1 = MiGA::Daemon.new($p1)
   end
 
   def teardown
     FileUtils.rm_rf $tmp
-    ENV["MIGA_HOME"] = nil
+    ENV['MIGA_HOME'] = nil
   end
 
   def test_check_project
-  
   end
 
   def test_check_datasets
@@ -31,20 +31,24 @@ class DaemonTest < Test::Unit::TestCase
     d = $d1
     d.runopts(:maxjobs, 0, true)
     assert(d.jobs_to_run.empty?)
-    ds = p.add_dataset("ds1")
+    ds = p.add_dataset('ds1')
     d.check_datasets
     assert(d.jobs_to_run.empty?)
-    FileUtils.cp(File.expand_path("daemon/daemon.json", p.path),
-      File.expand_path("data/01.raw_reads/ds1.1.fastq", p.path))
-    FileUtils.cp(File.expand_path("daemon/daemon.json", p.path),
-      File.expand_path("data/01.raw_reads/ds1.done", p.path))
+    FileUtils.cp(
+      File.expand_path('daemon/daemon.json', p.path),
+      File.expand_path('data/01.raw_reads/ds1.1.fastq', p.path)
+    )
+    FileUtils.cp(
+      File.expand_path('daemon/daemon.json', p.path),
+      File.expand_path('data/01.raw_reads/ds1.done', p.path)
+    )
     ds.first_preprocessing(true)
     out = capture_stdout do
       d.check_datasets
     end
     assert(out.string =~ /Queueing #{ds.name}:d/)
     assert_equal(1, d.jobs_to_run.size)
-    assert_equal("project1:d:ds1", d.jobs_to_run.first[:cmd])
+    assert_equal('project1:d:ds1', d.jobs_to_run.first[:cmd])
     assert_equal(d.jobs_to_run.first, d.get_job(:d, ds))
   end
 
@@ -54,16 +58,12 @@ class DaemonTest < Test::Unit::TestCase
     d.runopts(:latency, 0, true)
     assert_equal(-1, d.loop_i)
     assert_nil(d.last_alive)
-    out = capture_stdout do
-      d.in_loop
-    end
+    out = capture_stdout { d.in_loop }
     assert_equal(Time, d.last_alive.class)
     assert(out.string =~ /-{20}\n.*MiGA:#{p.name} launched/)
     10.times{ d.in_loop }
     assert_equal(11, d.loop_i)
-    out = capture_stdout do
-      d.in_loop
-    end
+    out = capture_stdout { d.in_loop }
     assert(out.string =~ /Probing running jobs/)
     assert_equal(0, d.loop_i)
   end
@@ -73,8 +73,8 @@ class DaemonTest < Test::Unit::TestCase
     d = $d1
     d.runopts(:latency, 0, true)
     assert_equal(0, d.latency)
-    omit_if($jruby_tests, "JRuby doesn't implement fork.")
-    $child = fork { d.start(["--shush"]) }
+    omit_if($jruby_tests, 'JRuby doesn\'t implement fork.')
+    $child = fork { d.start(['--shush']) }
     sleep(3)
     dpath = File.expand_path("daemon/MiGA:#{p.name}",p.path)
     assert(File.exist?("#{dpath}.pid"))
@@ -90,11 +90,11 @@ class DaemonTest < Test::Unit::TestCase
       assert(l[3] =~ /Probing running jobs\n/)
     end
   ensure
-    Process.kill("KILL", $child) unless $child.nil?
+    Process.kill('KILL', $child) unless $child.nil?
   end
 
   def test_last_alive
-    p = MiGA::Project.new(File.expand_path("last_alive", $tmp))
+    p = MiGA::Project.new(File.expand_path('last_alive', $tmp))
     d = MiGA::Daemon.new(p)
     assert_nil(d.last_alive)
     d.declare_alive
@@ -110,18 +110,14 @@ class DaemonTest < Test::Unit::TestCase
     assert_equal(1, $d1.ppn)
     $d1.runopts(:alo, :ha)
     assert_equal(:ha, $d1.runopts(:alo))
-    $d1.runopts(:maxjobs, "1")
+    $d1.runopts(:maxjobs, '1')
     assert_equal(1, $d1.maxjobs)
-    assert_raise do
-      $d1.runopts(:latency, "!")
-    end
-    assert_equal("bash", $d1.runopts(:type))
+    assert_raise { $d1.runopts(:latency, '!') }
+    assert_equal('bash', $d1.runopts(:type))
   end
 
   def test_say
-    out = capture_stdout do
-      $d1.say "Olm"
-    end
+    out = capture_stdout { $d1.say 'Olm' }
     assert(out.string =~ /^\[.*\] Olm/)
   end
 
@@ -129,9 +125,7 @@ class DaemonTest < Test::Unit::TestCase
     d = $d1
     d.declare_alive
     assert_not_nil(d.last_alive)
-    out = capture_stdout do
-      d.terminate
-    end
+    out = capture_stdout { d.terminate }
     assert(out.string =~ /Terminating daemon/)
     assert_nil(d.last_alive)
   end