bio-gemma-wrapper 0.99.3 → 0.99.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0bd37b153e121de9c1758af736cd6904744da2de3540f2a7c547cc423382d8d1
-  data.tar.gz: 84298a943e7cfe6126653895d9714babc83a7be2bf903c7b61ff9072f1d4e4a8
+  metadata.gz: da5f26b8acd9c3782c2b3f5f2a39af965fc7e1785cc820b49faca82924d74e51
+  data.tar.gz: 17035ee5fada269ae88dd0ed91d84075b2af88b400de1d0e9829cbdb60d5d0cb
 SHA512:
-  metadata.gz: f32d48ec2f194a513e0cf8f15463b05662e1b269fdd22a110e4cdfb9f6bc541238bac7e766cbc31a8b782379891636e4f79fb3e3667d567ab3c610298d4f11c2
-  data.tar.gz: abc9b3faf8ef2f63d566caa14312bde2b2f438c722a83e90f7da775c55e2415171efbfab385273c82e45398c31e5306fb99437c38e287570652a3b04a5432883
+  metadata.gz: eaec3c7dad4fc1bda713765e056bfe11dd69d4ca850333fed5a1a27e344724365a705ddf7845ce63b5af6b35ab6140da10f4bc7067aaa4539e47f6c6f94de1f0
+  data.tar.gz: c26b282c0fd7c70a702467e58c3f6ea22f820d91a8a364b335cdab7e807add9cf1079faa25c46a87b89c78e4293990cdea2210427c5f9c3565bd5040fdbef496

data/VERSION

@@ -1 +1 @@
-0.99.3
+0.99.4

data/bin/gemma-wrapper

@@ -71,6 +71,8 @@ require 'optparse'
 require 'tempfile'
 require 'tmpdir'
 
+require 'lock'
+
 split_at = ARGV.index('--')
 
 if split_at
@@ -194,6 +196,7 @@ warning = lambda do |*msg|
   record[:warnings].push *msg.join("")
   OUTPUT.print "WARNING: ",*msg,"\n"
 end
+
 info = lambda do |*msg|
   record[:debug].push *msg.join("") if options[:json] and options[:debug]
   OUTPUT.print *msg,"\n" if !options[:quiet]
@@ -215,14 +218,14 @@ GEMMA_K_BANNER = "gemma-wrapper #{version} (Ruby #{RUBY_VERSION}) by Pjotr Prins
 info.call GEMMA_K_BANNER
 
 # Check gemma version
-GEMMA_COMMAND=options[:gemma_command]
-info.call "NOTE: gemma-wrapper is soon to be replaced by gemma2/lib"
-
 begin
-  GEMMA_INFO = `#{GEMMA_COMMAND}`
+  gemma_command2 = options[:gemma_command]
+  info.call "NOTE: gemma-wrapper is soon to be replaced by gemma2/lib"
+
+  GEMMA_INFO = `#{gemma_command2}`
 rescue Errno::ENOENT
-  GEMMA_COMMAND = "gemma" if not GEMMA_COMMAND
-  error.call "<#{GEMMA_COMMAND}> command not found"
+  gemma_command2 = "gemma"
+  error.call "<#{gemma_command2}> command not found"
 end
 
 gemma_version_header = GEMMA_INFO.split("\n").grep(/GEMMA|Version/)[0].strip
@@ -329,6 +332,14 @@ end
 HASH = compute_hash.call()
 options[:hash] = HASH
 
+at_exit do
+  Lock.release(HASH)
+end
+
+Lock.create(HASH) # this will wait for a lock to expire
+
+joblog = options[:cache_dir]+"/"+HASH+"-parallel.log"
+
 # Create cache dir
 FileUtils::mkdir_p options[:cache_dir]
 
@@ -352,7 +363,7 @@ hashme =
 debug.call "Options: ",options,"\n" if !options[:quiet]
 
 invoke_gemma = lambda do |extra_args, cache_hit = false, chr = "full", permutation = 1|
-  cmd = "#{GEMMA_COMMAND} #{GEMMA_ARGS.join(' ')} #{extra_args.join(' ')}"
+  cmd = "#{gemma_command2} #{GEMMA_ARGS.join(' ')} #{extra_args.join(' ')}"
   record[:gemma_command] = cmd
   return if cache_hit
   if options[:slurm]
@@ -552,15 +563,17 @@ if options[:parallel]
   cmd = parallel_cmds.join("\\n")
 
   cmd = "echo -e \"#{cmd}\""
-  err = execute.call(cmd+"|parallel") # all jobs in parallel
+  err = execute.call(cmd+"|parallel --joblog #{joblog}") # first try optimistically to run all jobs in parallel
   if err != 0
     [16,8,4,1].each do |jobs|
       info.call("Failed to complete parallel run -- retrying with smaller RAM footprint!")
-      err = execute.call(cmd+"|parallel -j #{jobs}")
+      err = execute.call(cmd+"|parallel -j #{jobs} --resume --joblog #{joblog}")
       break if err == 0
     end
     if err != 0
       info.call("Run failed!")
+      # Remove remaining files
+      FileUtils.rm_rf("#{tmpdir}/*", secure: true)
       exit err
     end
   end

data/gemma-wrapper.gemspec

@@ -6,6 +6,7 @@ Gem::Specification.new do |s|
   s.authors     = ["Pjotr Prins"]
   s.email       = 'pjotr.public01@thebird.nl'
   s.files       = ["bin/gemma-wrapper",
+                   "lib/lock.rb",
                    "Gemfile",
                    "LICENSE.txt",
                    "README.md",

data/lib/lock.rb

@@ -0,0 +1,95 @@
+# Locking module for gemma (wrapper)
+#
+
+=begin
+
+The logic is as follows:
+
+1. a program creates a named lock file (based on a hash of its inputs) with its PID
+2. on exit it destroys the file
+3. a new program checks for the lock file
+4. if it exists and the PID is still in the ps table - wait
+5. when the pid disappears or the lock file - continue
+6. a timeout will return an error in 3 minutes
+
+Note that there is a theoretical chance the lock file existed, but disappeared. I think I have it covered by ignoring the unlink errors. Also the use of /proc/PID is Linux specific.
+
+=end
+
+
+require 'timeout'
+
+module Lock
+
+  def self.local name
+    ENV['HOME']+"/."+name.gsub("/","-")+".lck"
+  end
+
+  def self.lock_pid name
+    lockfn = local(name)
+    if File.exist?(lockfn)
+      File.read(lockfn).to_i
+    else
+      0
+    end
+  end
+
+  def self.locked? name
+    lockfn = local(name)
+    pid = lock_pid(name)
+    if File.exist?("/proc/#{pid}")
+      true
+    else
+      # the program went away - remove any 'stale' lock
+      begin
+        File.unlink(lockfn)
+      rescue Errno::ENOENT
+        # ignore error when the lock file went missing
+      end
+      false # --> no longer locked
+    end
+  end
+
+  def Lock::create name
+    wait_for(name)
+    lockfn = local(name)
+    if File.exist?(lockfn)
+      $stderr.print "\nERROR: Can not steal #{lockfn}"
+      exit 1
+    end
+    File.open(lockfn, File::RDWR|File::CREAT, 0644) do |f|
+      f.flock(File::LOCK_EX)
+      f.print(Process.pid)
+    end
+  end
+
+  def Lock::wait_for name
+    lockfn = local(name)
+    begin
+      status = Timeout::timeout(180) { # 3 minutes
+        while locked?(name)
+          $stderr.print("\nWaiting for lock #{lockfn}...")
+          sleep 2
+        end
+      }
+    rescue Timeout::Error
+      $stderr.print "\nERROR: Timed out, but I can not steal #{lockfn}"
+      exit 1
+    end
+    # yah! lock is released
+  end
+
+  def Lock::release name
+    lockfn = local(name)
+    if Process.pid == lock_pid(name)
+      begin
+        File.unlink(lockfn) # PID expired
+      rescue Errno::ENOENT
+        # ignore error when the lock file went missing
+      end
+    else
+      $stderr.print "\nERROR: can not release #{lockfn} because it is not owned by me"
+    end
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bio-gemma-wrapper
 version: !ruby/object:Gem::Version
-  version: 0.99.3
+  version: 0.99.4
 platform: ruby
 authors:
 - Pjotr Prins
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-22 00:00:00.000000000 Z
+date: 2021-11-25 00:00:00.000000000 Z
 dependencies: []
 description: GEMMA wrapper adds LOCO and permutation support. Also runs in parallel
   and caches K between runs with LOCO support
@@ -24,6 +24,7 @@ files:
 - VERSION
 - bin/gemma-wrapper
 - gemma-wrapper.gemspec
+- lib/lock.rb
 homepage: https://github.com/genetics-statistics/gemma-wrapper
 licenses:
 - GPL3
@@ -43,7 +44,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.5
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: GEMMA with LOCO and permutations