bio-gemma-wrapper 0.99.1 → 0.99.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9ddfd904e74beebe0de1b97732d872fce171732965a835b101b9cc9be815bb05
-  data.tar.gz: 2dae1c019da23f2f87216694d641fc1eb852aa7800557bd10cfb08cb3425e844
+  metadata.gz: 5ec477c7560ae55b6d7c8a74b5e46cd95586a87f02ac698c540b2d6cc40c4392
+  data.tar.gz: bf79dc493baa99efd2e20a298d702ece9973c3fd377773cc80f0867c0a132ae5
 SHA512:
-  metadata.gz: 38454a3f12dab85bef711051e73e20a015fe6b6d9c71bafada2197b9aef1aa0eabe3f3709cb0dc9d0c39f4cc454c15bc4d3aea5d06140ccde72fa13aa6285f51
-  data.tar.gz: 28e77a6995893245c501e602d488b5e0c504549fa91d8c94f902591b87b4454fe9b7923667dfacae2ab1dac7f6f7d814df1ec036b2b4f616dfd4b84c549d35d1
+  metadata.gz: 0ca0c04ef86c22d332c10f21b66ef24aab7fd3dbfd1f2db67f78309ba8a5af47a9263bf94aaf2afeda021cdace3ed042af6f3a5ebf6628714a78ad18b494284f
+  data.tar.gz: b3692f74bb4437f70d2671fe48e1ec9f7c44c64b4553c93b511cc648e157e6a9f5b0f66acce2f07721a74a7ce20a1993c7eb2d2a74eb362c786cd7ee42eeab16

data/README.md

@@ -8,11 +8,12 @@ Nat. Genet., 2016)](cfw.gif)
 ## Introduction
 
 Gemma-wrapper allows running GEMMA with LOCO, GEMMA with caching,
-GEMMA in parallel (now the default), and GEMMA on PBS. Gemma-wrapper
-is used to run GEMMA as part of the https://genenetwork.org/
-environment.
+GEMMA in parallel (now the default with LOCO), and GEMMA on
+PBS. Gemma-wrapper is used to run GEMMA as part of the
+https://genenetwork.org/ environment.
 
-Note that gemma-wrapper is projected to be integrated into gemma2/lib.
+Note that a version of gemma-wrapper is projected to be integrated
+into gemma itself.
 
 GEMMA is a software toolkit for fast application of linear mixed
 models (LMMs) and related models to genome-wide association studies
@@ -29,6 +30,14 @@ does a pass-through of all standard GEMMA invocation switches. On
 return gemma-wrapper can return a JSON object (--json) which is
 useful for web-services.
 
+## Performance
+
+LOCO runs in parallel by default which is at least a 5x performance
+improvement on a machine with enough cores. GEMMA without LOCO,
+however, does not run in parallel by default.  Performance
+improvements with the parallel implementation for LOCO and non-LOCO
+can be viewed [here](./test/performance/releases.gmi).
+
 ## Installation
 
 Prerequisites are
@@ -53,15 +62,19 @@ and it will render something like
 Usage: gemma-wrapper [options] -- [gemma-options]
         --permutate n                Permutate # times by shuffling phenotypes
         --permute-phenotypes filen   Phenotypes to be shuffled in permutations
-        --loco [x,y,1,2,3...]        Run full LOCO
+        --loco                       Run full leave-one-chromosome-out (LOCO)
+        --chromosomes [1,2,3]        Run specific chromosomes
         --input filen                JSON input variables (used for LOCO)
         --cache-dir path             Use a cache directory
         --json                       Create output file in JSON format
-        --force                      Force computation
-        --slurm [options]            Submit to slurm PBS
+        --force                      Force computation (override cache)
+        --parallel                   Run jobs in parallel
+        --no-parallel                Do not run jobs in parallel
+        --slurm[=opts]               Use slurm PBS for submitting jobs
         --q, --quiet                 Run quietly
     -v, --verbose                    Run verbosely
-        --debug                      Show debug messages and keep intermediate output
+    -d, --debug                      Show debug messages and keep intermediate output
+        --dry-run                    Show commands, but don't execute
         --                           Anything after gets passed to GEMMA
 
     -h, --help                       display this help and exit
@@ -99,6 +112,7 @@ the data files are found):
     gemma-wrapper -- \
         -g test/data/input/BXD_geno.txt.gz \
         -p test/data/input/BXD_pheno.txt \
+        -a test/data/input/BXD_snps.txt \
         -gk \
         -debug
 
@@ -116,6 +130,7 @@ You can also get JSON output on STDOUT by providing the --json switch
     gemma-wrapper --json -- \
         -g test/data/input/BXD_geno.txt.gz \
         -p test/data/input/BXD_pheno.txt \
+        -a test/data/input/BXD_snps.txt \
         -gk \
         -debug > K.json
 
@@ -133,6 +148,7 @@ default. If you want something else provide a --cache-dir, e.g.
     gemma-wrapper --cache-dir ~/.gemma-cache -- \
         -g test/data/input/BXD_geno.txt.gz \
         -p test/data/input/BXD_pheno.txt \
+        -a test/data/input/BXD_snps.txt \
         -gk \
         -debug
 
@@ -143,7 +159,7 @@ will store K in ~/.gemma-cache.
 Run the LMM using the K's captured earlier in K.json using the --input
 switch
 
-    gemma-wrapper --json --loco --input K.json -- \
+    gemma-wrapper --json --input K.json -- \
         -g test/data/input/BXD_geno.txt.gz \
         -p test/data/input/BXD_pheno.txt \
         -c test/data/input/BXD_covariates2.txt \
@@ -163,7 +179,7 @@ https://github.com/genetics-statistics/GEMMA/issues/46). To loop all
 chromosomes first create all K's with
 
     gemma-wrapper --json \
-        --loco 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,X -- \
+        --loco -- \
         -g test/data/input/BXD_geno.txt.gz \
         -p test/data/input/BXD_pheno.txt \
         -a test/data/input/BXD_snps.txt \

data/VERSION

@@ -1 +1 @@
-0.99.1
+0.99.5

data/bin/gemma-wrapper

@@ -14,12 +14,12 @@ GEMMA wrapper example:
       gemma-wrapper -- \\
         -g test/data/input/BXD_geno.txt.gz \\
         -p test/data/input/BXD_pheno.txt \\
+        -a test/data/input/BXD_snps.txt \
         -gk
 
     LOCO K computation with caching and JSON output
 
-      gemma-wrapper --json \\
-        --loco 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,X -- \\
+      gemma-wrapper --json --loco -- \\
         -g test/data/input/BXD_geno.txt.gz \\
         -p test/data/input/BXD_pheno.txt \\
         -a test/data/input/BXD_snps.txt \\
@@ -41,7 +41,7 @@ Gemma gets used from the path. You can override by setting
 "
 # These are used for testing compatibility with the gemma tool
 GEMMA_V_MAJOR = 98
-GEMMA_V_MINOR = 1
+GEMMA_V_MINOR = 4
 
 basepath = File.dirname(File.dirname(__FILE__))
 $: << File.join(basepath,'lib')
@@ -71,12 +71,15 @@ require 'optparse'
 require 'tempfile'
 require 'tmpdir'
 
+require 'lock'
+
 split_at = ARGV.index('--')
+
 if split_at
   gemma_args = ARGV[split_at+1..-1]
 end
 
-options = { show_help: false, source: 'https://github.com/genetics-statistics/gemma-wrapper', version: version+' (Pjotr Prins)', date: Time.now.to_s, gemma_command: gemma_command, cache_dir: Dir.tmpdir(), quiet: false, parallel: true }
+options = { show_help: false, source: 'https://github.com/genetics-statistics/gemma-wrapper', version: version+' (Pjotr Prins)', date: Time.now.to_s, gemma_command: gemma_command, cache_dir: Dir.tmpdir(), quiet: false, permute_phenotypes: false, parallel: nil }
 
 opts = OptionParser.new do |o|
   o.banner = "\nUsage: #{File.basename($0)} [options] -- [gemma-options]"
@@ -91,8 +94,12 @@ opts = OptionParser.new do |o|
     raise "Phenotype input file #{phenotypes} does not exist" if !File.exist?(phenotypes)
   end
 
-  o.on('--loco [x,y,1,2,3...]', Array, 'Run full leave-one-chromosome-out (LOCO)') do |lst|
-    options[:loco] = lst
+  o.on('--loco', 'Run full leave-one-chromosome-out (LOCO)') do |b|
+    options[:loco] = b
+  end
+
+  o.on('--chromosomes [1,2,3]',Array,'Run specific chromosomes') do |lst|
+    options[:chromosomes] = lst
   end
 
   o.on('--input filen',String, 'JSON input variables (used for LOCO)') do |filen|
@@ -112,6 +119,10 @@ opts = OptionParser.new do |o|
     options[:force] = true
   end
 
+  o.on("--parallel", "Run jobs in parallel") do |b|
+    options[:parallel] = true
+  end
+
   o.on("--no-parallel", "Do not run jobs in parallel") do |b|
     options[:parallel] = false
   end
@@ -185,11 +196,21 @@ warning = lambda do |*msg|
   record[:warnings].push *msg.join("")
   OUTPUT.print "WARNING: ",*msg,"\n"
 end
+
 info = lambda do |*msg|
   record[:debug].push *msg.join("") if options[:json] and options[:debug]
   OUTPUT.print *msg,"\n" if !options[:quiet]
 end
 
+# Fetch chromosomes
+def get_chromosomes annofn
+  h = {}
+  File.open(annofn,"r").each_line do | line |
+    chr = line.split(/\s+/)[2]
+    h[chr] = true
+  end
+  h.map { |k,v| k }
+end
 # ---- Start banner
 
 GEMMA_K_VERSION=version
@@ -197,14 +218,14 @@ GEMMA_K_BANNER = "gemma-wrapper #{version} (Ruby #{RUBY_VERSION}) by Pjotr Prins
 info.call GEMMA_K_BANNER
 
 # Check gemma version
-GEMMA_COMMAND=options[:gemma_command]
-info.call "NOTE: gemma-wrapper is soon to be replaced by gemma2/lib"
-
 begin
-  GEMMA_INFO = `#{GEMMA_COMMAND}`
+  gemma_command2 = options[:gemma_command]
+  info.call "NOTE: gemma-wrapper is soon to be replaced by gemma2/lib"
+
+  GEMMA_INFO = `#{gemma_command2}`
 rescue Errno::ENOENT
-  GEMMA_COMMAND = "gemma" if not GEMMA_COMMAND
-  error.call "<#{GEMMA_COMMAND}> command not found"
+  gemma_command2 = "gemma"
+  error.call "<#{gemma_command2}> command not found"
 end
 
 gemma_version_header = GEMMA_INFO.split("\n").grep(/GEMMA|Version/)[0].strip
@@ -230,15 +251,21 @@ if RUBY_VERSION =~ /^1/
   warning "runs on Ruby 2.x only\n"
 end
 
+# ---- LOCO defaults to parallel
+if options[:parallel] == nil
+  options[:parallel] = true if options[:loco]
+end
+
 debug.call(options) # some debug output
 debug.call(record)
 
 DO_COMPUTE_KINSHIP = gemma_args.include?("-gk")
 DO_COMPUTE_GWA = !DO_COMPUTE_KINSHIP
 
-# ---- Set up parallel
 if options[:parallel]
   begin
+    skip_cite = `echo "will cite" |parallel --citation`
+    debug.call(skip_cite)
     PARALLEL_INFO = `parallel --help`
   rescue Errno::ENOENT
     error.call "<parallel> command not found"
@@ -246,6 +273,11 @@ if options[:parallel]
   parallel_cmds = []
 end
 
+# ---- Fetch chromosomes from SNP annotation file
+anno_idx = gemma_args.index '-a'
+raise "Expected GEMMA -a genotype file switch" if anno_idx == nil
+CHROMOSOMES = get_chromosomes(gemma_args[anno_idx+1])
+
 # ---- Compute HASH on inputs
 hashme = []
 geno_idx = gemma_args.index '-g'
@@ -256,7 +288,6 @@ if DO_COMPUTE_GWA and options[:permute_phenotypes]
   raise "Did not expect GEMMA -p phenotype whith permutations (only use --permutate-phenotypes)" if pheno_idx
 end
 
-
 execute = lambda { |cmd|
   info.call("Executing: #{cmd}")
   err = 0
@@ -276,14 +307,6 @@ execute = lambda { |cmd|
   err
 }
 
-hashme =
-  if DO_COMPUTE_KINSHIP and pheno_idx != nil
-    # Remove the phenotype file from the hash for GRM computation
-    gemma_args[0..pheno_idx-1] + gemma_args[pheno_idx+2..-1]
-  else
-    gemma_args
-  end
-
 compute_hash = lambda do | phenofn = nil |
   # Compute a HASH on the inputs
   debug.call "Hashing on ",hashme,"\n"
@@ -302,31 +325,51 @@ compute_hash = lambda do | phenofn = nil |
       hashes << item
     end
   end
+  debug.call(hashes)
   Digest::SHA1.hexdigest hashes.join(' ')
 end
 
 HASH = compute_hash.call()
 options[:hash] = HASH
 
+at_exit do
+  Lock.release(HASH)
+end
+
+Lock.create(HASH) # this will wait for a lock to expire
+
+joblog = options[:cache_dir]+"/"+HASH+"-parallel.log"
+
 # Create cache dir
 FileUtils::mkdir_p options[:cache_dir]
 
+Dir.mktmpdir do |tmpdir| # tmpdir for GEMMA output
+
 error.call "Do not use the GEMMA -o switch!" if gemma_args.include? '-o'
 error.call "Do not use the GEMMA -outdir switch!" if gemma_args.include? '-outdir'
+GEMMA_ARGS_HASH = gemma_args.dup # do not include outdir
 gemma_args << '-outdir'
-gemma_args << options[:cache_dir]
+gemma_args << tmpdir
 GEMMA_ARGS = gemma_args
 
+hashme =
+  if DO_COMPUTE_KINSHIP and pheno_idx != nil
+    # Remove the phenotype file from the hash for GRM computation
+    GEMMA_ARGS_HASH[0..pheno_idx-1] + GEMMA_ARGS_HASH[pheno_idx+2..-1]
+  else
+    GEMMA_ARGS_HASH
+  end
+
 debug.call "Options: ",options,"\n" if !options[:quiet]
 
 invoke_gemma = lambda do |extra_args, cache_hit = false, chr = "full", permutation = 1|
-  cmd = "#{GEMMA_COMMAND} #{GEMMA_ARGS.join(' ')} #{extra_args.join(' ')}"
+  cmd = "#{gemma_command2} #{GEMMA_ARGS.join(' ')} #{extra_args.join(' ')}"
   record[:gemma_command] = cmd
   return if cache_hit
   if options[:slurm]
     info.call cmd
     hashi = HASH
-    prefix = options[:cache_dir]+'/'+hashi
+    prefix = tmpdir+'/'+hashi
     scriptfn  = prefix+".#{chr}.#{permutation}-pbs.sh"
     script = "#!/bin/bash
 #SBATCH --job-name=gemma-#{scriptfn}
@@ -371,6 +414,7 @@ srun #{cmd}
   end
 end
 
+# Takes the hash value and checks whether the (output) file exists
 # returns datafn, logfn, cache_hit
 cache = lambda do | chr, ext, h=HASH, permutation=0 |
   inject = (chr==nil ? "" : ".#{chr}" )+ext
@@ -428,11 +472,17 @@ gwas = lambda do | chr, kfn, pfn, permutation=0 |
 end
 
 LOCO = options[:loco]
+if LOCO
+  if options[:chromosomes]
+    CHROMOSOMES = options[:chromosomes]
+  end
+end
+
 if DO_COMPUTE_KINSHIP
   # compute K
-  info.call LOCO
-  if LOCO != nil
-    LOCO.each do |chr|
+  info.call CHROMOSOMES
+  if LOCO
+    CHROMOSOMES.each do |chr|
       info.call "LOCO for ",chr
       kinship.call(chr)
     end
@@ -441,13 +491,24 @@ if DO_COMPUTE_KINSHIP
   end
 else
   # DO_COMPUTE_GWA
-  json_in = JSON.parse(File.read(options[:input]))
+  begin
+    json_in = JSON.parse(File.read(options[:input]))
+  rescue TypeError
+    raise "Missing JSON input file?"
+  end
   raise "JSON problem, file #{options[:input]} is not -gk derived" if json_in["type"] != "K"
 
   pfn = options[:permute_phenotypes] # can be nil
-  k_files = json_in["files"].map { |rec| [rec[0],rec[2]] }
-  k_files.each do | chr, kfn | # call a GWA for each chromosome
-    gwas.call(chr,kfn,pfn)
+  if LOCO
+    k_files = json_in["files"].map { |rec| [rec[0],rec[2]] }
+    k_files.each do | chr, kfn | # call a GWA for each chromosome
+      gwas.call(chr,kfn,pfn)
+    end
+  else
+    kfn = json_in["files"][0][2]
+    CHROMOSOMES.each do | chr |
+      gwas.call(chr,kfn,pfn)
+    end
   end
   # Permute
   if options[:permutate]
@@ -502,18 +563,36 @@ if options[:parallel]
   cmd = parallel_cmds.join("\\n")
 
   cmd = "echo -e \"#{cmd}\""
-  err = execute.call(cmd+"|parallel") # all jobs in parallel
+  err = execute.call(cmd+"|parallel --joblog #{joblog}") # first try optimistically to run all jobs in parallel
   if err != 0
     [16,8,4,1].each do |jobs|
       info.call("Failed to complete parallel run -- retrying with smaller RAM footprint!")
-      err = execute.call(cmd+"|parallel -j #{jobs}")
+      err = execute.call(cmd+"|parallel -j #{jobs} --resume --joblog #{joblog}")
       break if err == 0
     end
     if err != 0
       info.call("Run failed!")
+      # Remove remaining files
+      FileUtils.rm_rf("#{tmpdir}/*", secure: true)
+      FileUtils.mv joblog, joblog+".bak", verbose: false, force: true
       exit err
     end
   end
   info.call("Run successful!")
+  FileUtils.mv joblog, joblog+".bak", verbose: false, force: true
 end
 json_out.call
+
+# copy all output files to the cache_dir. If a file exists only emit a warning
+Dir.glob("*.txt", base: tmpdir) do | fn |
+  source = tmpdir + "/" + fn
+  dest = options[:cache_dir] + "/" + fn
+  if not File.exist?(dest) or options[:force]
+    info.call "Move #{source} to #{dest}"
+    FileUtils.mv source, dest, verbose: false
+  else
+    warning.call "File #{dest} already exists. Not overwriting"
+  end
+end
+
+end # tmpdir

data/gemma-wrapper.gemspec

@@ -2,10 +2,11 @@ Gem::Specification.new do |s|
   s.name        = 'bio-gemma-wrapper'
   s.version     = File.read('VERSION')
   s.summary     = "GEMMA with LOCO and permutations"
-  s.description = "GEMMA wrapper adds LOCO and permutation support. Also caches K between runs with LOCO support"
+  s.description = "GEMMA wrapper adds LOCO and permutation support. Also runs in parallel and caches K between runs with LOCO support"
   s.authors     = ["Pjotr Prins"]
   s.email       = 'pjotr.public01@thebird.nl'
   s.files       = ["bin/gemma-wrapper",
+                   "lib/lock.rb",
                    "Gemfile",
                    "LICENSE.txt",
                    "README.md",

data/lib/lock.rb

@@ -0,0 +1,95 @@
+# Locking module for gemma (wrapper)
+#
+
+=begin
+
+The logic is as follows:
+
+1. a program creates a named lock file (based on a hash of its inputs) with its PID
+2. on exit it destroys the file
+3. a new program checks for the lock file
+4. if it exists and the PID is still in the ps table - wait
+5. when the pid disappears or the lock file - continue
+6. a timeout will return an error in 3 minutes
+
+Note that there is a theoretical chance the lock file existed, but disappeared. I think I have it covered by ignoring the unlink errors. Also the use of /proc/PID is Linux specific.
+
+=end
+
+
+require 'timeout'
+
+module Lock
+
+  def self.local name
+    ENV['HOME']+"/."+name.gsub("/","-")+".lck"
+  end
+
+  def self.lock_pid name
+    lockfn = local(name)
+    if File.exist?(lockfn)
+      File.read(lockfn).to_i
+    else
+      0
+    end
+  end
+
+  def self.locked? name
+    lockfn = local(name)
+    pid = lock_pid(name)
+    if File.exist?("/proc/#{pid}")
+      true
+    else
+      # the program went away - remove any 'stale' lock
+      begin
+        File.unlink(lockfn)
+      rescue Errno::ENOENT
+        # ignore error when the lock file went missing
+      end
+      false # --> no longer locked
+    end
+  end
+
+  def Lock::create name
+    wait_for(name)
+    lockfn = local(name)
+    if File.exist?(lockfn)
+      $stderr.print "\nERROR: Can not steal #{lockfn}"
+      exit 1
+    end
+    File.open(lockfn, File::RDWR|File::CREAT, 0644) do |f|
+      f.flock(File::LOCK_EX)
+      f.print(Process.pid)
+    end
+  end
+
+  def Lock::wait_for name
+    lockfn = local(name)
+    begin
+      status = Timeout::timeout(180) { # 3 minutes
+        while locked?(name)
+          $stderr.print("\nWaiting for lock #{lockfn}...")
+          sleep 2
+        end
+      }
+    rescue Timeout::Error
+      $stderr.print "\nERROR: Timed out, but I can not steal #{lockfn}"
+      exit 1
+    end
+    # yah! lock is released
+  end
+
+  def Lock::release name
+    lockfn = local(name)
+    if Process.pid == lock_pid(name)
+      begin
+        File.unlink(lockfn) # PID expired
+      rescue Errno::ENOENT
+        # ignore error when the lock file went missing
+      end
+    else
+      $stderr.print "\nERROR: can not release #{lockfn} because it is not owned by me"
+    end
+  end
+
+end

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: bio-gemma-wrapper
 version: !ruby/object:Gem::Version
-  version: 0.99.1
+  version: 0.99.5
 platform: ruby
 authors:
 - Pjotr Prins
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-11 00:00:00.000000000 Z
+date: 2021-11-26 00:00:00.000000000 Z
 dependencies: []
-description: GEMMA wrapper adds LOCO and permutation support. Also caches K between
-  runs with LOCO support
+description: GEMMA wrapper adds LOCO and permutation support. Also runs in parallel
+  and caches K between runs with LOCO support
 email: pjotr.public01@thebird.nl
 executables:
 - gemma-wrapper
@@ -24,6 +24,7 @@ files:
 - VERSION
 - bin/gemma-wrapper
 - gemma-wrapper.gemspec
+- lib/lock.rb
 homepage: https://github.com/genetics-statistics/gemma-wrapper
 licenses:
 - GPL3
@@ -43,8 +44,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: GEMMA with LOCO and permutations