bio-gemma-wrapper 0.99.2 → 0.99.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e27a8a3abb00b758095df5956b3854674faf5ff681a93bc028df273c40125c0d
-  data.tar.gz: e9675dbb0ea0f087dd21774635d38f3cda11b46a88b36c77dd308086fd0ec5f2
+  metadata.gz: 69d74ac5f1a705132d7ddc86bd1182c11fc37cf32062dbb28909f16684a827cb
+  data.tar.gz: 6f912b3c03474c1334a105d6c9471c02d39c205e94b0d07e5281718beec65ee7
 SHA512:
-  metadata.gz: 81cf5440fa531d5a831efa787800c8bea230d47cddc666a31fff066551ff347708a41ddf1368c0d3946c7ba9faef8e5882e398ad340850253c53961cce96f662
-  data.tar.gz: 582ae78c48a1eb8eeca01172eaeaba9d5ca23e69601967e334f8c218e3a4dd74b297861b01ce49b1357798b49a96c12e737100dcacec7fc34b70da1fc9c75f0d
+  metadata.gz: 35fbdf4ccfc482f6898e35b1e46e840db87f34efb35a096206ffef4a131ac8b21a4e6b7893be6ee053a59837eadba6547258d2360d796794c71a60670458943f
+  data.tar.gz: 43f7a7438f475583930e6cf1f50d34206fba7674b3df836791cf7064bd7ca55dddc2cc101f7c5a89d8042645b6dd06245b84d3b38501bc1eab7d18774703a01a

data/README.md

@@ -8,11 +8,12 @@ Nat. Genet., 2016)](cfw.gif)
 ## Introduction
 
 Gemma-wrapper allows running GEMMA with LOCO, GEMMA with caching,
-GEMMA in parallel (now the default), and GEMMA on PBS. Gemma-wrapper
-is used to run GEMMA as part of the https://genenetwork.org/
-environment.
+GEMMA in parallel (now the default with LOCO), and GEMMA on
+PBS. Gemma-wrapper is used to run GEMMA as part of the
+https://genenetwork.org/ environment.
 
-Note that gemma-wrapper is projected to be integrated into gemma2/lib.
+Note that a version of gemma-wrapper is projected to be integrated
+into gemma itself.
 
 GEMMA is a software toolkit for fast application of linear mixed
 models (LMMs) and related models to genome-wide association studies
@@ -29,6 +30,14 @@ does a pass-through of all standard GEMMA invocation switches. On
 return gemma-wrapper can return a JSON object (--json) which is
 useful for web-services.
 
+## Performance
+
+LOCO runs in parallel by default which is at least a 5x performance
+improvement on a machine with enough cores. GEMMA without LOCO,
+however, does not run in parallel by default.  Performance
+improvements with the parallel implementation for LOCO and non-LOCO
+can be viewed [here](./test/performance/releases.gmi).
+
 ## Installation
 
 Prerequisites are
@@ -53,15 +62,19 @@ and it will render something like
 Usage: gemma-wrapper [options] -- [gemma-options]
         --permutate n                Permutate # times by shuffling phenotypes
         --permute-phenotypes filen   Phenotypes to be shuffled in permutations
-        --loco [x,y,1,2,3...]        Run full LOCO
+        --loco                       Run full leave-one-chromosome-out (LOCO)
+        --chromosomes [1,2,3]        Run specific chromosomes
         --input filen                JSON input variables (used for LOCO)
         --cache-dir path             Use a cache directory
         --json                       Create output file in JSON format
-        --force                      Force computation
-        --slurm [options]            Submit to slurm PBS
+        --force                      Force computation (override cache)
+        --parallel                   Run jobs in parallel
+        --no-parallel                Do not run jobs in parallel
+        --slurm[=opts]               Use slurm PBS for submitting jobs
         --q, --quiet                 Run quietly
     -v, --verbose                    Run verbosely
-        --debug                      Show debug messages and keep intermediate output
+    -d, --debug                      Show debug messages and keep intermediate output
+        --dry-run                    Show commands, but don't execute
         --                           Anything after gets passed to GEMMA
 
     -h, --help                       display this help and exit
@@ -99,6 +112,7 @@ the data files are found):
     gemma-wrapper -- \
         -g test/data/input/BXD_geno.txt.gz \
         -p test/data/input/BXD_pheno.txt \
+        -a test/data/input/BXD_snps.txt \
         -gk \
         -debug
 
@@ -116,6 +130,7 @@ You can also get JSON output on STDOUT by providing the --json switch
     gemma-wrapper --json -- \
         -g test/data/input/BXD_geno.txt.gz \
         -p test/data/input/BXD_pheno.txt \
+        -a test/data/input/BXD_snps.txt \
         -gk \
         -debug > K.json
 
@@ -133,6 +148,7 @@ default. If you want something else provide a --cache-dir, e.g.
     gemma-wrapper --cache-dir ~/.gemma-cache -- \
         -g test/data/input/BXD_geno.txt.gz \
         -p test/data/input/BXD_pheno.txt \
+        -a test/data/input/BXD_snps.txt \
         -gk \
         -debug
 
@@ -143,7 +159,7 @@ will store K in ~/.gemma-cache.
 Run the LMM using the K's captured earlier in K.json using the --input
 switch
 
-    gemma-wrapper --json --loco --input K.json -- \
+    gemma-wrapper --json --input K.json -- \
         -g test/data/input/BXD_geno.txt.gz \
         -p test/data/input/BXD_pheno.txt \
         -c test/data/input/BXD_covariates2.txt \
@@ -163,7 +179,7 @@ https://github.com/genetics-statistics/GEMMA/issues/46). To loop all
 chromosomes first create all K's with
 
     gemma-wrapper --json \
-        --loco 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,X -- \
+        --loco -- \
         -g test/data/input/BXD_geno.txt.gz \
         -p test/data/input/BXD_pheno.txt \
         -a test/data/input/BXD_snps.txt \

data/VERSION

@@ -1 +1 @@
-0.99.2
+0.99.6

data/bin/gemma-wrapper

@@ -4,7 +4,7 @@
 # Author:: Pjotr Prins
 # License:: GPL3
 #
-# Copyright (C) 2017-2021 Pjotr Prins <pjotr.prins@thebird.nl>
+# Copyright (C) 2017-2022 Pjotr Prins <pjotr.prins@thebird.nl>
 
 USAGE = "
 GEMMA wrapper example:
@@ -14,12 +14,12 @@ GEMMA wrapper example:
       gemma-wrapper -- \\
         -g test/data/input/BXD_geno.txt.gz \\
         -p test/data/input/BXD_pheno.txt \\
+        -a test/data/input/BXD_snps.txt \
         -gk
 
     LOCO K computation with caching and JSON output
 
-      gemma-wrapper --json \\
-        --loco 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,X -- \\
+      gemma-wrapper --json --loco -- \\
         -g test/data/input/BXD_geno.txt.gz \\
         -p test/data/input/BXD_pheno.txt \\
         -a test/data/input/BXD_snps.txt \\
@@ -64,6 +64,7 @@ if not gemma_command
   end
 end
 
+hashme = nil
 
 require 'digest/sha1'
 require 'fileutils'
@@ -71,12 +72,15 @@ require 'optparse'
 require 'tempfile'
 require 'tmpdir'
 
+require 'lock'
+
 split_at = ARGV.index('--')
+
 if split_at
   gemma_args = ARGV[split_at+1..-1]
 end
 
-options = { show_help: false, source: 'https://github.com/genetics-statistics/gemma-wrapper', version: version+' (Pjotr Prins)', date: Time.now.to_s, gemma_command: gemma_command, cache_dir: Dir.tmpdir(), quiet: false, parallel: true }
+options = { show_help: false, source: 'https://github.com/genetics-statistics/gemma-wrapper', version: version+' (Pjotr Prins)', date: Time.now.to_s, gemma_command: gemma_command, cache_dir: Dir.tmpdir(), quiet: false, permute_phenotypes: false, parallel: nil }
 
 opts = OptionParser.new do |o|
   o.banner = "\nUsage: #{File.basename($0)} [options] -- [gemma-options]"
@@ -91,8 +95,12 @@ opts = OptionParser.new do |o|
     raise "Phenotype input file #{phenotypes} does not exist" if !File.exist?(phenotypes)
   end
 
-  o.on('--loco [x,y,1,2,3...]', Array, 'Run full leave-one-chromosome-out (LOCO)') do |lst|
-    options[:loco] = lst
+  o.on('--loco', 'Run full leave-one-chromosome-out (LOCO)') do |b|
+    options[:loco] = b
+  end
+
+  o.on('--chromosomes [1,2,3]',Array,'Run specific chromosomes') do |lst|
+    options[:chromosomes] = lst
   end
 
   o.on('--input filen',String, 'JSON input variables (used for LOCO)') do |filen|
@@ -112,6 +120,10 @@ opts = OptionParser.new do |o|
     options[:force] = true
   end
 
+  o.on("--parallel", "Run jobs in parallel") do |b|
+    options[:parallel] = true
+  end
+
   o.on("--no-parallel", "Do not run jobs in parallel") do |b|
     options[:parallel] = false
   end
@@ -185,26 +197,36 @@ warning = lambda do |*msg|
   record[:warnings].push *msg.join("")
   OUTPUT.print "WARNING: ",*msg,"\n"
 end
+
 info = lambda do |*msg|
   record[:debug].push *msg.join("") if options[:json] and options[:debug]
   OUTPUT.print *msg,"\n" if !options[:quiet]
 end
 
+# Fetch chromosomes
+def get_chromosomes annofn
+  h = {}
+  File.open(annofn,"r").each_line do | line |
+    chr = line.split(/\s+/)[2]
+    h[chr] = true
+  end
+  h.map { |k,v| k }
+end
 # ---- Start banner
 
 GEMMA_K_VERSION=version
-GEMMA_K_BANNER = "gemma-wrapper #{version} (Ruby #{RUBY_VERSION}) by Pjotr Prins 2017-2021\n"
+GEMMA_K_BANNER = "gemma-wrapper #{version} (Ruby #{RUBY_VERSION}) by Pjotr Prins 2017-2022\n"
 info.call GEMMA_K_BANNER
 
 # Check gemma version
-GEMMA_COMMAND=options[:gemma_command]
-info.call "NOTE: gemma-wrapper is soon to be replaced by gemma2/lib"
-
 begin
-  GEMMA_INFO = `#{GEMMA_COMMAND}`
+  gemma_command2 = options[:gemma_command]
+  info.call "NOTE: gemma-wrapper is soon to be replaced"
+
+  GEMMA_INFO = `#{gemma_command2}`
 rescue Errno::ENOENT
-  GEMMA_COMMAND = "gemma" if not GEMMA_COMMAND
-  error.call "<#{GEMMA_COMMAND}> command not found"
+  gemma_command2 = "gemma"
+  error.call "<#{gemma_command2}> command not found"
 end
 
 gemma_version_header = GEMMA_INFO.split("\n").grep(/GEMMA|Version/)[0].strip
@@ -230,13 +252,17 @@ if RUBY_VERSION =~ /^1/
   warning "runs on Ruby 2.x only\n"
 end
 
+# ---- LOCO defaults to parallel
+if options[:parallel] == nil
+  options[:parallel] = true if options[:loco]
+end
+
 debug.call(options) # some debug output
 debug.call(record)
 
 DO_COMPUTE_KINSHIP = gemma_args.include?("-gk")
 DO_COMPUTE_GWA = !DO_COMPUTE_KINSHIP
 
-# ---- Set up parallel
 if options[:parallel]
   begin
     skip_cite = `echo "will cite" |parallel --citation`
@@ -248,8 +274,12 @@ if options[:parallel]
   parallel_cmds = []
 end
 
+# ---- Fetch chromosomes from SNP annotation file
+anno_idx = gemma_args.index '-a'
+raise "Expected GEMMA -a genotype file switch" if anno_idx == nil
+CHROMOSOMES = get_chromosomes(gemma_args[anno_idx+1])
+
 # ---- Compute HASH on inputs
-hashme = []
 geno_idx = gemma_args.index '-g'
 raise "Expected GEMMA -g genotype file switch" if geno_idx == nil
 pheno_idx = gemma_args.index '-p'
@@ -279,14 +309,15 @@ execute = lambda { |cmd|
 
 compute_hash = lambda do | phenofn = nil |
   # Compute a HASH on the inputs
-  debug.call "Hashing on ",hashme,"\n"
+  error.call "Hash is empty" if hashme == nil or hashme == []
+  debug.call "Hashing on ",hashme," before phenofn inject"
   hashes = []
   hm = if phenofn
          hashme + ["-p", phenofn]
        else
          hashme
        end
-  debug.call(hm)
+  debug.call("Hashing on ",hm)
   hm.each do | item |
     if File.file?(item)
       hashes << Digest::SHA1.hexdigest(File.read(item))
@@ -299,20 +330,9 @@ compute_hash = lambda do | phenofn = nil |
   Digest::SHA1.hexdigest hashes.join(' ')
 end
 
-HASH = compute_hash.call()
-options[:hash] = HASH
-
-# Create cache dir
-FileUtils::mkdir_p options[:cache_dir]
-
-Dir.mktmpdir do |tmpdir| # tmpdir for GEMMA output
-
 error.call "Do not use the GEMMA -o switch!" if gemma_args.include? '-o'
 error.call "Do not use the GEMMA -outdir switch!" if gemma_args.include? '-outdir'
 GEMMA_ARGS_HASH = gemma_args.dup # do not include outdir
-gemma_args << '-outdir'
-gemma_args << tmpdir
-GEMMA_ARGS = gemma_args
 
 hashme =
   if DO_COMPUTE_KINSHIP and pheno_idx != nil
@@ -322,10 +342,30 @@ hashme =
     GEMMA_ARGS_HASH
   end
 
+HASH = compute_hash.call()
+options[:hash] = HASH
+
+at_exit do
+  Lock.release(HASH)
+end
+
+Lock.create(HASH) # this will wait for a lock to expire
+
+joblog = options[:cache_dir]+"/"+HASH+"-parallel.log"
+
+# Create cache dir
+FileUtils::mkdir_p options[:cache_dir]
+
+Dir.mktmpdir do |tmpdir| # tmpdir for GEMMA output
+
+gemma_args << '-outdir'
+gemma_args << tmpdir
+GEMMA_ARGS = gemma_args
+
 debug.call "Options: ",options,"\n" if !options[:quiet]
 
 invoke_gemma = lambda do |extra_args, cache_hit = false, chr = "full", permutation = 1|
-  cmd = "#{GEMMA_COMMAND} #{GEMMA_ARGS.join(' ')} #{extra_args.join(' ')}"
+  cmd = "#{gemma_command2} #{GEMMA_ARGS.join(' ')} #{extra_args.join(' ')}"
   record[:gemma_command] = cmd
   return if cache_hit
   if options[:slurm]
@@ -434,11 +474,17 @@ gwas = lambda do | chr, kfn, pfn, permutation=0 |
 end
 
 LOCO = options[:loco]
+if LOCO
+  if options[:chromosomes]
+    CHROMOSOMES = options[:chromosomes]
+  end
+end
+
 if DO_COMPUTE_KINSHIP
   # compute K
-  info.call LOCO
-  if LOCO != nil
-    LOCO.each do |chr|
+  info.call CHROMOSOMES
+  if LOCO
+    CHROMOSOMES.each do |chr|
       info.call "LOCO for ",chr
       kinship.call(chr)
     end
@@ -447,13 +493,24 @@ if DO_COMPUTE_KINSHIP
   end
 else
   # DO_COMPUTE_GWA
-  json_in = JSON.parse(File.read(options[:input]))
+  begin
+    json_in = JSON.parse(File.read(options[:input]))
+  rescue TypeError
+    raise "Missing JSON input file?"
+  end
   raise "JSON problem, file #{options[:input]} is not -gk derived" if json_in["type"] != "K"
 
   pfn = options[:permute_phenotypes] # can be nil
-  k_files = json_in["files"].map { |rec| [rec[0],rec[2]] }
-  k_files.each do | chr, kfn | # call a GWA for each chromosome
-    gwas.call(chr,kfn,pfn)
+  if LOCO
+    k_files = json_in["files"].map { |rec| [rec[0],rec[2]] }
+    k_files.each do | chr, kfn | # call a GWA for each chromosome
+      gwas.call(chr,kfn,pfn)
+    end
+  else
+    kfn = json_in["files"][0][2]
+    CHROMOSOMES.each do | chr |
+      gwas.call(chr,kfn,pfn)
+    end
   end
   # Permute
   if options[:permutate]
@@ -505,22 +562,34 @@ end
 # ---- Invoke parallel
 if options[:parallel]
   # parallel_cmds = ["echo 1","sleep 1 && echo 2", "false", "echo 3"]
-  cmd = parallel_cmds.join("\\n")
-
-  cmd = "echo -e \"#{cmd}\""
-  err = execute.call(cmd+"|parallel") # all jobs in parallel
-  if err != 0
-    [16,8,4,1].each do |jobs|
-      info.call("Failed to complete parallel run -- retrying with smaller RAM footprint!")
-      err = execute.call(cmd+"|parallel -j #{jobs}")
-      break if err == 0
+
+  Tempfile.open("commands.txt") do |f|
+    cmdfn = f.path
+    File.open(cmdfn,"w") do |f|
+      parallel_cmds.each do |c|
+        f.puts(c)
+      end
     end
+    cmd = "cat \"#{cmdfn}\""
+    err = execute.call(cmd+"|parallel --joblog #{joblog}") # first try optimistically to run all jobs in parallel
     if err != 0
-      info.call("Run failed!")
-      exit err
+      [16,8,4,1].each do |jobs|
+        info.call("Failed to complete parallel run -- retrying with smaller RAM footprint!")
+        err = execute.call(cmd+"|parallel -j #{jobs} --resume --joblog #{joblog}")
+        break if err == 0
+      end
+      if err != 0
+        info.call("Parallel run failed!")
+        debug.call("Job log is: ",File.read(joblog))
+        # Remove remaining files
+        FileUtils.mv joblog, joblog+".bak", verbose: false, force: true
+        FileUtils.rm_rf("#{tmpdir}/*", secure: true)
+        exit err
+      end
     end
   end
   info.call("Run successful!")
+  FileUtils.mv joblog, joblog+".bak", verbose: false, force: true
 end
 json_out.call
 

data/gemma-wrapper.gemspec

@@ -6,6 +6,7 @@ Gem::Specification.new do |s|
   s.authors     = ["Pjotr Prins"]
   s.email       = 'pjotr.public01@thebird.nl'
   s.files       = ["bin/gemma-wrapper",
+                   "lib/lock.rb",
                    "Gemfile",
                    "LICENSE.txt",
                    "README.md",

data/lib/lock.rb

@@ -0,0 +1,95 @@
+# Locking module for gemma (wrapper)
+#
+
+=begin
+
+The logic is as follows:
+
+1. a program creates a named lock file (based on a hash of its inputs) with its PID
+2. on exit it destroys the file
+3. a new program checks for the lock file
+4. if it exists and the PID is still in the ps table - wait
+5. when the pid disappears or the lock file - continue
+6. a timeout will return an error in 3 minutes
+
+Note that there is a theoretical chance the lock file existed, but disappeared. I think I have it covered by ignoring the unlink errors. Also the use of /proc/PID is Linux specific.
+
+=end
+
+
+require 'timeout'
+
+module Lock
+
+  def self.local name
+    ENV['HOME']+"/."+name.gsub("/","-")+".lck"
+  end
+
+  def self.lock_pid name
+    lockfn = local(name)
+    if File.exist?(lockfn)
+      File.read(lockfn).to_i
+    else
+      0
+    end
+  end
+
+  def self.locked? name
+    lockfn = local(name)
+    pid = lock_pid(name)
+    if File.exist?("/proc/#{pid}")
+      true
+    else
+      # the program went away - remove any 'stale' lock
+      begin
+        File.unlink(lockfn)
+      rescue Errno::ENOENT
+        # ignore error when the lock file went missing
+      end
+      false # --> no longer locked
+    end
+  end
+
+  def Lock::create name
+    wait_for(name)
+    lockfn = local(name)
+    if File.exist?(lockfn)
+      $stderr.print "\nERROR: Can not steal #{lockfn}"
+      exit 1
+    end
+    File.open(lockfn, File::RDWR|File::CREAT, 0644) do |f|
+      f.flock(File::LOCK_EX)
+      f.print(Process.pid)
+    end
+  end
+
+  def Lock::wait_for name
+    lockfn = local(name)
+    begin
+      status = Timeout::timeout(180) { # 3 minutes
+        while locked?(name)
+          $stderr.print("\nWaiting for lock #{lockfn}...")
+          sleep 2
+        end
+      }
+    rescue Timeout::Error
+      $stderr.print "\nERROR: Timed out, but I can not steal #{lockfn}"
+      exit 1
+    end
+    # yah! lock is released
+  end
+
+  def Lock::release name
+    lockfn = local(name)
+    if Process.pid == lock_pid(name)
+      begin
+        File.unlink(lockfn) # PID expired
+      rescue Errno::ENOENT
+        # ignore error when the lock file went missing
+      end
+    else
+      $stderr.print "\nERROR: can not release #{lockfn} because it is not owned by me"
+    end
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bio-gemma-wrapper
 version: !ruby/object:Gem::Version
-  version: 0.99.2
+  version: 0.99.6
 platform: ruby
 authors:
 - Pjotr Prins
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-08 00:00:00.000000000 Z
+date: 2022-01-22 00:00:00.000000000 Z
 dependencies: []
 description: GEMMA wrapper adds LOCO and permutation support. Also runs in parallel
   and caches K between runs with LOCO support
@@ -24,11 +24,12 @@ files:
 - VERSION
 - bin/gemma-wrapper
 - gemma-wrapper.gemspec
+- lib/lock.rb
 homepage: https://github.com/genetics-statistics/gemma-wrapper
 licenses:
 - GPL3
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -43,8 +44,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.5
-signing_key: 
+rubygems_version: 3.2.22
+signing_key:
 specification_version: 4
 summary: GEMMA with LOCO and permutations
 test_files: []