bio-gemma-wrapper 0.98.1 → 0.99.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: b7a8bfe787236f397dba6e05aef202486ba53389
-  data.tar.gz: 9fe2398ed3fcd053e8258f73a64b1610bfddf9e2
+SHA256:
+  metadata.gz: 9ddfd904e74beebe0de1b97732d872fce171732965a835b101b9cc9be815bb05
+  data.tar.gz: 2dae1c019da23f2f87216694d641fc1eb852aa7800557bd10cfb08cb3425e844
 SHA512:
-  metadata.gz: 0a691906f13da3469597517d160874315f5962822b0273757d33c37283c2a9a98da9a41d08c51e4ed7854b3d3593f9d96ac81d59212690edd628a2677499d501
-  data.tar.gz: 5d4b898ff3566f52652cbb7db6597dc245e9a5da6b0d9da107de81deec0d51c02c822762e0ab921a2309c1dbc3455b55c1adc5fe94c891e31a9c32cb6d343f2e
+  metadata.gz: 38454a3f12dab85bef711051e73e20a015fe6b6d9c71bafada2197b9aef1aa0eabe3f3709cb0dc9d0c39f4cc454c15bc4d3aea5d06140ccde72fa13aa6285f51
+  data.tar.gz: 28e77a6995893245c501e602d488b5e0c504549fa91d8c94f902591b87b4454fe9b7923667dfacae2ab1dac7f6f7d814df1ec036b2b4f616dfd4b84c549d35d1

data/README.md

@@ -1,12 +1,19 @@
 [![gemma-wrapper gem version](https://badge.fury.io/rb/bio-gemma-wrapper.svg)](https://badge.fury.io/rb/bio-gemma-wrapper)
 
-# GEMMA wrapper caches K between runs with LOCO support
+# GEMMA with LOCO, permutations and slurm support (and caching)
 
 ![Genetic associations identified in CFW mice using GEMMA (Parker et al,
 Nat. Genet., 2016)](cfw.gif)
 
 ## Introduction
 
+Gemma-wrapper allows running GEMMA with LOCO, GEMMA with caching,
+GEMMA in parallel (now the default), and GEMMA on PBS. Gemma-wrapper
+is used to run GEMMA as part of the https://genenetwork.org/
+environment.
+
+Note that gemma-wrapper is projected to be integrated into gemma2/lib.
+
 GEMMA is a software toolkit for fast application of linear mixed
 models (LMMs) and related models to genome-wide association studies
 (GWAS) and other large-scale data sets.
@@ -14,16 +21,14 @@ models (LMMs) and related models to genome-wide association studies
 This repository contains gemma-wrapper, essentially a wrapper of
 GEMMA that provides support for caching the kinship or relatedness
 matrix (K) and caching LM and LMM computations with the option of full
-leave-one-chromosome-out genome scans (LOCO).
+leave-one-chromosome-out genome scans (LOCO). Jobs can also be
+submitted to HPC PBS, i.e., slurm.
 
 gemma-wrapper requires a recent version of GEMMA and essentially
 does a pass-through of all standard GEMMA invocation switches. On
 return gemma-wrapper can return a JSON object (--json) which is
 useful for web-services.
 
-Note that this a work in progress (WIP). What is described below
-should work.
-
 ## Installation
 
 Prerequisites are
@@ -32,8 +37,9 @@ Prerequisites are
 * Standard [Ruby >2.0 ](https://www.ruby-lang.org/en/) which comes on
   almost all Linux systems
 
-gemma-wrapper comes as a Ruby [gem](https://rubygems.org/gems/bio-gemma-wrapper) and
-can be installed with
+gemma-wrapper comes as a Ruby
+[gem](https://rubygems.org/gems/bio-gemma-wrapper) and can be
+installed with
 
     gem install bio-gemma-wrapper
 
@@ -52,6 +58,7 @@ Usage: gemma-wrapper [options] -- [gemma-options]
         --cache-dir path             Use a cache directory
         --json                       Create output file in JSON format
         --force                      Force computation
+        --slurm [options]            Submit to slurm PBS
         --q, --quiet                 Run quietly
     -v, --verbose                    Run verbosely
         --debug                      Show debug messages and keep intermediate output
@@ -69,6 +76,8 @@ Unpack it and run the tool as
 
     ./bin/gemma-wrapper --help
 
+See below for using a GNU Guix environment.
+
 ## Usage
 
 gemma-wrapper picks up GEMMA from the PATH. To override that behaviour
@@ -95,7 +104,7 @@ the data files are found):
 
 Run it twice to see
 
-    /tmp/3079151e14b219c3b243b673d88001c1675168b4.log.txt gemma-wrapper CACHE HIT!
+    /tmp/0bdd7add5e8f7d9af36b283d0341c115124273e0.log.txt CACHE HIT!
 
 gemma-wrapper computes the unique HASH value over the command
 line switches passed into GEMMA as well as the contents of the files
@@ -108,9 +117,10 @@ You can also get JSON output on STDOUT by providing the --json switch
         -g test/data/input/BXD_geno.txt.gz \
         -p test/data/input/BXD_pheno.txt \
         -gk \
-        -debug
+        -debug > K.json
 
-prints out something that can be parsed with a calling program
+K.json is something that can be parsed with a calling program, and is
+also below as input for the GWA step. Example:
 
 ```json
 {"warnings":[],"errno":0,"debug":[],"type":"K","files":[["/tmp/18ce786ab92064a7ee38a7422e7838abf91f5eb0.log.txt","/tmp/18ce786ab92064a7ee38a7422e7838abf91f5eb0.cXX.txt"]],"cache_hit":true,"gemma_command":"../gemma/bin/gemma -g test/data/input/BXD_geno.txt.gz -p test/data/input/BXD_pheno.txt -gk -debug -outdir /tmp -o 18ce786ab92064a7ee38a7422e7838abf91f5eb0"}
@@ -130,7 +140,7 @@ will store K in ~/.gemma-cache.
 
 ### GWA
 
-Run the LMM using the K's captured in K.json using the --input
+Run the LMM using the K's captured earlier in K.json using the --input
 switch
 
     gemma-wrapper --json --loco --input K.json -- \
@@ -201,12 +211,24 @@ Next, using K.json, permute the phenotypes with something like
         -lmm 2 -maf 0.1 \
         -debug > GWA.json
 
-This should get the 95% significant and 67% suggestive thresholds:
+This should get the estimated 95% (significant) and 67% (suggestive) thresholds:
 
-    ["95 percentile (significant) ", 2.015475e-05, 4.7]
-    ["67 percentile (suggestive)  ", 2.015475e-05, 4.7]
+    ["95 percentile (significant) ", 1.92081e-05, 4.7]
+    ["67 percentile (suggestive)  ", 5.227785e-05, 4.3]
 
+### Slurm PBS
+
+To run gemma-wrapper on HPC use the '--slurm' switch.
+
+## Development
+
+We use GNU Guix for development and deployment. Use the [.guix-deploy](.guix-deploy) script in the checked out git repo:
+
+```
+source .guix-deploy
+ruby bin/gemma-wrapper --help
+```
 
 ## Copyright
 
-Copyright (c) 2017,2018 Pjotr Prins. See [LICENSE.txt](LICENSE.txt) for further details.
+Copyright (c) 2017-2021 Pjotr Prins. See [LICENSE.txt](LICENSE.txt) for further details.

data/VERSION

@@ -1 +1 @@
-0.98.1
+0.99.1

data/bin/gemma-wrapper

@@ -4,7 +4,7 @@
 # Author:: Pjotr Prins
 # License:: GPL3
 #
-# Copyright (C) 2017,2018 Pjotr Prins <pjotr.prins@thebird.nl>
+# Copyright (C) 2017-2021 Pjotr Prins <pjotr.prins@thebird.nl>
 
 USAGE = "
 GEMMA wrapper example:
@@ -38,11 +38,10 @@ GEMMA wrapper example:
 Gemma gets used from the path. You can override by setting
 
       env GEMMA_COMMAND=path/bin/gemma gemma-wrapper ...
-
 "
 # These are used for testing compatibility with the gemma tool
 GEMMA_V_MAJOR = 98
-GEMMA_V_MINOR = 0
+GEMMA_V_MINOR = 1
 
 basepath = File.dirname(File.dirname(__FILE__))
 $: << File.join(basepath,'lib')
@@ -66,17 +65,18 @@ if not gemma_command
 end
 
 
+require 'digest/sha1'
 require 'fileutils'
 require 'optparse'
-require 'tmpdir'
 require 'tempfile'
+require 'tmpdir'
 
 split_at = ARGV.index('--')
 if split_at
   gemma_args = ARGV[split_at+1..-1]
 end
 
-options = { show_help: false, source: 'https://github.com/genetics-statistics/gemma-wrapper', version: version+' (Pjotr Prins)', date: Time.now.to_s, gemma_command: gemma_command, cache_dir: Dir.tmpdir() }
+options = { show_help: false, source: 'https://github.com/genetics-statistics/gemma-wrapper', version: version+' (Pjotr Prins)', date: Time.now.to_s, gemma_command: gemma_command, cache_dir: Dir.tmpdir(), quiet: false, parallel: true }
 
 opts = OptionParser.new do |o|
   o.banner = "\nUsage: #{File.basename($0)} [options] -- [gemma-options]"
@@ -91,7 +91,7 @@ opts = OptionParser.new do |o|
     raise "Phenotype input file #{phenotypes} does not exist" if !File.exist?(phenotypes)
   end
 
-  o.on('--loco [x,y,1,2,3...]', Array, 'Run full LOCO') do |lst|
+  o.on('--loco [x,y,1,2,3...]', Array, 'Run full leave-one-chromosome-out (LOCO)') do |lst|
     options[:loco] = lst
   end
 
@@ -112,6 +112,18 @@ opts = OptionParser.new do |o|
     options[:force] = true
   end
 
+  o.on("--no-parallel", "Do not run jobs in parallel") do |b|
+    options[:parallel] = false
+  end
+
+  o.on("--slurm[=opts]",String,"Use slurm PBS for submitting jobs") do |slurm|
+    options[:slurm_opts] = ""
+    options[:slurm] = true
+    if slurm
+      options[:slurm_opts] = slurm
+    end
+  end
+
   o.on("--q", "--quiet", "Run quietly") do |q|
     options[:quiet] = true
   end
@@ -120,15 +132,20 @@ opts = OptionParser.new do |o|
     options[:verbose] = true
   end
 
-  o.on("--debug", "Show debug messages and keep intermediate output") do |v|
+  o.on("-d", "--debug", "Show debug messages and keep intermediate output") do |v|
     options[:debug] = true
   end
 
+  o.on("--dry-run", "Show commands, but don't execute") do |b|
+    options[:dry_run] = b
+  end
+
   o.on('--','Anything after gets passed to GEMMA') do
     o.terminate()
   end
 
   o.separator ""
+
   o.on_tail('-h', '--help', 'display this help and exit') do
     options[:show_help] = true
   end
@@ -176,18 +193,28 @@ end
 # ---- Start banner
 
 GEMMA_K_VERSION=version
-GEMMA_K_BANNER = "gemma-wrapper #{version} (Ruby #{RUBY_VERSION}) by Pjotr Prins 2017,2018\n"
+GEMMA_K_BANNER = "gemma-wrapper #{version} (Ruby #{RUBY_VERSION}) by Pjotr Prins 2017-2021\n"
 info.call GEMMA_K_BANNER
 
 # Check gemma version
 GEMMA_COMMAND=options[:gemma_command]
+info.call "NOTE: gemma-wrapper is soon to be replaced by gemma2/lib"
 
-gemma_version_header = `#{GEMMA_COMMAND}`.split("\n").grep(/GEMMA|Version/)[0].strip
+begin
+  GEMMA_INFO = `#{GEMMA_COMMAND}`
+rescue Errno::ENOENT
+  GEMMA_COMMAND = "gemma" if not GEMMA_COMMAND
+  error.call "<#{GEMMA_COMMAND}> command not found"
+end
+
+gemma_version_header = GEMMA_INFO.split("\n").grep(/GEMMA|Version/)[0].strip
 info.call "Using ",gemma_version_header,"\n"
 gemma_version = gemma_version_header.split(/[,\s]+/)[1]
 v_version, v_major, v_minor = gemma_version.split(".")
 info.call "Found #{gemma_version}, comparing against expected v0.#{GEMMA_V_MAJOR}.#{GEMMA_V_MINOR}"
 
+info.call gemma_version_header
+
 warning.call "GEMMA version is out of date. Update GEMMA to 0.#{GEMMA_V_MAJOR}.#{GEMMA_V_MINOR}!" if v_major.to_i < GEMMA_V_MAJOR or (v_major.to_i == GEMMA_V_MAJOR and (v_minor != nil and v_minor.to_i < GEMMA_V_MINOR))
 
 options[:gemma_version_header] = gemma_version_header
@@ -203,40 +230,82 @@ if RUBY_VERSION =~ /^1/
   warning "runs on Ruby 2.x only\n"
 end
 
+debug.call(options) # some debug output
+debug.call(record)
+
 DO_COMPUTE_KINSHIP = gemma_args.include?("-gk")
 DO_COMPUTE_GWA = !DO_COMPUTE_KINSHIP
 
+# ---- Set up parallel
+if options[:parallel]
+  begin
+    PARALLEL_INFO = `parallel --help`
+  rescue Errno::ENOENT
+    error.call "<parallel> command not found"
+  end
+  parallel_cmds = []
+end
+
 # ---- Compute HASH on inputs
 hashme = []
 geno_idx = gemma_args.index '-g'
 raise "Expected GEMMA -g genotype file switch" if geno_idx == nil
 pheno_idx = gemma_args.index '-p'
+
+if DO_COMPUTE_GWA and options[:permute_phenotypes]
+  raise "Did not expect GEMMA -p phenotype whith permutations (only use --permutate-phenotypes)" if pheno_idx
+end
+
+
+execute = lambda { |cmd|
+  info.call("Executing: #{cmd}")
+  err = 0
+  if not options[:debug]
+    # send output to stderr line by line
+    IO.popen("#{cmd}") do |io|
+      while s = io.gets
+        $stderr.print s
+      end
+      io.close
+      err = $?.to_i
+    end
+  else
+    $stderr.print `#{cmd}`
+    err = $?.to_i
+  end
+  err
+}
+
 hashme =
   if DO_COMPUTE_KINSHIP and pheno_idx != nil
-    # Remove the phenotype file from the hash
+    # Remove the phenotype file from the hash for GRM computation
     gemma_args[0..pheno_idx-1] + gemma_args[pheno_idx+2..-1]
   else
     gemma_args
   end
 
-if DO_COMPUTE_GWA and options[:permute_phenotypes]
-  raise "Did not expect GEMMA -p phenotype whith permutations (only use --permutate-phenotypes)" if pheno_idx
-  hashme += ['-p', options[:permute_phenotypes]]
-end
-
-require 'digest/sha1'
-debug.call "Hashing on ",hashme,"\n"
-hashes = []
-hashme.each do | item |
-  if File.exist?(item)
-    hashes << Digest::SHA1.hexdigest(File.read(item))
-    debug.call [item,hashes.last]
-  else
-    hashes << item
+compute_hash = lambda do | phenofn = nil |
+  # Compute a HASH on the inputs
+  debug.call "Hashing on ",hashme,"\n"
+  hashes = []
+  hm = if phenofn
+         hashme + ["-p", phenofn]
+       else
+         hashme
+       end
+  debug.call(hm)
+  hm.each do | item |
+    if File.file?(item)
+      hashes << Digest::SHA1.hexdigest(File.read(item))
+      debug.call [item,hashes.last]
+    else
+      hashes << item
+    end
   end
+  Digest::SHA1.hexdigest hashes.join(' ')
 end
-HASH = Digest::SHA1.hexdigest hashes.join(' ')
 
+HASH = compute_hash.call()
 options[:hash] = HASH
 
 # Create cache dir
@@ -250,27 +319,49 @@ GEMMA_ARGS = gemma_args
 
 debug.call "Options: ",options,"\n" if !options[:quiet]
 
-invoke_gemma = lambda do |extra_args, cache_hit = false|
-  cmd="#{GEMMA_COMMAND} #{GEMMA_ARGS.join(' ')} #{extra_args.join(' ')}"
+invoke_gemma = lambda do |extra_args, cache_hit = false, chr = "full", permutation = 1|
+  cmd = "#{GEMMA_COMMAND} #{GEMMA_ARGS.join(' ')} #{extra_args.join(' ')}"
   record[:gemma_command] = cmd
   return if cache_hit
-  debug.call cmd
+  if options[:slurm]
+    info.call cmd
+    hashi = HASH
+    prefix = options[:cache_dir]+'/'+hashi
+    scriptfn  = prefix+".#{chr}.#{permutation}-pbs.sh"
+    script = "#!/bin/bash
+#SBATCH --job-name=gemma-#{scriptfn}
+#SBATCH --ntasks=1
+#SBATCH --time=20:00
+srun #{cmd}
+"
+    debug.call(script)
+    File.open(scriptfn,"w") { |f|
+      f.write(script)
+    }
+    cmd = "sbatch "+options[:slurm_opts] + scriptfn
+  end
   errno =
     if options[:json]
       # capture output
       err = 0
-      IO.popen(cmd) do |io|
-        while s = io.gets
-          $stderr.print s
-        end
-        io.close
-        err = $?.to_i
+      if options[:dry_run]
+        info.call("Would have invoked: ",cmd)
+      elsif options[:parallel]
+        info.call("Add parallel job: ",cmd)
+        parallel_cmds << cmd
+      else
+        err = execute.call(cmd)
       end
       err
     else
-      debug.call("Invoking ",cmd) if options[:debug]
-      system(cmd)
-      $?.exitstatus
+      if options[:dry_run]
+        info.call("Would have invoked ",cmd)
+        0
+      else
+        debug.call("Invoking ",cmd) if options[:debug]
+        system(cmd)
+        $?.exitstatus
+      end
     end
   if errno != 0
     debug.call "Gemma exit ",errno
@@ -281,10 +372,12 @@ invoke_gemma = lambda do |extra_args, cache_hit = false|
 end
 
 # returns datafn, logfn, cache_hit
-cache = lambda do | chr, ext |
+cache = lambda do | chr, ext, h=HASH, permutation=0 |
   inject = (chr==nil ? "" : ".#{chr}" )+ext
-  hashi = (chr==nil ? HASH : HASH+inject)
-  prefix = options[:cache_dir]+'/'+hashi
+  hashi = (chr==nil ? h : h+inject)
+  prefix = options[:cache_dir]+'/'+hashi+(permutation!=0 ? "."+permutation.to_s : "")
+  # for chr 3 and permutation 1 forms something like
+  #   /tmp/1b700-a996f.3.cXX.txt.1.log.txt
   logfn  = prefix+".log.txt"
   datafn = prefix+ext
   record[:files] ||= []
@@ -320,20 +413,21 @@ kinship = lambda do | chr = nil |
 end
 
 # ---- Run GWA
-gwas = lambda do | chr, kfn, pfn |
+gwas = lambda do | chr, kfn, pfn, permutation=0 |
   record[:type] = "GWA"
   error.call "Do not use the GEMMA -k switch with gemma-wrapper - it is automatic!" if GEMMA_ARGS.include? '-k' # K is automatic
-  hashi, cache_hit = cache.call chr,".assoc.txt"
+  # Update hash for each permutation
+  hash = compute_hash.call(pfn)
+  hashi, cache_hit = cache.call(chr,".assoc.txt",hash,permutation)
   if not cache_hit
     args = [ '-k', kfn, '-o', hashi ]
     args << [ '-loco', chr ] if chr != nil
     args << [ '-p', pfn ] if pfn
-    invoke_gemma.call args
+    invoke_gemma.call args,false,chr,permutation
   end
 end
 
 LOCO = options[:loco]
-# if GEMMA_ARGS.include? '-gk'
 if DO_COMPUTE_KINSHIP
   # compute K
   info.call LOCO
@@ -364,10 +458,10 @@ else
     end
     score_list = []
     debug.call(options[:permutate],"x permutations")
-    (1..options[:permutate]).each do |i|
-      $stderr.print "Iteration ",i,"\n"
+    (1..options[:permutate]).each do |permutation|
+      $stderr.print "Iteration ",permutation,"\n"
       # Create a shuffled phenotype file
-      file = File.open("phenotypes-#{i}","w")
+      file = File.open("phenotypes-#{permutation}","w")
       tmp_pfn = file.path
       p tmp_pfn
       ps.shuffle.each do | l |
@@ -375,20 +469,23 @@ else
       end
       file.close
       k_files.each do | chr, kfn | # call a GWA for each chromosome
-        gwas.call(chr,kfn,tmp_pfn)
+        gwas.call(chr,kfn,tmp_pfn,permutation)
       end
-      # p [:HEY,record[:files].last]
-      assocfn = record[:files].last[2]
-      debug.call("Reading ",assocfn)
       score_min = 1000.0
-      File.foreach(assocfn).with_index do |assoc, assoc_line_num|
-        if assoc_line_num > 0
-          value = assoc.strip.split(/\t/).last.to_f
-          score_min = value if value < score_min
+      if false and not options[:slurm]
+        # p [:HEY,record[:files].last]
+        assocfn = record[:files].last[2]
+        debug.call("Reading ",assocfn)
+        File.foreach(assocfn).with_index do |assoc, assoc_line_num|
+          if assoc_line_num > 0
+            value = assoc.strip.split(/\t/).last.to_f
+            score_min = value if value < score_min
+          end
         end
       end
       score_list << score_min
     end
+    exit 0 if options[:slurm]
     ls = score_list.sort
     p ls
     significant = ls[(ls.size - ls.size*0.95).floor]
@@ -399,5 +496,24 @@ else
   end
 end
 
+# ---- Invoke parallel
+if options[:parallel]
+  # parallel_cmds = ["echo 1","sleep 1 && echo 2", "false", "echo 3"]
+  cmd = parallel_cmds.join("\\n")
+
+  cmd = "echo -e \"#{cmd}\""
+  err = execute.call(cmd+"|parallel") # all jobs in parallel
+  if err != 0
+    [16,8,4,1].each do |jobs|
+      info.call("Failed to complete parallel run -- retrying with smaller RAM footprint!")
+      err = execute.call(cmd+"|parallel -j #{jobs}")
+      break if err == 0
+    end
+    if err != 0
+      info.call("Run failed!")
+      exit err
+    end
+  end
+  info.call("Run successful!")
+end
 json_out.call
-exit 0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bio-gemma-wrapper
 version: !ruby/object:Gem::Version
-  version: 0.98.1
+  version: 0.99.1
 platform: ruby
 authors:
 - Pjotr Prins
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-20 00:00:00.000000000 Z
+date: 2021-07-11 00:00:00.000000000 Z
 dependencies: []
 description: GEMMA wrapper adds LOCO and permutation support. Also caches K between
   runs with LOCO support
@@ -44,7 +44,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.7.6.2
 signing_key: 
 specification_version: 4
 summary: GEMMA with LOCO and permutations