bio-gemma-wrapper 0.99.2 → 0.99.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e27a8a3abb00b758095df5956b3854674faf5ff681a93bc028df273c40125c0d
-  data.tar.gz: e9675dbb0ea0f087dd21774635d38f3cda11b46a88b36c77dd308086fd0ec5f2
+  metadata.gz: 0bd37b153e121de9c1758af736cd6904744da2de3540f2a7c547cc423382d8d1
+  data.tar.gz: 84298a943e7cfe6126653895d9714babc83a7be2bf903c7b61ff9072f1d4e4a8
 SHA512:
-  metadata.gz: 81cf5440fa531d5a831efa787800c8bea230d47cddc666a31fff066551ff347708a41ddf1368c0d3946c7ba9faef8e5882e398ad340850253c53961cce96f662
-  data.tar.gz: 582ae78c48a1eb8eeca01172eaeaba9d5ca23e69601967e334f8c218e3a4dd74b297861b01ce49b1357798b49a96c12e737100dcacec7fc34b70da1fc9c75f0d
+  metadata.gz: f32d48ec2f194a513e0cf8f15463b05662e1b269fdd22a110e4cdfb9f6bc541238bac7e766cbc31a8b782379891636e4f79fb3e3667d567ab3c610298d4f11c2
+  data.tar.gz: abc9b3faf8ef2f63d566caa14312bde2b2f438c722a83e90f7da775c55e2415171efbfab385273c82e45398c31e5306fb99437c38e287570652a3b04a5432883

data/README.md

@@ -8,11 +8,12 @@ Nat. Genet., 2016)](cfw.gif)
 ## Introduction
 
 Gemma-wrapper allows running GEMMA with LOCO, GEMMA with caching,
-GEMMA in parallel (now the default), and GEMMA on PBS. Gemma-wrapper
-is used to run GEMMA as part of the https://genenetwork.org/
-environment.
+GEMMA in parallel (now the default with LOCO), and GEMMA on
+PBS. Gemma-wrapper is used to run GEMMA as part of the
+https://genenetwork.org/ environment.
 
-Note that gemma-wrapper is projected to be integrated into gemma2/lib.
+Note that a version of gemma-wrapper is projected to be integrated
+into gemma itself.
 
 GEMMA is a software toolkit for fast application of linear mixed
 models (LMMs) and related models to genome-wide association studies
@@ -29,6 +30,14 @@ does a pass-through of all standard GEMMA invocation switches. On
 return gemma-wrapper can return a JSON object (--json) which is
 useful for web-services.
 
+## Performance
+
+LOCO runs in parallel by default which is at least a 5x performance
+improvement on a machine with enough cores. GEMMA without LOCO,
+however, does not run in parallel by default.  Performance
+improvements with the parallel implementation for LOCO and non-LOCO
+can be viewed [here](./test/performance/releases.gmi).
+
 ## Installation
 
 Prerequisites are
@@ -53,15 +62,19 @@ and it will render something like
 Usage: gemma-wrapper [options] -- [gemma-options]
         --permutate n                Permutate # times by shuffling phenotypes
         --permute-phenotypes filen   Phenotypes to be shuffled in permutations
-        --loco [x,y,1,2,3...]        Run full LOCO
+        --loco                       Run full leave-one-chromosome-out (LOCO)
+        --chromosomes [1,2,3]        Run specific chromosomes
         --input filen                JSON input variables (used for LOCO)
         --cache-dir path             Use a cache directory
         --json                       Create output file in JSON format
-        --force                      Force computation
-        --slurm [options]            Submit to slurm PBS
+        --force                      Force computation (override cache)
+        --parallel                   Run jobs in parallel
+        --no-parallel                Do not run jobs in parallel
+        --slurm[=opts]               Use slurm PBS for submitting jobs
         --q, --quiet                 Run quietly
     -v, --verbose                    Run verbosely
-        --debug                      Show debug messages and keep intermediate output
+    -d, --debug                      Show debug messages and keep intermediate output
+        --dry-run                    Show commands, but don't execute
         --                           Anything after gets passed to GEMMA
 
     -h, --help                       display this help and exit
@@ -99,6 +112,7 @@ the data files are found):
     gemma-wrapper -- \
         -g test/data/input/BXD_geno.txt.gz \
         -p test/data/input/BXD_pheno.txt \
+        -a test/data/input/BXD_snps.txt \
         -gk \
         -debug
 
@@ -116,6 +130,7 @@ You can also get JSON output on STDOUT by providing the --json switch
     gemma-wrapper --json -- \
         -g test/data/input/BXD_geno.txt.gz \
         -p test/data/input/BXD_pheno.txt \
+        -a test/data/input/BXD_snps.txt \
         -gk \
         -debug > K.json
 
@@ -133,6 +148,7 @@ default. If you want something else provide a --cache-dir, e.g.
     gemma-wrapper --cache-dir ~/.gemma-cache -- \
         -g test/data/input/BXD_geno.txt.gz \
         -p test/data/input/BXD_pheno.txt \
+        -a test/data/input/BXD_snps.txt \
         -gk \
         -debug
 
@@ -143,7 +159,7 @@ will store K in ~/.gemma-cache.
 Run the LMM using the K's captured earlier in K.json using the --input
 switch
 
-    gemma-wrapper --json --loco --input K.json -- \
+    gemma-wrapper --json --input K.json -- \
         -g test/data/input/BXD_geno.txt.gz \
         -p test/data/input/BXD_pheno.txt \
         -c test/data/input/BXD_covariates2.txt \
@@ -163,7 +179,7 @@ https://github.com/genetics-statistics/GEMMA/issues/46). To loop all
 chromosomes first create all K's with
 
     gemma-wrapper --json \
-        --loco 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,X -- \
+        --loco -- \
         -g test/data/input/BXD_geno.txt.gz \
         -p test/data/input/BXD_pheno.txt \
         -a test/data/input/BXD_snps.txt \

data/VERSION

@@ -1 +1 @@
-0.99.2
+0.99.3

data/bin/gemma-wrapper

@@ -14,12 +14,12 @@ GEMMA wrapper example:
       gemma-wrapper -- \\
         -g test/data/input/BXD_geno.txt.gz \\
         -p test/data/input/BXD_pheno.txt \\
+        -a test/data/input/BXD_snps.txt \
         -gk
 
     LOCO K computation with caching and JSON output
 
-      gemma-wrapper --json \\
-        --loco 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,X -- \\
+      gemma-wrapper --json --loco -- \\
         -g test/data/input/BXD_geno.txt.gz \\
         -p test/data/input/BXD_pheno.txt \\
         -a test/data/input/BXD_snps.txt \\
@@ -72,11 +72,12 @@ require 'tempfile'
 require 'tmpdir'
 
 split_at = ARGV.index('--')
+
 if split_at
   gemma_args = ARGV[split_at+1..-1]
 end
 
-options = { show_help: false, source: 'https://github.com/genetics-statistics/gemma-wrapper', version: version+' (Pjotr Prins)', date: Time.now.to_s, gemma_command: gemma_command, cache_dir: Dir.tmpdir(), quiet: false, parallel: true }
+options = { show_help: false, source: 'https://github.com/genetics-statistics/gemma-wrapper', version: version+' (Pjotr Prins)', date: Time.now.to_s, gemma_command: gemma_command, cache_dir: Dir.tmpdir(), quiet: false, permute_phenotypes: false, parallel: nil }
 
 opts = OptionParser.new do |o|
   o.banner = "\nUsage: #{File.basename($0)} [options] -- [gemma-options]"
@@ -91,8 +92,12 @@ opts = OptionParser.new do |o|
     raise "Phenotype input file #{phenotypes} does not exist" if !File.exist?(phenotypes)
   end
 
-  o.on('--loco [x,y,1,2,3...]', Array, 'Run full leave-one-chromosome-out (LOCO)') do |lst|
-    options[:loco] = lst
+  o.on('--loco', 'Run full leave-one-chromosome-out (LOCO)') do |b|
+    options[:loco] = b
+  end
+
+  o.on('--chromosomes [1,2,3]',Array,'Run specific chromosomes') do |lst|
+    options[:chromosomes] = lst
   end
 
   o.on('--input filen',String, 'JSON input variables (used for LOCO)') do |filen|
@@ -112,6 +117,10 @@ opts = OptionParser.new do |o|
     options[:force] = true
   end
 
+  o.on("--parallel", "Run jobs in parallel") do |b|
+    options[:parallel] = true
+  end
+
   o.on("--no-parallel", "Do not run jobs in parallel") do |b|
     options[:parallel] = false
   end
@@ -190,6 +199,15 @@ info = lambda do |*msg|
   OUTPUT.print *msg,"\n" if !options[:quiet]
 end
 
+# Fetch chromosomes
+def get_chromosomes annofn
+  h = {}
+  File.open(annofn,"r").each_line do | line |
+    chr = line.split(/\s+/)[2]
+    h[chr] = true
+  end
+  h.map { |k,v| k }
+end
 # ---- Start banner
 
 GEMMA_K_VERSION=version
@@ -230,13 +248,17 @@ if RUBY_VERSION =~ /^1/
   warning "runs on Ruby 2.x only\n"
 end
 
+# ---- LOCO defaults to parallel
+if options[:parallel] == nil
+  options[:parallel] = true if options[:loco]
+end
+
 debug.call(options) # some debug output
 debug.call(record)
 
 DO_COMPUTE_KINSHIP = gemma_args.include?("-gk")
 DO_COMPUTE_GWA = !DO_COMPUTE_KINSHIP
 
-# ---- Set up parallel
 if options[:parallel]
   begin
     skip_cite = `echo "will cite" |parallel --citation`
@@ -248,6 +270,11 @@ if options[:parallel]
   parallel_cmds = []
 end
 
+# ---- Fetch chromosomes from SNP annotation file
+anno_idx = gemma_args.index '-a'
+raise "Expected GEMMA -a genotype file switch" if anno_idx == nil
+CHROMOSOMES = get_chromosomes(gemma_args[anno_idx+1])
+
 # ---- Compute HASH on inputs
 hashme = []
 geno_idx = gemma_args.index '-g'
@@ -434,11 +461,17 @@ gwas = lambda do | chr, kfn, pfn, permutation=0 |
 end
 
 LOCO = options[:loco]
+if LOCO
+  if options[:chromosomes]
+    CHROMOSOMES = options[:chromosomes]
+  end
+end
+
 if DO_COMPUTE_KINSHIP
   # compute K
-  info.call LOCO
-  if LOCO != nil
-    LOCO.each do |chr|
+  info.call CHROMOSOMES
+  if LOCO
+    CHROMOSOMES.each do |chr|
       info.call "LOCO for ",chr
       kinship.call(chr)
     end
@@ -447,13 +480,24 @@ if DO_COMPUTE_KINSHIP
   end
 else
   # DO_COMPUTE_GWA
-  json_in = JSON.parse(File.read(options[:input]))
+  begin
+    json_in = JSON.parse(File.read(options[:input]))
+  rescue TypeError
+    raise "Missing JSON input file?"
+  end
   raise "JSON problem, file #{options[:input]} is not -gk derived" if json_in["type"] != "K"
 
   pfn = options[:permute_phenotypes] # can be nil
-  k_files = json_in["files"].map { |rec| [rec[0],rec[2]] }
-  k_files.each do | chr, kfn | # call a GWA for each chromosome
-    gwas.call(chr,kfn,pfn)
+  if LOCO
+    k_files = json_in["files"].map { |rec| [rec[0],rec[2]] }
+    k_files.each do | chr, kfn | # call a GWA for each chromosome
+      gwas.call(chr,kfn,pfn)
+    end
+  else
+    kfn = json_in["files"][0][2]
+    CHROMOSOMES.each do | chr |
+      gwas.call(chr,kfn,pfn)
+    end
   end
   # Permute
   if options[:permutate]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bio-gemma-wrapper
 version: !ruby/object:Gem::Version
-  version: 0.99.2
+  version: 0.99.3
 platform: ruby
 authors:
 - Pjotr Prins
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-08 00:00:00.000000000 Z
+date: 2021-08-22 00:00:00.000000000 Z
 dependencies: []
 description: GEMMA wrapper adds LOCO and permutation support. Also runs in parallel
   and caches K between runs with LOCO support