bio-gemma-wrapper 0.99.6 → 0.99.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 69d74ac5f1a705132d7ddc86bd1182c11fc37cf32062dbb28909f16684a827cb
-  data.tar.gz: 6f912b3c03474c1334a105d6c9471c02d39c205e94b0d07e5281718beec65ee7
+  metadata.gz: 8516f4e6692ceed95f95d5d55f530a2bfbf2f6f8fd6daf8e5918752c6be6cae7
+  data.tar.gz: b6942b33acc903f423a9c6bcf53920df326a18f58eb307fc713d220c1d3d88ed
 SHA512:
-  metadata.gz: 35fbdf4ccfc482f6898e35b1e46e840db87f34efb35a096206ffef4a131ac8b21a4e6b7893be6ee053a59837eadba6547258d2360d796794c71a60670458943f
-  data.tar.gz: 43f7a7438f475583930e6cf1f50d34206fba7674b3df836791cf7064bd7ca55dddc2cc101f7c5a89d8042645b6dd06245b84d3b38501bc1eab7d18774703a01a
+  metadata.gz: 15fdc14eafe7a33aa330a1e156f23cfd6a1d7bc43e48aacb126aa8157b8c0e1a8e649d02bc06eda4bf5704e1c0e04520ab07be492096ab92eb198372140442e5
+  data.tar.gz: c2a70aedae7743e63276285129665860d58bbdf7b701289b05cdbfc82d35207e44240b92b533bed55c6ced28f893cd08af228281dcb7de0c8c54d272a0e1474e

data/README.md

@@ -247,4 +247,4 @@ ruby bin/gemma-wrapper --help
 
 ## Copyright
 
-Copyright (c) 2017-2021 Pjotr Prins. See [LICENSE.txt](LICENSE.txt) for further details.
+Copyright (c) 2017-2023 Pjotr Prins. See [LICENSE.txt](LICENSE.txt) for further details.

data/Rakefile

@@ -0,0 +1,87 @@
+# encoding: utf-8
+#
+# Run tests with, for example
+#
+#   env GEMMA_COMMAND=../gemma/bin/gemma rake test
+
+require 'rubygems'
+require 'rake'
+
+task default: %w[test]
+
+task :test do
+  ruby "bin/gemma-wrapper --json --force -- \
+        -g test/data/input/BXD_geno.txt.gz \
+        -p test/data/input/BXD_pheno.txt \
+        -a test/data/input/BXD_snps.txt \
+        -gk \
+        -debug > K0.json"
+  K0 = File.read("K0.json")
+  fail "Wrong Hash in #{K0}" if K0 !~ /1b700de28f242d561fc6769a07d88403764a996f/
+  fail "Expected error is 0 in #{K0}" if K0 !~ /errno\":0/
+  fail "Test failed" if $? != 0
+  ruby "bin/gemma-wrapper --json --input K0.json -- \
+        -g test/data/input/BXD_geno.txt.gz \
+        -p test/data/input/BXD_pheno.txt \
+        -c test/data/input/BXD_covariates2.txt \
+        -a test/data/input/BXD_snps.txt \
+        -lmm 2 -maf 0.1 \
+        -debug > GWA0.json"
+  gwa0 = File.read("GWA0.json")
+  fail "Wrong Hash in #{gwa0}" if gwa0 !~ /9e411810ad341de6456ce0c6efd4f973356d0bad/
+  fail "Expected cache hit in #{gwa0}" if gwa0 !~ /cache_hit\":true/
+  fail "Test failed" if $? != 0
+  ruby "bin/gemma-wrapper --debug --json --force \
+        --loco --chromosomes 1,2,3,4 -- \
+        -g test/data/input/BXD_geno.txt.gz \
+        -p test/data/input/BXD_pheno.txt \
+        -a test/data/input/BXD_snps.txt \
+        -gk -debug > KLOCO1.json"
+  kloco1 = File.read("KLOCO1.json")
+  fail "Wrong Hash in #{kloco1}" if kloco1 !~ /1b700de28f242d561fc6769a07d88403764a996f/
+  fail "Expected error is 0 in #{kloco1}" if kloco1 !~ /errno\":0/
+  fail "Test failed" if $? != 0
+  # run again for cache hits
+  ruby "bin/gemma-wrapper  --json  \
+        --loco --chromosomes 1,2,3,4 -- \
+        -g test/data/input/BXD_geno.txt.gz \
+        -p test/data/input/BXD_pheno.txt \
+        -a test/data/input/BXD_snps.txt \
+        -gk -debug > KLOCO2.json"
+  kloco2 = File.read("KLOCO2.json")
+  fail "Wrong Hash in #{kloco2}" if kloco2 !~ /1b700de28f242d561fc6769a07d88403764a996f/
+  fail "Expected cache hit in #{kloco2}" if kloco2 !~ /cache_hit\":true/
+  fail "Test failed" if $? != 0
+  ruby "bin/gemma-wrapper --json --force --loco --input KLOCO1.json -- \
+        -g test/data/input/BXD_geno.txt.gz \
+        -p test/data/input/BXD_pheno.txt \
+        -c test/data/input/BXD_covariates2.txt \
+        -a test/data/input/BXD_snps.txt \
+        -lmm 2 -maf 0.1 \
+        -debug > GWA1.json"
+  gwa1 = File.read("GWA1.json")
+  fail "Wrong Hash in #{gwa1}" if gwa1 !~ /9e411810ad341de6456ce0c6efd4f973356d0bad/
+  fail "Test failed" if $? != 0
+  # and run again
+  ruby "bin/gemma-wrapper --json --loco --input KLOCO2.json -- \
+        -g test/data/input/BXD_geno.txt.gz \
+        -p test/data/input/BXD_pheno.txt \
+        -c test/data/input/BXD_covariates2.txt \
+        -a test/data/input/BXD_snps.txt \
+        -lmm 2 -maf 0.1 \
+        -debug > GWA2.json"
+  fail "Test failed" if $? != 0
+  gwa2 = File.read("GWA2.json")
+  fail "Wrong Hash in #{gwa2}" if gwa2 !~ /9e411810ad341de6456ce0c6efd4f973356d0bad/
+  fail "Expected cache hit in #{gwa2}" if gwa2 !~ /cache_hit\":true/
+end
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "bio-gemma-wrapper #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -1 +1 @@
-0.99.6
+0.99.7

data/bin/gemma-wrapper

@@ -4,35 +4,35 @@
 # Author:: Pjotr Prins
 # License:: GPL3
 #
-# Copyright (C) 2017-2022 Pjotr Prins <pjotr.prins@thebird.nl>
+# Copyright (C) 2017-2024 Pjotr Prins <pjotr.prins@thebird.nl>
 
 USAGE = "
 GEMMA wrapper example:
 
     Simple caching of K computation with
 
-      gemma-wrapper -- \\
-        -g test/data/input/BXD_geno.txt.gz \\
-        -p test/data/input/BXD_pheno.txt \\
+      gemma-wrapper -- \
+        -g test/data/input/BXD_geno.txt.gz \
+        -p test/data/input/BXD_pheno.txt \
         -a test/data/input/BXD_snps.txt \
-        -gk
+        -gk > K.json
 
     LOCO K computation with caching and JSON output
 
-      gemma-wrapper --json --loco -- \\
-        -g test/data/input/BXD_geno.txt.gz \\
-        -p test/data/input/BXD_pheno.txt \\
-        -a test/data/input/BXD_snps.txt \\
+      gemma-wrapper --json --loco -- \
+        -g test/data/input/BXD_geno.txt.gz \
+        -p test/data/input/BXD_pheno.txt \
+        -a test/data/input/BXD_snps.txt \
         -gk -debug > K.json
 
     LMM's using the K's captured in K.json using the --input switch
 
-      gemma-wrapper --json --loco --input K.json -- \\
-        -g test/data/input/BXD_geno.txt.gz \\
-        -p test/data/input/BXD_pheno.txt \\
-        -c test/data/input/BXD_covariates2.txt \\
-        -a test/data/input/BXD_snps.txt \\
-        -lmm 2 -maf 0.1 \\
+      gemma-wrapper --json --loco --input K.json -- \
+        -g test/data/input/BXD_geno.txt.gz \
+        -p test/data/input/BXD_pheno.txt \
+        -c test/data/input/BXD_covariates2.txt \
+        -a test/data/input/BXD_snps.txt \
+        -lmm 9 -maf 0.1 \
         -debug > GWA.json
 
 Gemma gets used from the path. You can override by setting
@@ -45,6 +45,7 @@ GEMMA_V_MINOR = 4
 
 basepath = File.dirname(File.dirname(__FILE__))
 $: << File.join(basepath,'lib')
+BIN = File.join(basepath,'bin')
 
 VERSION_FILENAME=File.join(basepath,'VERSION')
 version = File.new(VERSION_FILENAME).read.chomp
@@ -69,7 +70,10 @@ hashme = nil
 require 'digest/sha1'
 require 'fileutils'
 require 'optparse'
+require 'open3'
+require 'socket' # for hostname
 require 'tempfile'
+require 'time'
 require 'tmpdir'
 
 require 'lock'
@@ -80,7 +84,7 @@ if split_at
   gemma_args = ARGV[split_at+1..-1]
 end
 
-options = { show_help: false, source: 'https://github.com/genetics-statistics/gemma-wrapper', version: version+' (Pjotr Prins)', date: Time.now.to_s, gemma_command: gemma_command, cache_dir: Dir.tmpdir(), quiet: false, permute_phenotypes: false, parallel: nil }
+options = { show_help: false, source: 'https://github.com/genetics-statistics/gemma-wrapper', version: version+' (Pjotr Prins)', date: Time.now.to_s, gemma_command: gemma_command, cache_dir: Dir.tmpdir(), quiet: false, permute_phenotypes: false, lmdb: nil, parallel: nil }
 
 opts = OptionParser.new do |o|
   o.banner = "\nUsage: #{File.basename($0)} [options] -- [gemma-options]"
@@ -99,6 +103,22 @@ opts = OptionParser.new do |o|
     options[:loco] = b
   end
 
+  o.on('--population NAME', 'Add population identifier to metadata') do |n|
+    options[:population] = n
+  end
+
+  o.on('--name NAME', 'Add dataset identifier to metadata') do |n|
+    options[:name] = n
+  end
+
+  o.on('--id ID', 'Add identifier to metadata') do |n|
+    options[:id] = n
+  end
+
+  o.on('--trait TRAIT', 'Add trait identifier to metadata') do |n|
+    options[:trait] = n
+  end
+
   o.on('--chromosomes [1,2,3]',Array,'Run specific chromosomes') do |lst|
     options[:chromosomes] = lst
   end
@@ -120,6 +140,10 @@ opts = OptionParser.new do |o|
     options[:force] = true
   end
 
+  o.on("--keep", "Keep intermediate files in output") do |q|
+    options[:keep] = true
+  end
+
   o.on("--parallel", "Run jobs in parallel") do |b|
     options[:parallel] = true
   end
@@ -128,6 +152,10 @@ opts = OptionParser.new do |o|
     options[:parallel] = false
   end
 
+  o.on("--lmdb", "Generate lmdb output") do |b|
+    options[:lmdb] = true
+  end
+
   o.on("--slurm[=opts]",String,"Use slurm PBS for submitting jobs") do |slurm|
     options[:slurm_opts] = ""
     options[:slurm] = true
@@ -169,11 +197,18 @@ opts.parse!(ARGV)
 OUTPUT = (options[:json] ? $stderr : $stdout )
 
 record = { warnings: [], errno: 0, debug: [] }
+record[:name] = options[:name] if options[:name]
+record[:id] = options[:id] if options[:id]
+record[:trait] = options[:trait] if options[:trait]
+d = DateTime.now
+record[:time] = d.strftime("%Y/%m/%d %H:%M")
+record[:user] = ENV["USER"]
+record[:hostname] = Socket.gethostname
 
 require 'json'
 
 json_out = lambda do
-  print record.to_json if options[:json]
+  record.to_json if options[:json]
 end
 
 # ---- Some error handlers
@@ -215,17 +250,18 @@ end
 # ---- Start banner
 
 GEMMA_K_VERSION=version
-GEMMA_K_BANNER = "gemma-wrapper #{version} (Ruby #{RUBY_VERSION}) by Pjotr Prins 2017-2022\n"
+GEMMA_K_BANNER = "gemma-wrapper #{version} (Ruby #{RUBY_VERSION}) by Pjotr Prins 2017-2024\n"
 info.call GEMMA_K_BANNER
 
 # Check gemma version
 begin
   gemma_command2 = options[:gemma_command]
-  info.call "NOTE: gemma-wrapper is soon to be replaced"
+  # info.call "NOTE: gemma-wrapper is soon to be replaced"
 
+  debug.call("Invoke #{gemma_command2}")
   GEMMA_INFO = `#{gemma_command2}`
 rescue Errno::ENOENT
-  gemma_command2 = "gemma"
+  gemma_command2 = "gemma" if not gemma_command2
   error.call "<#{gemma_command2}> command not found"
 end
 
@@ -249,7 +285,7 @@ if options[:show_help] or gemma_args == nil
 end
 
 if RUBY_VERSION =~ /^1/
-  warning "runs on Ruby 2.x only\n"
+  warning "does not run on Ruby 1.x\n"
 end
 
 # ---- LOCO defaults to parallel
@@ -272,6 +308,9 @@ if options[:parallel]
     error.call "<parallel> command not found"
   end
   parallel_cmds = []
+  if not options[:json]
+    error.call "<parallel> needs --json switch"
+  end
 end
 
 # ---- Fetch chromosomes from SNP annotation file
@@ -288,23 +327,69 @@ if DO_COMPUTE_GWA and options[:permute_phenotypes]
   raise "Did not expect GEMMA -p phenotype whith permutations (only use --permutate-phenotypes)" if pheno_idx
 end
 
-execute = lambda { |cmd|
-  info.call("Executing: #{cmd}")
-  err = 0
-  if not options[:debug]
-    # send output to stderr line by line
-    IO.popen("#{cmd}") do |io|
-      while s = io.gets
-        $stderr.print s
+matches = {
+  chr: [:string, /-loco (\S+) /],
+  user_time: [:float, /User time \(seconds\): ([\d\.]+)/],
+  system_time: [:float, /System time \(seconds\): ([\d\.]+)/],
+  perc_cpu: [:int, /Percent of CPU this job got: (\d+)%/],
+  wall_clock: [:string, /Elapsed \(wall clock\) time \(h:mm:ss or m:ss\): (\S+)/],
+  ram_usage_gb: [:gb, /Maximum resident set size \(kbytes\): (\d+)/],
+  command: [:string, /Command being timed: (.+)/]
+}
+
+parse_stats = lambda { |buf|
+  stats = {}
+  buf.split("\\n").each do |s|
+    if s =~ /^\t/
+      matches.each do |k,v|
+        type,m = v
+        if s =~ m
+          # $stderr.print $1,s
+          stats[k] =
+            case type
+            when :float
+              $1.to_f
+            when :int
+              $1.to_i
+            when :gb
+              (($1.to_f)/1048576.0).round(3)
+            else
+              $1
+            end
+        end
       end
-      io.close
-      err = $?.to_i
     end
-  else
-    $stderr.print `#{cmd}`
-    err = $?.to_i
   end
-  err
+  stats
+}
+
+run_stat = {}
+
+execute = lambda { |cmd|
+  info.call("Executing: #{cmd}")
+  err = 0
+  stdout_buf = ""
+  stderr_buf = ""
+  stats = {}
+  Open3.popen3("time -v #{cmd}") do |stdin,stdout,stderr,wait_thr|
+    stderr_buf = stderr.read
+    stdout_buf = stdout.read
+    stats = parse_stats.call(stderr_buf)
+    stdin.close
+    stdout.close
+    stderr.close
+    err = wait_thr.value
+  end
+  $stderr.print(stderr_buf) if options[:debug]
+  if err and err != 0
+    $stderr.print(stdout_buf)
+    $stderr.print(stderr_buf) if not options[:debug]
+    $stderr.print "FATAL ERROR: gemma-wrapper bailed out with #{err}\n"
+    # sleep 10_000
+    $stderr.print Kernel.caller().join("\n")
+    exit 1
+  end
+  return err,stats
 }
 
 compute_hash = lambda do | phenofn = nil |
@@ -319,6 +404,7 @@ compute_hash = lambda do | phenofn = nil |
        end
   debug.call("Hashing on ",hm)
   hm.each do | item |
+    # if entry is a file use the hash of its content, otherwise just the entry itself
     if File.file?(item)
       hashes << Digest::SHA1.hexdigest(File.read(item))
       debug.call [item,hashes.last]
@@ -343,6 +429,7 @@ hashme =
   end
 
 HASH = compute_hash.call()
+options[:compute_hash_on] = hashme
 options[:hash] = HASH
 
 at_exit do
@@ -351,7 +438,7 @@ end
 
 Lock.create(HASH) # this will wait for a lock to expire
 
-joblog = options[:cache_dir]+"/"+HASH+"-parallel.log"
+JOBLOG  = HASH+"-parallel.log"
 
 # Create cache dir
 FileUtils::mkdir_p options[:cache_dir]
@@ -365,7 +452,7 @@ GEMMA_ARGS = gemma_args
 debug.call "Options: ",options,"\n" if !options[:quiet]
 
 invoke_gemma = lambda do |extra_args, cache_hit = false, chr = "full", permutation = 1|
-  cmd = "#{gemma_command2} #{GEMMA_ARGS.join(' ')} #{extra_args.join(' ')}"
+  cmd = "time -v #{gemma_command2} #{extra_args.join(' ')} #{GEMMA_ARGS.join(' ')}"
   record[:gemma_command] = cmd
   return if cache_hit
   if options[:slurm]
@@ -395,7 +482,7 @@ srun #{cmd}
         info.call("Add parallel job: ",cmd)
         parallel_cmds << cmd
       else
-        err = execute.call(cmd)
+        err,stats = execute.call(cmd)
       end
       err
     else
@@ -416,6 +503,8 @@ srun #{cmd}
   end
 end
 
+create_archive = false
+
 # Takes the hash value and checks whether the (output) file exists
 # returns datafn, logfn, cache_hit
 cache = lambda do | chr, ext, h=HASH, permutation=0 |
@@ -427,10 +516,15 @@ cache = lambda do | chr, ext, h=HASH, permutation=0 |
   logfn  = prefix+".log.txt"
   datafn = prefix+ext
   record[:files] ||= []
-  record[:files].push [chr,logfn,datafn]
+  log_basefn = File.basename(logfn)
+  data_basefn = File.basename(datafn)
+  log_tmpfn = tmpdir+"/"+log_basefn
+  data_tmpfn = tmpdir+"/"+data_basefn
+  record[:files].push [chr,log_basefn,data_basefn]
   if !options[:force]
-    if File.exist? logfn and File.exist? datafn
-      if File.read(logfn).include? "total computation time"
+    info.call "Checking for #{data_tmpfn}"
+    if File.exist? log_tmpfn and File.exist? data_tmpfn
+      if File.read(log_tmpfn).include? "total computation time"
         record[:cache_hit] = true
         info.call "#{logfn} CACHE HIT!\n"
         return hashi, true
@@ -448,8 +542,10 @@ kinship = lambda do | chr = nil |
         when 2 then '.sXX.txt'
         else error.call "Unknown kinship type"
         end
+  # ---- check cache:
   hashi, cache_hit = cache.call chr,ext
   if not cache_hit
+    create_archive = true
     if chr != nil
       invoke_gemma.call [ '-loco', chr, '-o', hashi ], cache_hit
     else
@@ -466,8 +562,10 @@ gwas = lambda do | chr, kfn, pfn, permutation=0 |
   hash = compute_hash.call(pfn)
   hashi, cache_hit = cache.call(chr,".assoc.txt",hash,permutation)
   if not cache_hit
-    args = [ '-k', kfn, '-o', hashi ]
+    create_archive = true
+    args = []
     args << [ '-loco', chr ] if chr != nil
+    args << [ '-k', kfn, '-o', hashi ]
     args << [ '-p', pfn ] if pfn
     invoke_gemma.call args,false,chr,permutation
   end
@@ -480,12 +578,20 @@ if LOCO
   end
 end
 
+json_in = nil
+
 if DO_COMPUTE_KINSHIP
   # compute K
+  ARCHIVE = options[:cache_dir]+"/"+HASH+"-gemma-cXX.tar.xz"
+
+  if File.exist? ARCHIVE and not options[:force]
+    info.call "Unpack archive #{ARCHIVE}!"
+    execute.call "tar xJf #{ARCHIVE} -C #{tmpdir}"
+  end
   info.call CHROMOSOMES
   if LOCO
     CHROMOSOMES.each do |chr|
-      info.call "LOCO for ",chr
+      info.call "Compute kinship LOCO for chr ",chr
       kinship.call(chr)
     end
   else
@@ -493,6 +599,11 @@ if DO_COMPUTE_KINSHIP
   end
 else
   # DO_COMPUTE_GWA
+  ARCHIVE = options[:cache_dir]+"/"+HASH+"-gemma-GWA.tar.xz"
+  if File.exist? ARCHIVE and not options[:force]
+    info.call "Unpack archive #{ARCHIVE}!"
+    execute.call "env XZ_OPT='-T0' tar xJf #{ARCHIVE} -C #{tmpdir}"
+  end
   begin
     json_in = JSON.parse(File.read(options[:input]))
   rescue TypeError
@@ -504,12 +615,20 @@ else
   if LOCO
     k_files = json_in["files"].map { |rec| [rec[0],rec[2]] }
     k_files.each do | chr, kfn | # call a GWA for each chromosome
-      gwas.call(chr,kfn,pfn)
+
+      kfn2 = options[:cache_dir]+"/"+kfn
+      if not File.exist?(kfn2) and json_in["archive"]
+        # we aim to unpack the archive once on reuse
+        archive_grm = options[:cache_dir]+"/"+json_in["archive"]
+        execute.call "env XZ_OPT='-T0' tar xJf #{archive_grm} -C #{options[:cache_dir]}"
+      end
+
+      gwas.call(chr,kfn2,pfn)
     end
   else
     kfn = json_in["files"][0][2]
     CHROMOSOMES.each do | chr |
-      gwas.call(chr,kfn,pfn)
+      gwas.call(chr,tmpdir+"/"+kfn,pfn)
     end
   end
   # Permute
@@ -562,6 +681,7 @@ end
 # ---- Invoke parallel
 if options[:parallel]
   # parallel_cmds = ["echo 1","sleep 1 && echo 2", "false", "echo 3"]
+  joblog = tmpdir+"/"+JOBLOG
 
   Tempfile.open("commands.txt") do |f|
     cmdfn = f.path
@@ -571,38 +691,95 @@ if options[:parallel]
       end
     end
     cmd = "cat \"#{cmdfn}\""
-    err = execute.call(cmd+"|parallel --joblog #{joblog}") # first try optimistically to run all jobs in parallel
+    debug.call("tmpdir=#{tmpdir}")
+    err,stats = execute.call(cmd+"|parallel --results #{tmpdir} --joblog #{joblog}") # first try optimistically to run all jobs in parallel
     if err != 0
-      [16,8,4,1].each do |jobs|
+      [4,1].each do |jobs|
         info.call("Failed to complete parallel run -- retrying with smaller RAM footprint!")
-        err = execute.call(cmd+"|parallel -j #{jobs} --resume --joblog #{joblog}")
+        err,stats = execute.call(cmd+"|parallel  -j #{jobs} --results #{tmpdir} --resume --joblog #{joblog}")
         break if err == 0
       end
       if err != 0
         info.call("Parallel run failed!")
         debug.call("Job log is: ",File.read(joblog))
-        # Remove remaining files
-        FileUtils.mv joblog, joblog+".bak", verbose: false, force: true
-        FileUtils.rm_rf("#{tmpdir}/*", secure: true)
         exit err
       end
     end
   end
   info.call("Run successful!")
-  FileUtils.mv joblog, joblog+".bak", verbose: false, force: true
 end
-json_out.call
-
-# copy all output files to the cache_dir. If a file exists only emit a warning
-Dir.glob("*.txt", base: tmpdir) do | fn |
-  source = tmpdir + "/" + fn
-  dest = options[:cache_dir] + "/" + fn
-  if not File.exist?(dest) or options[:force]
-    info.call "Move #{source} to #{dest}"
-    FileUtils.mv source, dest, verbose: false
-  else
-    warning.call "File #{dest} already exists. Not overwriting"
+
+# Collect stats from parallel run
+
+run_stats = {}
+$stderr.print "STATS"
+Dir.glob(tmpdir+'/*/*' ).each do | dir |
+  File.open("#{dir}/stderr") { |f|
+    run_stat = parse_stats.call(f.read)
+    chr = run_stat[:chr]
+    run_stats[chr] = run_stat
+  }
+end
+# Now add up the stats
+user_time = 0.0
+system_time = 0.0
+wall_clock = "0"
+ram_usage_gb = 0.0
+run_stats.each do | k, v |
+  wall_clock=v[:wall_clock] if v[:wall_clock]>wall_clock
+  ram_usage_gb += v[:ram_usage_gb]
+  user_time += v[:user_time]
+  system_time += v[:system_time]
+end
+
+record[:user_time] = user_time
+record[:system_time] = system_time
+record[:wall_clock] = wall_clock
+record[:ram_usage_gb] = ram_usage_gb.round(2)
+record[:run_stats] = run_stats
+
+if create_archive
+  if DO_COMPUTE_GWA
+    LMDB = tmpdir+"/"+HASH+'.mdb'
+    # create lmdb database - we call out into a python script for that.
+    # first create a JSON record
+
+    meta = {
+      type: "gemma-wrapper",
+      version: version,
+      population: options[:population],
+      name: options[:name],
+      trait: options[:trait],
+      url: "https://genenetwork.org/show_trait?trait_id="+options[:trait]+"&dataset="+options[:name],
+      archive_GRM: json_in["archive"],
+      archive_GWA: File.basename(ARCHIVE),
+    }
+    if options[:id] and options[:id] =~ /,/ # this is GN specific
+      dataid,probesetid,probesetfreezeid = options[:id].split(",")
+      meta[:dataid] = dataid.to_i
+      meta[:probesetid] = probesetid.to_i
+      meta[:probesetfreezeid] = probesetfreezeid.to_i
+    end
+    record[:meta] = meta
+    metafn = tmpdir+"/"+HASH+"-meta.json"
+    File.write(metafn,record.to_json)
+    # sleep 10_000
+    if options[:lmdb]
+      File.unlink(LMDB) if File.exist?(LMDB) # removed any cached lmdb
+      execute.call "python3 #{BIN}/gemma2lmdb.py --db=#{LMDB} --meta=#{metafn} #{tmpdir}/*assoc.txt"
+    end
+    if not options[:keep]
+      execute.call "rm -f #{tmpdir}/1/*/* #{tmpdir}/*.txt #{tmpdir}/*.log #{tmpdir}/*.mdb-lock" # remove GEMMA output files
+      FileUtils.rm_rf("#{tmpdir}/1", secure: true)
+    end
   end
+  File.write(tmpdir+"/"+HASH+"-gemma-wrapper-output.json",json_out.call)
+  info.call "Creating archive #{ARCHIVE}..."
+  execute.call "env XZ_OPT='-T0' tar -cvJf #{ARCHIVE} -C #{tmpdir} ."
 end
 
 end # tmpdir
+
+record[:archive] = File.basename(ARCHIVE)
+
+print json_out.call

data/gemma-wrapper.gemspec

@@ -7,6 +7,7 @@ Gem::Specification.new do |s|
   s.email       = 'pjotr.public01@thebird.nl'
   s.files       = ["bin/gemma-wrapper",
                    "lib/lock.rb",
+                   "Rakefile",
                    "Gemfile",
                    "LICENSE.txt",
                    "README.md",

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bio-gemma-wrapper
 version: !ruby/object:Gem::Version
-  version: 0.99.6
+  version: 0.99.7
 platform: ruby
 authors:
 - Pjotr Prins
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-22 00:00:00.000000000 Z
+date: 2024-07-06 00:00:00.000000000 Z
 dependencies: []
 description: GEMMA wrapper adds LOCO and permutation support. Also runs in parallel
   and caches K between runs with LOCO support
@@ -21,6 +21,7 @@ files:
 - Gemfile
 - LICENSE.txt
 - README.md
+- Rakefile
 - VERSION
 - bin/gemma-wrapper
 - gemma-wrapper.gemspec
@@ -44,7 +45,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.4.19
 signing_key:
 specification_version: 4
 summary: GEMMA with LOCO and permutations