arvados-cli 0.1.20160608142315 → 0.1.20160913014253

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a5fc889cbdad1ec1ac89ed8674cf1499e90c20d4
-  data.tar.gz: aed390277dc320517d93fd8ab7b14da48b3b0a2c
+  metadata.gz: 2a3dae7abb2565c96c930378ff0804dee060904f
+  data.tar.gz: ae346cc798ae73cc9e97299c245af5133f53cf29
 SHA512:
-  metadata.gz: 92372e41f978b51561327fba6bf9107e1291aac4d8dd36de024e8714399a1a9cd904f4713a28bdf663a34049de94b82c4c67e718b27c277b686a4ed0e28f601e
-  data.tar.gz: 11bef7ffe6f6f12c0c10e3739727d7376457069b547ce1d80953d16c7079b00fa4518ff68cf668eb09719d6df404fc6e7a8142feecb390f9f4ed244bf6d5958c
+  metadata.gz: 9185b99de69e9576411307ce24383734a9f03875a9f9723b238c4cb6d5049341132215360956d9060f3b91732378158bce64149281d3ffb2d7b55cc9238cc259
+  data.tar.gz: 4fd3ab654fc817827d1cfb79c03519112c7397b5f9c66f33e6d49d0f9f0ad9b4bac7a03a94d52c36442f5f5d0898e7f09004e74860d5c894276fd8f1acea7e3b

data/bin/crunch-job

@@ -355,6 +355,7 @@ my @jobstep_done = ();
 my @jobstep_tomerge = ();
 my $jobstep_tomerge_level = 0;
 my $squeue_checked = 0;
+my $sinfo_checked = 0;
 my $latest_refresh = scalar time;
 
 
@@ -1401,6 +1402,37 @@ sub check_squeue
   }
 }
 
+sub check_sinfo
+{
+  # If a node fails in a multi-node "srun" call during job setup, the call
+  # may hang instead of exiting with a nonzero code.  This function checks
+  # "sinfo" for the health of the nodes that were allocated and ensures that
+  # they are all still in the "alloc" state.  If a node that is allocated to
+  # this job is not in "alloc" state, then set please_freeze.
+  #
+  # This is only called from srun_sync() for node configuration.  If a
+  # node fails doing actual work, there are other recovery mechanisms.
+
+  # Do not call `sinfo` more than once every 15 seconds.
+  return if $sinfo_checked > time - 15;
+  $sinfo_checked = time;
+
+  # The output format "%t" means output node states.
+  my @sinfo = `sinfo --nodes=\Q$ENV{SLURM_NODELIST}\E --noheader -o "%t"`;
+  if ($? != 0)
+  {
+    Log(undef, "warning: sinfo exit status $? ($!)");
+    return;
+  }
+  chop @sinfo;
+
+  foreach (@sinfo)
+  {
+    if ($_ != "alloc" && $_ != "alloc*") {
+      $main::please_freeze = 1;
+    }
+  }
+}
 
 sub release_allocation
 {
@@ -1906,7 +1938,6 @@ sub freezeunquote
   return $s;
 }
 
-
 sub srun_sync
 {
   my $srunargs = shift;
@@ -1961,6 +1992,7 @@ sub srun_sync
     if (!$busy || ($latest_refresh + 2 < scalar time)) {
       check_refresh_wanted();
       check_squeue();
+      check_sinfo();
     }
     if (!$busy) {
       select(undef, undef, undef, 0.1);

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: arvados-cli
 version: !ruby/object:Gem::Version
-  version: 0.1.20160608142315
+  version: 0.1.20160913014253
 platform: ruby
 authors:
 - Arvados Authors
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-08 00:00:00.000000000 Z
+date: 2016-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: arvados
@@ -164,7 +164,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
-description: Arvados command line tools, git commit 548e0c54db524cb7317850d4dfd8f3ee0b93cdb0
+description: Arvados command line tools, git commit b54478ea1b7c8aaeaf565d591f32769bcdc09b8f
 email: gem-dev@curoverse.com
 executables:
 - arv