arvados-cli 0.1.20150526143156 → 0.1.20150527201024

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: df49631b2b1c7ae33eb0d5d9e988e50a2fc0a311
-  data.tar.gz: fe0a40bf178a62a8518fec9c8ca699188d2a8d32
+  metadata.gz: bafef169f074ca61d533298a5e462905d9764846
+  data.tar.gz: bd2e304a1cef255c8362650d82ed490d88d7a56d
 SHA512:
-  metadata.gz: 59df31ffb182cdcc78b960b8ba439bed2f8b2d916d740ace0b92eb76a3de96169c193de30e3cedfb14e0df96cdda9a4eea32562735ff509f4e9a6b273028d1bb
-  data.tar.gz: b2afa69c6273326c575cc6e3c4a8c166c544288cb4d90eb6709ae813b69c953e4fad3aa6444797cba3f7318e3c6304f63afa8b198c635199c94a8422b28d5d65
+  metadata.gz: fb00c43e5d7d57f3b17f82efaf494426766b12d5af483195ce89252ec11d402c9dcc5149c3d5dc340f393a125d37c53d93e6ef90d7a60808f2124aa44e7c6688
+  data.tar.gz: d7879e42888f1bb9a0b4305f4d1b1d2e24acb2b0bbf64fde6be45b33f560aee935c1ab936d62e21ddd330467b05cb234032214e47deee6dc76432b775ea3ad9a

data/bin/crunch-job

@@ -118,6 +118,7 @@ $ENV{"CRUNCH_INSTALL"} = "$ENV{CRUNCH_TMP}/opt";
 $ENV{"CRUNCH_WORK"} = $ENV{"JOB_WORK"}; # deprecated
 mkdir ($ENV{"JOB_WORK"});
 
+my %proc;
 my $force_unlock;
 my $git_dir;
 my $jobspec;
@@ -589,56 +590,89 @@ if (!defined $git_archive) {
   }
 }
 else {
-  Log(undef, "Run install script on all workers");
-
-  my @srunargs = ("srun",
-                  "--nodelist=$nodelist",
-                  "-D", $ENV{'TMPDIR'}, "--job-name=$job_id");
-  my @execargs = ("sh", "-c",
-                  "mkdir -p $ENV{CRUNCH_INSTALL} && cd $ENV{CRUNCH_TMP} && perl -");
-
-  $ENV{"CRUNCH_GIT_ARCHIVE_HASH"} = md5_hex($git_archive);
-  my ($install_stderr_r, $install_stderr_w);
-  pipe $install_stderr_r, $install_stderr_w or croak("pipe() failed: $!");
-  set_nonblocking($install_stderr_r);
-  my $installpid = fork();
-  if ($installpid == 0)
-  {
-    close($install_stderr_r);
-    fcntl($install_stderr_w, F_SETFL, 0) or croak($!); # no close-on-exec
-    open(STDOUT, ">&", $install_stderr_w);
-    open(STDERR, ">&", $install_stderr_w);
-    srun (\@srunargs, \@execargs, {}, $build_script . $git_archive);
-    exit (1);
-  }
-  close($install_stderr_w);
-  my $stderr_buf = '';
-  while ($installpid != waitpid(-1, WNOHANG)) {
-    freeze_if_want_freeze ($installpid);
-    # Wait up to 0.1 seconds for something to appear on stderr, then
-    # do a non-blocking read.
-    my $bits = fhbits($install_stderr_r);
-    select ($bits, undef, $bits, 0.1);
-    if (0 < sysread ($install_stderr_r, $stderr_buf, 8192, length($stderr_buf)))
+  my $install_exited;
+  my $install_script_tries_left = 3;
+  for (my $attempts = 0; $attempts < 3; $attempts++) {
+    Log(undef, "Run install script on all workers");
+
+    my @srunargs = ("srun",
+                    "--nodelist=$nodelist",
+                    "-D", $ENV{'TMPDIR'}, "--job-name=$job_id");
+    my @execargs = ("sh", "-c",
+                    "mkdir -p $ENV{CRUNCH_INSTALL} && cd $ENV{CRUNCH_TMP} && perl -");
+
+    $ENV{"CRUNCH_GIT_ARCHIVE_HASH"} = md5_hex($git_archive);
+    my ($install_stderr_r, $install_stderr_w);
+    pipe $install_stderr_r, $install_stderr_w or croak("pipe() failed: $!");
+    set_nonblocking($install_stderr_r);
+    my $installpid = fork();
+    if ($installpid == 0)
     {
-      while ($stderr_buf =~ /^(.*?)\n/) {
-        my $line = $1;
-        substr $stderr_buf, 0, 1+length($line), "";
-        Log(undef, "stderr $line");
+      close($install_stderr_r);
+      fcntl($install_stderr_w, F_SETFL, 0) or croak($!); # no close-on-exec
+      open(STDOUT, ">&", $install_stderr_w);
+      open(STDERR, ">&", $install_stderr_w);
+      srun (\@srunargs, \@execargs, {}, $build_script . $git_archive);
+      exit (1);
+    }
+    close($install_stderr_w);
+    # Tell freeze_if_want_freeze how to kill the child, otherwise the
+    # "waitpid(installpid)" loop won't get interrupted by a freeze:
+    $proc{$installpid} = {};
+    my $stderr_buf = '';
+    # Track whether anything appears on stderr other than slurm errors
+    # ("srun: ...") and the "starting: ..." message printed by the
+    # srun subroutine itself:
+    my $stderr_anything_from_script = 0;
+    my $match_our_own_errors = '^(srun: error: |starting: \[)';
+    while ($installpid != waitpid(-1, WNOHANG)) {
+      freeze_if_want_freeze ($installpid);
+      # Wait up to 0.1 seconds for something to appear on stderr, then
+      # do a non-blocking read.
+      my $bits = fhbits($install_stderr_r);
+      select ($bits, undef, $bits, 0.1);
+      if (0 < sysread ($install_stderr_r, $stderr_buf, 8192, length($stderr_buf)))
+      {
+        while ($stderr_buf =~ /^(.*?)\n/) {
+          my $line = $1;
+          substr $stderr_buf, 0, 1+length($line), "";
+          Log(undef, "stderr $line");
+          if ($line !~ /$match_our_own_errors/) {
+            $stderr_anything_from_script = 1;
+          }
+        }
       }
     }
-  }
-  my $install_exited = $?;
-  close($install_stderr_r);
-  if (length($stderr_buf) > 0) {
-    Log(undef, "stderr $stderr_buf")
+    delete $proc{$installpid};
+    $install_exited = $?;
+    close($install_stderr_r);
+    if (length($stderr_buf) > 0) {
+      if ($stderr_buf !~ /$match_our_own_errors/) {
+        $stderr_anything_from_script = 1;
+      }
+      Log(undef, "stderr $stderr_buf")
+    }
+
+    Log (undef, "Install script exited ".exit_status_s($install_exited));
+    last if $install_exited == 0 || $main::please_freeze;
+    # If the install script fails but doesn't print an error message,
+    # the next thing anyone is likely to do is just run it again in
+    # case it was a transient problem like "slurm communication fails
+    # because the network isn't reliable enough". So we'll just do
+    # that ourselves (up to 3 attempts in total). OTOH, if there is an
+    # error message, the problem is more likely to have a real fix and
+    # we should fail the job so the fixing process can start, instead
+    # of doing 2 more attempts.
+    last if $stderr_anything_from_script;
   }
 
-  Log (undef, "Install script exited ".exit_status_s($install_exited));
   foreach my $tar_filename (map { tar_filename_n($_); } (1..$git_tar_count)) {
     unlink($tar_filename);
   }
-  exit (1) if $install_exited != 0;
+
+  if ($install_exited != 0) {
+    croak("Giving up");
+  }
 }
 
 foreach (qw (script script_version script_parameters runtime_constraints))
@@ -704,7 +738,6 @@ for (my $ii = $#freeslot; $ii >= 0; $ii--) {
 }
 
 Log(undef, "start level $level with $round_num_freeslots slots");
-my %proc;
 my @holdslot;
 my %reader;
 my $progress_is_dirty = 1;

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: arvados-cli
 version: !ruby/object:Gem::Version
-  version: 0.1.20150526143156
+  version: 0.1.20150527201024
 platform: ruby
 authors:
 - Arvados Authors
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-26 00:00:00.000000000 Z
+date: 2015-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: arvados
@@ -178,7 +178,7 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
-description: Arvados command line tools, git commit 9b67268ff45d2a552d21fa39f5086180f537ab4a
+description: Arvados command line tools, git commit 7a53d874994a5a9af273cee1329d9635b7e03edb
 email: gem-dev@curoverse.com
 executables:
 - arv