arvados-cli 0.1.20141006171315 → 0.1.20141006182226

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 68b3a31004a457fb98b8cfdeb371c54e148dc1ff
-  data.tar.gz: aa6fabddda25ea6217cc38c90d6c800ba0401e4c
+  metadata.gz: dda195888cf5d8edd4cd36823fa416d43ceefd6a
+  data.tar.gz: 7de92aa09c4f47fc65704a4aecf48b568e82d73c
 SHA512:
-  metadata.gz: f2b6c2ff8518e6458cb616bd97fdd3f87d20352969494a174a56b48a5e21f64d19e2749f809c964e190b0a14270657923b1635bf2167709b20b21d5222ac1144
-  data.tar.gz: aaaf716e41d5e6bf1910233c2bc3821b88c768097bdd5c87db53e149676ff2aa232cd8206aacdf484088d1c6d6459e8b2c41efb6577e544dfeb8538c81b7063f
+  metadata.gz: 3e4bc3c2153dd7a015976950935b338bbe796df5839f7d2073f342ff0c3f54ef5b21b9c7f7ad289f0bde5864ab9e11a1d0cb2149d74c1d880308ac3f88f38111
+  data.tar.gz: a9fd3975250cc52032538fc740ac8b48acd223e2a9b603a9c2198b64f51608cafefe0fb513a1548b135d2068aecd7888d43989ed08b63cf52cd1c8ff4c2f217d

data/bin/crunch-job

@@ -141,7 +141,6 @@ $SIG{'USR2'} = sub
 
 
 my $arv = Arvados->new('apiVersion' => 'v1');
-my $local_logfile;
 
 my $User = $arv->{'users'}->{'current'}->execute;
 
@@ -184,7 +183,7 @@ else
 $job_id = $Job->{'uuid'};
 
 my $keep_logfile = $job_id . '.log.txt';
-$local_logfile = File::Temp->new();
+log_writer_start($keep_logfile);
 
 $Job->{'runtime_constraints'} ||= {};
 $Job->{'runtime_constraints'}->{'max_tasks_per_node'} ||= 0;
@@ -1275,6 +1274,73 @@ sub fhbits
 }
 
 
+# Send log output to Keep via arv-put.
+#
+# $log_pipe_in and $log_pipe_out are the input and output filehandles to the arv-put pipe.
+# $log_pipe_pid is the pid of the arv-put subprocess.
+#
+# The only functions that should access these variables directly are:
+#
+# log_writer_start($logfilename)
+#     Starts an arv-put pipe, reading data on stdin and writing it to
+#     a $logfilename file in an output collection.
+#
+# log_writer_send($txt)
+#     Writes $txt to the output log collection.
+#
+# log_writer_finish()
+#     Closes the arv-put pipe and returns the output that it produces.
+#
+# log_writer_is_active()
+#     Returns a true value if there is currently a live arv-put
+#     process, false otherwise.
+#
+my ($log_pipe_in, $log_pipe_out, $log_pipe_pid);
+
+sub log_writer_start($)
+{
+  my $logfilename = shift;
+  $log_pipe_pid = open2($log_pipe_out, $log_pipe_in,
+                        'arv-put', '--portable-data-hash',
+                        '--retries', '3',
+                        '--filename', $logfilename,
+                        '-');
+}
+
+sub log_writer_send($)
+{
+  my $txt = shift;
+  print $log_pipe_in $txt;
+}
+
+sub log_writer_finish()
+{
+  return unless $log_pipe_pid;
+
+  close($log_pipe_in);
+  my $arv_put_output;
+
+  my $s = IO::Select->new($log_pipe_out);
+  if ($s->can_read(120)) {
+    sysread($log_pipe_out, $arv_put_output, 1024);
+    chomp($arv_put_output);
+  } else {
+    Log (undef, "timed out reading from 'arv-put'");
+  }
+
+  waitpid($log_pipe_pid, 0);
+  $log_pipe_pid = $log_pipe_in = $log_pipe_out = undef;
+  if ($?) {
+    Log("log_writer_finish: arv-put returned error $?")
+  }
+
+  return $arv_put_output;
+}
+
+sub log_writer_is_active() {
+  return $log_pipe_pid;
+}
+
 sub Log				# ($jobstep_id, $logmessage)
 {
   if ($_[1] =~ /\n/) {
@@ -1288,15 +1354,15 @@ sub Log				# ($jobstep_id, $logmessage)
   $message =~ s{([^ -\176])}{"\\" . sprintf ("%03o", ord($1))}ge;
   $message .= "\n";
   my $datetime;
-  if ($local_logfile || -t STDERR) {
+  if (log_writer_is_active() || -t STDERR) {
     my @gmtime = gmtime;
     $datetime = sprintf ("%04d-%02d-%02d_%02d:%02d:%02d",
 			 $gmtime[5]+1900, $gmtime[4]+1, @gmtime[3,2,1,0]);
   }
   print STDERR ((-t STDERR) ? ($datetime." ".$message) : $message);
 
-  if ($local_logfile) {
-    print $local_logfile $datetime . " " . $message;
+  if (log_writer_is_active()) {
+    log_writer_send($datetime . " " . $message);
   }
 }
 
@@ -1309,7 +1375,7 @@ sub croak
   freeze() if @jobstep_todo;
   collate_output() if @jobstep_todo;
   cleanup();
-  save_meta() if $local_logfile;
+  save_meta() if log_writer_is_active();
   die;
 }
 
@@ -1330,15 +1396,7 @@ sub save_meta
   my $justcheckpoint = shift; # false if this will be the last meta saved
   return if $justcheckpoint;  # checkpointing is not relevant post-Warehouse.pm
 
-  $local_logfile->flush;
-  my $retry_count = put_retry_count();
-  my $cmd = "arv-put --portable-data-hash --retries $retry_count " .
-      "--filename ''\Q$keep_logfile\E " . quotemeta($local_logfile->filename);
-  my $loglocator = `$cmd`;
-  die "system $cmd failed: $?" if $?;
-  chomp($loglocator);
-
-  $local_logfile = undef;   # the temp file is automatically deleted
+  my $loglocator = log_writer_finish();
   Log (undef, "log manifest is $loglocator");
   $Job->{'log'} = $loglocator;
   $Job->update_attributes('log', $loglocator) if $job_has_uuid;

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: arvados-cli
 version: !ruby/object:Gem::Version
-  version: 0.1.20141006171315
+  version: 0.1.20141006182226
 platform: ruby
 authors:
 - Arvados Authors
@@ -178,7 +178,7 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
-description: Arvados command line tools, git commit 77e3dec8727c1fc00665531a39583c85b2c404bb
+description: Arvados command line tools, git commit d65963976174bd9c94fdb0b91eeb8a281f01e7b3
 email: gem-dev@curoverse.com
 executables:
 - arv