arvados-cli 0.1.20141006171315 → 0.1.20141006182226
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/crunch-job +73 -15
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: dda195888cf5d8edd4cd36823fa416d43ceefd6a
|
4
|
+
data.tar.gz: 7de92aa09c4f47fc65704a4aecf48b568e82d73c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3e4bc3c2153dd7a015976950935b338bbe796df5839f7d2073f342ff0c3f54ef5b21b9c7f7ad289f0bde5864ab9e11a1d0cb2149d74c1d880308ac3f88f38111
|
7
|
+
data.tar.gz: a9fd3975250cc52032538fc740ac8b48acd223e2a9b603a9c2198b64f51608cafefe0fb513a1548b135d2068aecd7888d43989ed08b63cf52cd1c8ff4c2f217d
|
data/bin/crunch-job
CHANGED
@@ -141,7 +141,6 @@ $SIG{'USR2'} = sub
|
|
141
141
|
|
142
142
|
|
143
143
|
my $arv = Arvados->new('apiVersion' => 'v1');
|
144
|
-
my $local_logfile;
|
145
144
|
|
146
145
|
my $User = $arv->{'users'}->{'current'}->execute;
|
147
146
|
|
@@ -184,7 +183,7 @@ else
|
|
184
183
|
$job_id = $Job->{'uuid'};
|
185
184
|
|
186
185
|
my $keep_logfile = $job_id . '.log.txt';
|
187
|
-
$
|
186
|
+
log_writer_start($keep_logfile);
|
188
187
|
|
189
188
|
$Job->{'runtime_constraints'} ||= {};
|
190
189
|
$Job->{'runtime_constraints'}->{'max_tasks_per_node'} ||= 0;
|
@@ -1275,6 +1274,73 @@ sub fhbits
|
|
1275
1274
|
}
|
1276
1275
|
|
1277
1276
|
|
1277
|
+
# Send log output to Keep via arv-put.
|
1278
|
+
#
|
1279
|
+
# $log_pipe_in and $log_pipe_out are the input and output filehandles to the arv-put pipe.
|
1280
|
+
# $log_pipe_pid is the pid of the arv-put subprocess.
|
1281
|
+
#
|
1282
|
+
# The only functions that should access these variables directly are:
|
1283
|
+
#
|
1284
|
+
# log_writer_start($logfilename)
|
1285
|
+
# Starts an arv-put pipe, reading data on stdin and writing it to
|
1286
|
+
# a $logfilename file in an output collection.
|
1287
|
+
#
|
1288
|
+
# log_writer_send($txt)
|
1289
|
+
# Writes $txt to the output log collection.
|
1290
|
+
#
|
1291
|
+
# log_writer_finish()
|
1292
|
+
# Closes the arv-put pipe and returns the output that it produces.
|
1293
|
+
#
|
1294
|
+
# log_writer_is_active()
|
1295
|
+
# Returns a true value if there is currently a live arv-put
|
1296
|
+
# process, false otherwise.
|
1297
|
+
#
|
1298
|
+
my ($log_pipe_in, $log_pipe_out, $log_pipe_pid);
|
1299
|
+
|
1300
|
+
sub log_writer_start($)
|
1301
|
+
{
|
1302
|
+
my $logfilename = shift;
|
1303
|
+
$log_pipe_pid = open2($log_pipe_out, $log_pipe_in,
|
1304
|
+
'arv-put', '--portable-data-hash',
|
1305
|
+
'--retries', '3',
|
1306
|
+
'--filename', $logfilename,
|
1307
|
+
'-');
|
1308
|
+
}
|
1309
|
+
|
1310
|
+
sub log_writer_send($)
|
1311
|
+
{
|
1312
|
+
my $txt = shift;
|
1313
|
+
print $log_pipe_in $txt;
|
1314
|
+
}
|
1315
|
+
|
1316
|
+
sub log_writer_finish()
|
1317
|
+
{
|
1318
|
+
return unless $log_pipe_pid;
|
1319
|
+
|
1320
|
+
close($log_pipe_in);
|
1321
|
+
my $arv_put_output;
|
1322
|
+
|
1323
|
+
my $s = IO::Select->new($log_pipe_out);
|
1324
|
+
if ($s->can_read(120)) {
|
1325
|
+
sysread($log_pipe_out, $arv_put_output, 1024);
|
1326
|
+
chomp($arv_put_output);
|
1327
|
+
} else {
|
1328
|
+
Log (undef, "timed out reading from 'arv-put'");
|
1329
|
+
}
|
1330
|
+
|
1331
|
+
waitpid($log_pipe_pid, 0);
|
1332
|
+
$log_pipe_pid = $log_pipe_in = $log_pipe_out = undef;
|
1333
|
+
if ($?) {
|
1334
|
+
Log("log_writer_finish: arv-put returned error $?")
|
1335
|
+
}
|
1336
|
+
|
1337
|
+
return $arv_put_output;
|
1338
|
+
}
|
1339
|
+
|
1340
|
+
sub log_writer_is_active() {
|
1341
|
+
return $log_pipe_pid;
|
1342
|
+
}
|
1343
|
+
|
1278
1344
|
sub Log # ($jobstep_id, $logmessage)
|
1279
1345
|
{
|
1280
1346
|
if ($_[1] =~ /\n/) {
|
@@ -1288,15 +1354,15 @@ sub Log # ($jobstep_id, $logmessage)
|
|
1288
1354
|
$message =~ s{([^ -\176])}{"\\" . sprintf ("%03o", ord($1))}ge;
|
1289
1355
|
$message .= "\n";
|
1290
1356
|
my $datetime;
|
1291
|
-
if (
|
1357
|
+
if (log_writer_is_active() || -t STDERR) {
|
1292
1358
|
my @gmtime = gmtime;
|
1293
1359
|
$datetime = sprintf ("%04d-%02d-%02d_%02d:%02d:%02d",
|
1294
1360
|
$gmtime[5]+1900, $gmtime[4]+1, @gmtime[3,2,1,0]);
|
1295
1361
|
}
|
1296
1362
|
print STDERR ((-t STDERR) ? ($datetime." ".$message) : $message);
|
1297
1363
|
|
1298
|
-
if (
|
1299
|
-
|
1364
|
+
if (log_writer_is_active()) {
|
1365
|
+
log_writer_send($datetime . " " . $message);
|
1300
1366
|
}
|
1301
1367
|
}
|
1302
1368
|
|
@@ -1309,7 +1375,7 @@ sub croak
|
|
1309
1375
|
freeze() if @jobstep_todo;
|
1310
1376
|
collate_output() if @jobstep_todo;
|
1311
1377
|
cleanup();
|
1312
|
-
save_meta() if
|
1378
|
+
save_meta() if log_writer_is_active();
|
1313
1379
|
die;
|
1314
1380
|
}
|
1315
1381
|
|
@@ -1330,15 +1396,7 @@ sub save_meta
|
|
1330
1396
|
my $justcheckpoint = shift; # false if this will be the last meta saved
|
1331
1397
|
return if $justcheckpoint; # checkpointing is not relevant post-Warehouse.pm
|
1332
1398
|
|
1333
|
-
$
|
1334
|
-
my $retry_count = put_retry_count();
|
1335
|
-
my $cmd = "arv-put --portable-data-hash --retries $retry_count " .
|
1336
|
-
"--filename ''\Q$keep_logfile\E " . quotemeta($local_logfile->filename);
|
1337
|
-
my $loglocator = `$cmd`;
|
1338
|
-
die "system $cmd failed: $?" if $?;
|
1339
|
-
chomp($loglocator);
|
1340
|
-
|
1341
|
-
$local_logfile = undef; # the temp file is automatically deleted
|
1399
|
+
my $loglocator = log_writer_finish();
|
1342
1400
|
Log (undef, "log manifest is $loglocator");
|
1343
1401
|
$Job->{'log'} = $loglocator;
|
1344
1402
|
$Job->update_attributes('log', $loglocator) if $job_has_uuid;
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: arvados-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.20141006182226
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Arvados Authors
|
@@ -178,7 +178,7 @@ dependencies:
|
|
178
178
|
- - "<"
|
179
179
|
- !ruby/object:Gem::Version
|
180
180
|
version: 1.0.0
|
181
|
-
description: Arvados command line tools, git commit
|
181
|
+
description: Arvados command line tools, git commit d65963976174bd9c94fdb0b91eeb8a281f01e7b3
|
182
182
|
email: gem-dev@curoverse.com
|
183
183
|
executables:
|
184
184
|
- arv
|