arvados-cli 0.1.20141006182226 → 0.1.20141006212502
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/crunch-job +227 -155
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b84e06d6def466e49e5c53c836efe17babcc9267
|
4
|
+
data.tar.gz: 957bec18268b60540dd207a9b53d3b1bb45164de
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a6dd627d31352cdb3a0958516090466fc0bcf1e5b2d85f66189023642a7258f89d25dce7b90d4b53223b5bf3176e716bbe2082a7ecc6adafa9edfc4f6ce22bcd
|
7
|
+
data.tar.gz: 9a2321cb216a5c8bd596137a78e4c846cee688eacde21322486a81986fd46c799c0f765816f8effbb2467296cfefd0d7b397a5e408ae6e035df379ea75fea54c
|
data/bin/crunch-job
CHANGED
@@ -83,7 +83,7 @@ use IPC::Open2;
|
|
83
83
|
use IO::Select;
|
84
84
|
use File::Temp;
|
85
85
|
use Fcntl ':flock';
|
86
|
-
use File::Path qw( make_path );
|
86
|
+
use File::Path qw( make_path remove_tree );
|
87
87
|
|
88
88
|
use constant EX_TEMPFAIL => 75;
|
89
89
|
|
@@ -125,8 +125,7 @@ if (defined $job_api_token) {
|
|
125
125
|
}
|
126
126
|
|
127
127
|
my $have_slurm = exists $ENV{SLURM_JOBID} && exists $ENV{SLURM_NODELIST};
|
128
|
-
my $
|
129
|
-
my $local_job = !$job_has_uuid;
|
128
|
+
my $local_job = 0;
|
130
129
|
|
131
130
|
|
132
131
|
$SIG{'USR1'} = sub
|
@@ -144,12 +143,13 @@ my $arv = Arvados->new('apiVersion' => 'v1');
|
|
144
143
|
|
145
144
|
my $User = $arv->{'users'}->{'current'}->execute;
|
146
145
|
|
147
|
-
my $Job
|
146
|
+
my $Job;
|
148
147
|
my $job_id;
|
149
148
|
my $dbh;
|
150
149
|
my $sth;
|
151
|
-
if ($
|
150
|
+
if ($jobspec =~ /^[-a-z\d]+$/)
|
152
151
|
{
|
152
|
+
# $jobspec is an Arvados UUID, not a JSON job specification
|
153
153
|
$Job = $arv->{'jobs'}->{'get'}->execute('uuid' => $jobspec);
|
154
154
|
if (!$force_unlock) {
|
155
155
|
# Claim this job, and make sure nobody else does
|
@@ -175,10 +175,9 @@ else
|
|
175
175
|
|
176
176
|
$Job->{'is_locked_by_uuid'} = $User->{'uuid'};
|
177
177
|
$Job->{'started_at'} = gmtime;
|
178
|
+
$Job->{'state'} = 'Running';
|
178
179
|
|
179
180
|
$Job = $arv->{'jobs'}->{'create'}->execute('job' => $Job);
|
180
|
-
|
181
|
-
$job_has_uuid = 1;
|
182
181
|
}
|
183
182
|
$job_id = $Job->{'uuid'};
|
184
183
|
|
@@ -330,137 +329,202 @@ if (!$have_slurm)
|
|
330
329
|
|
331
330
|
|
332
331
|
my $build_script;
|
332
|
+
do {
|
333
|
+
local $/ = undef;
|
334
|
+
$build_script = <DATA>;
|
335
|
+
};
|
336
|
+
my $nodelist = join(",", @node);
|
333
337
|
|
338
|
+
if (!defined $no_clear_tmp) {
|
339
|
+
# Clean out crunch_tmp/work, crunch_tmp/opt, crunch_tmp/src*
|
340
|
+
Log (undef, "Clean work dirs");
|
334
341
|
|
335
|
-
$
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
my $clear_tmp_cmd = 'rm -rf $JOB_WORK $CRUNCH_TMP/opt $CRUNCH_TMP/src*';
|
342
|
-
system($clear_tmp_cmd) == 0
|
343
|
-
or croak ("`$clear_tmp_cmd` failed: ".($?>>8));
|
344
|
-
}
|
345
|
-
$ENV{"CRUNCH_SRC"} = $Job->{script_version};
|
346
|
-
for my $src_path ("$ENV{CRUNCH_SRC}/arvados/sdk/python") {
|
347
|
-
if (-d $src_path) {
|
348
|
-
system("virtualenv", "$ENV{CRUNCH_TMP}/opt") == 0
|
349
|
-
or croak ("virtualenv $ENV{CRUNCH_TMP}/opt failed: exit ".($?>>8));
|
350
|
-
system ("cd $src_path && ./build.sh && \$CRUNCH_TMP/opt/bin/python setup.py install")
|
351
|
-
== 0
|
352
|
-
or croak ("setup.py in $src_path failed: exit ".($?>>8));
|
353
|
-
}
|
342
|
+
my $cleanpid = fork();
|
343
|
+
if ($cleanpid == 0)
|
344
|
+
{
|
345
|
+
srun (["srun", "--nodelist=$nodelist", "-D", $ENV{'TMPDIR'}],
|
346
|
+
['bash', '-c', 'if mount | grep -q $JOB_WORK/; then for i in $JOB_WORK/*keep; do /bin/fusermount -z -u $i; done; fi; sleep 1; rm -rf $JOB_WORK $CRUNCH_TMP/opt $CRUNCH_TMP/src*']);
|
347
|
+
exit (1);
|
354
348
|
}
|
349
|
+
while (1)
|
350
|
+
{
|
351
|
+
last if $cleanpid == waitpid (-1, WNOHANG);
|
352
|
+
freeze_if_want_freeze ($cleanpid);
|
353
|
+
select (undef, undef, undef, 0.1);
|
354
|
+
}
|
355
|
+
Log (undef, "Cleanup command exited ".exit_status_s($?));
|
355
356
|
}
|
356
|
-
else
|
357
|
-
{
|
358
|
-
do {
|
359
|
-
local $/ = undef;
|
360
|
-
$build_script = <DATA>;
|
361
|
-
};
|
362
|
-
Log (undef, "Install revision ".$Job->{script_version});
|
363
|
-
my $nodelist = join(",", @node);
|
364
357
|
|
365
|
-
if (!defined $no_clear_tmp) {
|
366
|
-
# Clean out crunch_tmp/work, crunch_tmp/opt, crunch_tmp/src*
|
367
358
|
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
select (undef, undef, undef, 0.1);
|
380
|
-
}
|
381
|
-
Log (undef, "Clean-work-dir exited $?");
|
382
|
-
}
|
359
|
+
my $git_archive;
|
360
|
+
if (!defined $git_dir && $Job->{'script_version'} =~ m{^/}) {
|
361
|
+
# If we're in user-land (i.e., not called from crunch-dispatch)
|
362
|
+
# script_version can be an absolute directory path, signifying we
|
363
|
+
# should work straight out of that directory instead of using a git
|
364
|
+
# commit.
|
365
|
+
$ENV{"CRUNCH_SRC_COMMIT"} = $Job->{'script_version'};
|
366
|
+
$ENV{"CRUNCH_SRC"} = $Job->{'script_version'};
|
367
|
+
}
|
368
|
+
else {
|
369
|
+
$ENV{"CRUNCH_SRC"} = "$ENV{CRUNCH_TMP}/src";
|
383
370
|
|
384
371
|
# Install requested code version
|
385
|
-
|
386
|
-
my @execargs;
|
387
|
-
my @srunargs = ("srun",
|
388
|
-
"--nodelist=$nodelist",
|
389
|
-
"-D", $ENV{'TMPDIR'}, "--job-name=$job_id");
|
372
|
+
Log (undef, "Looking for version ".$Job->{script_version}." from repository ".$Job->{repository});
|
390
373
|
|
391
374
|
$ENV{"CRUNCH_SRC_COMMIT"} = $Job->{script_version};
|
392
|
-
$ENV{"CRUNCH_SRC"} = "$ENV{CRUNCH_TMP}/src";
|
393
|
-
|
394
|
-
my $commit;
|
395
|
-
my $git_archive;
|
396
|
-
my $treeish = $Job->{'script_version'};
|
397
375
|
|
398
|
-
# If we're running under crunch-dispatch, it will have
|
399
|
-
# appropriate source tree into its own repository, and
|
400
|
-
# repo's path as $git_dir.
|
401
|
-
# script_version was specified, it's up to the user to provide the
|
402
|
-
# full path to a local repository in Job->{repository}.
|
376
|
+
# If we're running under crunch-dispatch, it will have already
|
377
|
+
# pulled the appropriate source tree into its own repository, and
|
378
|
+
# given us that repo's path as $git_dir.
|
403
379
|
#
|
404
|
-
#
|
405
|
-
#
|
380
|
+
# If we're running a "local" job, we might have to fetch content
|
381
|
+
# from a remote repository.
|
406
382
|
#
|
407
|
-
#
|
408
|
-
#
|
409
|
-
#
|
410
|
-
my $repo = $git_dir || $
|
411
|
-
|
383
|
+
# (Currently crunch-dispatch gives a local path with --git-dir, but
|
384
|
+
# we might as well accept URLs there too in case it changes its
|
385
|
+
# mind.)
|
386
|
+
my $repo = $git_dir || $Job->{'repository'};
|
387
|
+
|
388
|
+
# Repository can be remote or local. If remote, we'll need to fetch it
|
389
|
+
# to a local dir before doing `git log` et al.
|
390
|
+
my $repo_location;
|
391
|
+
|
392
|
+
if ($repo =~ m{://|^[^/]*:}) {
|
393
|
+
# $repo is a git url we can clone, like git:// or https:// or
|
394
|
+
# file:/// or [user@]host:repo.git. Note "user/name@host:foo" is
|
395
|
+
# not recognized here because distinguishing that from a local
|
396
|
+
# path is too fragile. If you really need something strange here,
|
397
|
+
# use the ssh:// form.
|
398
|
+
$repo_location = 'remote';
|
399
|
+
} elsif ($repo =~ m{^\.*/}) {
|
400
|
+
# $repo is a local path to a git index. We'll also resolve ../foo
|
401
|
+
# to ../foo/.git if the latter is a directory. To help
|
402
|
+
# disambiguate local paths from named hosted repositories, this
|
403
|
+
# form must be given as ./ or ../ if it's a relative path.
|
404
|
+
if (-d "$repo/.git") {
|
405
|
+
$repo = "$repo/.git";
|
406
|
+
}
|
407
|
+
$repo_location = 'local';
|
408
|
+
} else {
|
409
|
+
# $repo is none of the above. It must be the name of a hosted
|
410
|
+
# repository.
|
411
|
+
my $arv_repo_list = $arv->{'repositories'}->{'list'}->execute(
|
412
|
+
'filters' => [['name','=',$repo]]
|
413
|
+
)->{'items'};
|
414
|
+
my $n_found = scalar @{$arv_repo_list};
|
415
|
+
if ($n_found > 0) {
|
416
|
+
Log(undef, "Repository '$repo' -> "
|
417
|
+
. join(", ", map { $_->{'uuid'} } @{$arv_repo_list}));
|
418
|
+
}
|
419
|
+
if ($n_found != 1) {
|
420
|
+
croak("Error: Found $n_found repositories with name '$repo'.");
|
421
|
+
}
|
422
|
+
$repo = $arv_repo_list->[0]->{'fetch_url'};
|
423
|
+
$repo_location = 'remote';
|
424
|
+
}
|
425
|
+
Log(undef, "Using $repo_location repository '$repo'");
|
412
426
|
$ENV{"CRUNCH_SRC_URL"} = $repo;
|
413
427
|
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
428
|
+
# Resolve given script_version (we'll call that $treeish here) to a
|
429
|
+
# commit sha1 ($commit).
|
430
|
+
my $treeish = $Job->{'script_version'};
|
431
|
+
my $commit;
|
432
|
+
if ($repo_location eq 'remote') {
|
433
|
+
# We minimize excess object-fetching by re-using the same bare
|
434
|
+
# repository in CRUNCH_TMP/.git for multiple crunch-jobs -- we
|
435
|
+
# just keep adding remotes to it as needed.
|
436
|
+
my $local_repo = $ENV{'CRUNCH_TMP'}."/.git";
|
437
|
+
my $gitcmd = "git --git-dir=\Q$local_repo\E";
|
438
|
+
|
439
|
+
# Set up our local repo for caching remote objects, making
|
440
|
+
# archives, etc.
|
441
|
+
if (!-d $local_repo) {
|
442
|
+
make_path($local_repo) or croak("Error: could not create $local_repo");
|
443
|
+
}
|
444
|
+
# This works (exits 0 and doesn't delete fetched objects) even
|
445
|
+
# if $local_repo is already initialized:
|
446
|
+
`$gitcmd init --bare`;
|
447
|
+
if ($?) {
|
448
|
+
croak("Error: $gitcmd init --bare exited ".exit_status_s($?));
|
449
|
+
}
|
450
|
+
|
451
|
+
# If $treeish looks like a hash (or abbrev hash) we look it up in
|
452
|
+
# our local cache first, since that's cheaper. (We don't want to
|
453
|
+
# do that with tags/branches though -- those change over time, so
|
454
|
+
# they should always be resolved by the remote repo.)
|
455
|
+
if ($treeish =~ /^[0-9a-f]{3,40}$/s) {
|
456
|
+
# Hide stderr because it's normal for this to fail:
|
457
|
+
my $sha1 = `$gitcmd rev-list -n1 ''\Q$treeish\E 2>/dev/null`;
|
458
|
+
if ($? == 0 &&
|
459
|
+
$sha1 =~ /^$treeish/ && # Don't use commit 123 @ branch abc!
|
460
|
+
$sha1 =~ /^([0-9a-f]{40})$/s) {
|
461
|
+
$commit = $1;
|
462
|
+
Log(undef, "Commit $commit already present in $local_repo");
|
463
|
+
}
|
464
|
+
}
|
465
|
+
|
466
|
+
if (!defined $commit) {
|
467
|
+
# If $treeish isn't just a hash or abbrev hash, or isn't here
|
468
|
+
# yet, we need to fetch the remote to resolve it correctly.
|
419
469
|
|
420
|
-
|
470
|
+
# First, remove all local heads. This prevents a name that does
|
471
|
+
# not exist on the remote from resolving to (or colliding with)
|
472
|
+
# a previously fetched branch or tag (possibly from a different
|
473
|
+
# remote).
|
474
|
+
remove_tree("$local_repo/refs/heads", {keep_root => 1});
|
421
475
|
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
$commit = $gitlog;
|
428
|
-
Log(undef, "Using commit $commit for Subversion revision $treeish");
|
476
|
+
Log(undef, "Fetching objects from $repo to $local_repo");
|
477
|
+
`$gitcmd fetch --no-progress --tags ''\Q$repo\E \Q+refs/heads/*:refs/heads/*\E`;
|
478
|
+
if ($?) {
|
479
|
+
croak("Error: `$gitcmd fetch` exited ".exit_status_s($?));
|
480
|
+
}
|
429
481
|
}
|
482
|
+
|
483
|
+
# Now that the data is all here, we will use our local repo for
|
484
|
+
# the rest of our git activities.
|
485
|
+
$repo = $local_repo;
|
430
486
|
}
|
431
487
|
|
432
|
-
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
$Job->update_attributes('script_version' => $commit) or
|
449
|
-
croak("Error while updating job");
|
450
|
-
}
|
488
|
+
my $gitcmd = "git --git-dir=\Q$repo\E";
|
489
|
+
my $sha1 = `$gitcmd rev-list -n1 ''\Q$treeish\E`;
|
490
|
+
unless ($? == 0 && $sha1 =~ /^([0-9a-f]{40})$/) {
|
491
|
+
croak("`$gitcmd rev-list` exited "
|
492
|
+
.exit_status_s($?)
|
493
|
+
.", '$treeish' not found. Giving up.");
|
494
|
+
}
|
495
|
+
$commit = $1;
|
496
|
+
Log(undef, "Version $treeish is commit $commit");
|
497
|
+
|
498
|
+
if ($commit ne $Job->{'script_version'}) {
|
499
|
+
# Record the real commit id in the database, frozentokey, logs,
|
500
|
+
# etc. -- instead of an abbreviation or a branch name which can
|
501
|
+
# become ambiguous or point to a different commit in the future.
|
502
|
+
if (!$Job->update_attributes('script_version' => $commit)) {
|
503
|
+
croak("Error: failed to update job's script_version attribute");
|
451
504
|
}
|
452
505
|
}
|
453
506
|
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
|
458
|
-
$git_archive = `git --git-dir=\Q$repo\E archive ''\Q$commit\E`;
|
459
|
-
croak("git archive failed: exit " . ($? >> 8)) if ($? != 0);
|
507
|
+
$ENV{"CRUNCH_SRC_COMMIT"} = $commit;
|
508
|
+
$git_archive = `$gitcmd archive ''\Q$commit\E`;
|
509
|
+
if ($?) {
|
510
|
+
croak("Error: $gitcmd archive exited ".exit_status_s($?));
|
460
511
|
}
|
461
|
-
|
462
|
-
|
512
|
+
}
|
513
|
+
|
514
|
+
if (!defined $git_archive) {
|
515
|
+
Log(undef, "Skip install phase (no git archive)");
|
516
|
+
if ($have_slurm) {
|
517
|
+
Log(undef, "Warning: This probably means workers have no source tree!");
|
463
518
|
}
|
519
|
+
}
|
520
|
+
else {
|
521
|
+
Log(undef, "Run install script on all workers");
|
522
|
+
|
523
|
+
my @srunargs = ("srun",
|
524
|
+
"--nodelist=$nodelist",
|
525
|
+
"-D", $ENV{'TMPDIR'}, "--job-name=$job_id");
|
526
|
+
my @execargs = ("sh", "-c",
|
527
|
+
"mkdir -p $ENV{CRUNCH_INSTALL} && cd $ENV{CRUNCH_TMP} && perl -");
|
464
528
|
|
465
529
|
# Note: this section is almost certainly unnecessary if we're
|
466
530
|
# running tasks in docker containers.
|
@@ -476,7 +540,7 @@ else
|
|
476
540
|
freeze_if_want_freeze ($installpid);
|
477
541
|
select (undef, undef, undef, 0.1);
|
478
542
|
}
|
479
|
-
Log (undef, "Install exited $?
|
543
|
+
Log (undef, "Install script exited ".exit_status_s($?));
|
480
544
|
}
|
481
545
|
|
482
546
|
if (!$have_slurm)
|
@@ -515,7 +579,8 @@ fi
|
|
515
579
|
}
|
516
580
|
if ($? != 0)
|
517
581
|
{
|
518
|
-
croak("Installing Docker image from $docker_locator
|
582
|
+
croak("Installing Docker image from $docker_locator exited "
|
583
|
+
.exit_status_s($?));
|
519
584
|
}
|
520
585
|
}
|
521
586
|
|
@@ -838,7 +903,7 @@ else {
|
|
838
903
|
});
|
839
904
|
Log(undef, "output uuid " . $output->{uuid});
|
840
905
|
Log(undef, "output hash " . $output->{portable_data_hash});
|
841
|
-
$Job->update_attributes('output' => $output->{portable_data_hash})
|
906
|
+
$Job->update_attributes('output' => $output->{portable_data_hash});
|
842
907
|
};
|
843
908
|
if ($@) {
|
844
909
|
Log (undef, "Failed to register output manifest: $@");
|
@@ -849,15 +914,15 @@ Log (undef, "finish");
|
|
849
914
|
|
850
915
|
save_meta();
|
851
916
|
|
852
|
-
|
853
|
-
|
854
|
-
|
855
|
-
|
856
|
-
|
857
|
-
}
|
917
|
+
my $final_state;
|
918
|
+
if ($collated_output && $main::success) {
|
919
|
+
$final_state = 'Complete';
|
920
|
+
} else {
|
921
|
+
$final_state = 'Failed';
|
858
922
|
}
|
923
|
+
$Job->update_attributes('state' => $final_state);
|
859
924
|
|
860
|
-
exit ($
|
925
|
+
exit (($final_state eq 'Complete') ? 0 : 1);
|
861
926
|
|
862
927
|
|
863
928
|
|
@@ -872,9 +937,7 @@ sub update_progress_stats
|
|
872
937
|
$Job->{'tasks_summary'}->{'todo'} = $todo;
|
873
938
|
$Job->{'tasks_summary'}->{'done'} = $done;
|
874
939
|
$Job->{'tasks_summary'}->{'running'} = $running;
|
875
|
-
|
876
|
-
$Job->update_attributes('tasks_summary' => $Job->{'tasks_summary'});
|
877
|
-
}
|
940
|
+
$Job->update_attributes('tasks_summary' => $Job->{'tasks_summary'});
|
878
941
|
Log (undef, "status: $done done, $running running, $todo todo");
|
879
942
|
$progress_is_dirty = 0;
|
880
943
|
}
|
@@ -895,10 +958,7 @@ sub reapchildren
|
|
895
958
|
|
896
959
|
my $childstatus = $?;
|
897
960
|
my $exitvalue = $childstatus >> 8;
|
898
|
-
my $exitinfo =
|
899
|
-
$exitvalue,
|
900
|
-
$childstatus & 127,
|
901
|
-
($childstatus & 128 ? ' core dump' : ''));
|
961
|
+
my $exitinfo = "exit ".exit_status_s($childstatus);
|
902
962
|
$Jobstep->{'arvados_task'}->reload;
|
903
963
|
my $task_success = $Jobstep->{'arvados_task'}->{success};
|
904
964
|
|
@@ -944,10 +1004,8 @@ sub reapchildren
|
|
944
1004
|
$main::success = 0;
|
945
1005
|
$main::please_freeze = 1;
|
946
1006
|
}
|
947
|
-
|
948
|
-
|
949
|
-
push @jobstep_todo, $jobstepid;
|
950
|
-
}
|
1007
|
+
# Put this task back on the todo queue
|
1008
|
+
push @jobstep_todo, $jobstepid;
|
951
1009
|
$Job->{'tasks_summary'}->{'failed'}++;
|
952
1010
|
}
|
953
1011
|
else
|
@@ -1005,23 +1063,21 @@ sub check_refresh_wanted
|
|
1005
1063
|
my @stat = stat $ENV{"CRUNCH_REFRESH_TRIGGER"};
|
1006
1064
|
if (@stat && $stat[9] > $latest_refresh) {
|
1007
1065
|
$latest_refresh = scalar time;
|
1008
|
-
|
1009
|
-
|
1010
|
-
|
1011
|
-
|
1012
|
-
|
1013
|
-
|
1014
|
-
|
1015
|
-
|
1016
|
-
if ($Job->{'state'}
|
1017
|
-
|
1018
|
-
|
1019
|
-
|
1020
|
-
Log (undef, "Job state unexpectedly changed to " . $Job->{'state'});
|
1021
|
-
}
|
1022
|
-
$main::success = 0;
|
1023
|
-
$main::please_freeze = 1;
|
1066
|
+
my $Job2 = $arv->{'jobs'}->{'get'}->execute('uuid' => $jobspec);
|
1067
|
+
for my $attr ('cancelled_at',
|
1068
|
+
'cancelled_by_user_uuid',
|
1069
|
+
'cancelled_by_client_uuid',
|
1070
|
+
'state') {
|
1071
|
+
$Job->{$attr} = $Job2->{$attr};
|
1072
|
+
}
|
1073
|
+
if ($Job->{'state'} ne "Running") {
|
1074
|
+
if ($Job->{'state'} eq "Cancelled") {
|
1075
|
+
Log (undef, "Job cancelled at " . $Job->{'cancelled_at'} . " by user " . $Job->{'cancelled_by_user_uuid'});
|
1076
|
+
} else {
|
1077
|
+
Log (undef, "Job state unexpectedly changed to " . $Job->{'state'});
|
1024
1078
|
}
|
1079
|
+
$main::success = 0;
|
1080
|
+
$main::please_freeze = 1;
|
1025
1081
|
}
|
1026
1082
|
}
|
1027
1083
|
}
|
@@ -1223,10 +1279,13 @@ sub collate_output
|
|
1223
1279
|
if ($s->can_read(120)) {
|
1224
1280
|
sysread($child_out, $joboutput, 64 * 1024 * 1024);
|
1225
1281
|
chomp($joboutput);
|
1282
|
+
# TODO: Ensure exit status == 0.
|
1226
1283
|
} else {
|
1227
1284
|
Log (undef, "timed out reading from 'arv-put'");
|
1228
1285
|
}
|
1229
1286
|
}
|
1287
|
+
# TODO: kill $pid instead of waiting, now that we've decided to
|
1288
|
+
# ignore further output.
|
1230
1289
|
waitpid($pid, 0);
|
1231
1290
|
|
1232
1291
|
return $joboutput;
|
@@ -1331,7 +1390,7 @@ sub log_writer_finish()
|
|
1331
1390
|
waitpid($log_pipe_pid, 0);
|
1332
1391
|
$log_pipe_pid = $log_pipe_in = $log_pipe_out = undef;
|
1333
1392
|
if ($?) {
|
1334
|
-
Log("log_writer_finish: arv-put
|
1393
|
+
Log("log_writer_finish: arv-put exited ".exit_status_s($?))
|
1335
1394
|
}
|
1336
1395
|
|
1337
1396
|
return $arv_put_output;
|
@@ -1374,7 +1433,7 @@ sub croak
|
|
1374
1433
|
Log (undef, $message);
|
1375
1434
|
freeze() if @jobstep_todo;
|
1376
1435
|
collate_output() if @jobstep_todo;
|
1377
|
-
cleanup();
|
1436
|
+
cleanup() if $Job;
|
1378
1437
|
save_meta() if log_writer_is_active();
|
1379
1438
|
die;
|
1380
1439
|
}
|
@@ -1382,7 +1441,6 @@ sub croak
|
|
1382
1441
|
|
1383
1442
|
sub cleanup
|
1384
1443
|
{
|
1385
|
-
return if !$job_has_uuid;
|
1386
1444
|
if ($Job->{'state'} eq 'Cancelled') {
|
1387
1445
|
$Job->update_attributes('finished_at' => scalar gmtime);
|
1388
1446
|
} else {
|
@@ -1399,7 +1457,7 @@ sub save_meta
|
|
1399
1457
|
my $loglocator = log_writer_finish();
|
1400
1458
|
Log (undef, "log manifest is $loglocator");
|
1401
1459
|
$Job->{'log'} = $loglocator;
|
1402
|
-
$Job->update_attributes('log', $loglocator)
|
1460
|
+
$Job->update_attributes('log', $loglocator);
|
1403
1461
|
}
|
1404
1462
|
|
1405
1463
|
|
@@ -1427,7 +1485,7 @@ sub freeze_if_want_freeze
|
|
1427
1485
|
collate_output();
|
1428
1486
|
cleanup();
|
1429
1487
|
save_meta();
|
1430
|
-
exit
|
1488
|
+
exit 1;
|
1431
1489
|
}
|
1432
1490
|
}
|
1433
1491
|
|
@@ -1551,6 +1609,20 @@ sub put_retry_count {
|
|
1551
1609
|
return ($retries > 3) ? $retries : 3;
|
1552
1610
|
}
|
1553
1611
|
|
1612
|
+
sub exit_status_s {
|
1613
|
+
# Given a $?, return a human-readable exit code string like "0" or
|
1614
|
+
# "1" or "0 with signal 1" or "1 with signal 11".
|
1615
|
+
my $exitcode = shift;
|
1616
|
+
my $s = $exitcode >> 8;
|
1617
|
+
if ($exitcode & 0x7f) {
|
1618
|
+
$s .= " with signal " . ($exitcode & 0x7f);
|
1619
|
+
}
|
1620
|
+
if ($exitcode & 0x80) {
|
1621
|
+
$s .= " with core dump";
|
1622
|
+
}
|
1623
|
+
return $s;
|
1624
|
+
}
|
1625
|
+
|
1554
1626
|
__DATA__
|
1555
1627
|
#!/usr/bin/perl
|
1556
1628
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: arvados-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.20141006212502
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Arvados Authors
|
@@ -178,7 +178,7 @@ dependencies:
|
|
178
178
|
- - "<"
|
179
179
|
- !ruby/object:Gem::Version
|
180
180
|
version: 1.0.0
|
181
|
-
description: Arvados command line tools, git commit
|
181
|
+
description: Arvados command line tools, git commit 38cc5c0a51657c6b60f3d3f32c566845988dfb6b
|
182
182
|
email: gem-dev@curoverse.com
|
183
183
|
executables:
|
184
184
|
- arv
|