arvados-cli 0.1.20141007134429 → 0.1.20141007160434
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/crunch-job +33 -16
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9e07192a0e91503a3e234029df54195bbb24046d
|
4
|
+
data.tar.gz: b565a3c5de08ce709331106b97d03a55c542a8b1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0616516ef880b98b938eeef26741d0f25e6e116c0eb7fcec03db94b3d2f7139826398127c540d130c2009beb73ecec14db679ada5f82557569b1171860a8b035
|
7
|
+
data.tar.gz: 126b2996d4a63be3cc4cadc02f6277ca4c9e87b554d81dbe62a71d4c6940e448cd48b667ec77934fc93e4848d6baf4124a6bf17be4613f0a41e84f296c2b0605
|
data/bin/crunch-job
CHANGED
@@ -10,12 +10,14 @@ crunch-job: Execute job steps, save snapshots as requested, collate output.
|
|
10
10
|
Obtain job details from Arvados, run tasks on compute nodes (typically
|
11
11
|
invoked by scheduler on controller):
|
12
12
|
|
13
|
-
crunch-job --job x-y-z
|
13
|
+
crunch-job --job x-y-z --git-dir /path/to/repo/.git
|
14
14
|
|
15
15
|
Obtain job details from command line, run tasks on local machine
|
16
16
|
(typically invoked by application or developer on VM):
|
17
17
|
|
18
|
-
crunch-job --job '{"script_version":"/path/to/tree","script":"scriptname",...}'
|
18
|
+
crunch-job --job '{"script_version":"/path/to/working/tree","script":"scriptname",...}'
|
19
|
+
|
20
|
+
crunch-job --job '{"repository":"https://github.com/curoverse/arvados.git","script_version":"master","script":"scriptname",...}'
|
19
21
|
|
20
22
|
=head1 OPTIONS
|
21
23
|
|
@@ -27,7 +29,9 @@ If the job is already locked, steal the lock and run it anyway.
|
|
27
29
|
|
28
30
|
=item --git-dir
|
29
31
|
|
30
|
-
Path to .git directory where the
|
32
|
+
Path to a .git directory (or a git URL) where the commit given in the
|
33
|
+
job's C<script_version> attribute is to be found. If this is I<not>
|
34
|
+
given, the job's C<repository> attribute will be used.
|
31
35
|
|
32
36
|
=item --job-api-token
|
33
37
|
|
@@ -39,6 +43,11 @@ Do not clear per-job/task temporary directories during initial job
|
|
39
43
|
setup. This can speed up development and debugging when running jobs
|
40
44
|
locally.
|
41
45
|
|
46
|
+
=item --job
|
47
|
+
|
48
|
+
UUID of the job to run, or a JSON-encoded job resource without a
|
49
|
+
UUID. If the latter is given, a new job object will be created.
|
50
|
+
|
42
51
|
=back
|
43
52
|
|
44
53
|
=head1 RUNNING JOBS LOCALLY
|
@@ -363,17 +372,21 @@ if (!defined $no_clear_tmp) {
|
|
363
372
|
|
364
373
|
my $git_archive;
|
365
374
|
if (!defined $git_dir && $Job->{'script_version'} =~ m{^/}) {
|
366
|
-
# If
|
367
|
-
#
|
368
|
-
#
|
369
|
-
# commit
|
375
|
+
# If script_version looks like an absolute path, *and* the --git-dir
|
376
|
+
# argument was not given -- which implies we were not invoked by
|
377
|
+
# crunch-dispatch -- we will use the given path as a working
|
378
|
+
# directory instead of resolving script_version to a git commit (or
|
379
|
+
# doing anything else with git).
|
370
380
|
$ENV{"CRUNCH_SRC_COMMIT"} = $Job->{'script_version'};
|
371
381
|
$ENV{"CRUNCH_SRC"} = $Job->{'script_version'};
|
372
382
|
}
|
373
383
|
else {
|
384
|
+
# Resolve the given script_version to a git commit sha1. Also, if
|
385
|
+
# the repository is remote, clone it into our local filesystem: this
|
386
|
+
# ensures "git archive" will work, and is necessary to reliably
|
387
|
+
# resolve a symbolic script_version like "master^".
|
374
388
|
$ENV{"CRUNCH_SRC"} = "$ENV{CRUNCH_TMP}/src";
|
375
389
|
|
376
|
-
# Install requested code version
|
377
390
|
Log (undef, "Looking for version ".$Job->{script_version}." from repository ".$Job->{repository});
|
378
391
|
|
379
392
|
$ENV{"CRUNCH_SRC_COMMIT"} = $Job->{script_version};
|
@@ -415,17 +428,18 @@ else {
|
|
415
428
|
# repository.
|
416
429
|
my $arv_repo_list = retry_op(sub {
|
417
430
|
$arv->{'repositories'}->{'list'}->execute(
|
418
|
-
'filters' => [['name','=',$repo]])
|
431
|
+
'filters' => [['name','=',$repo]]);
|
419
432
|
});
|
420
|
-
my
|
433
|
+
my @repos_found = @{$arv_repo_list->{'items'}};
|
434
|
+
my $n_found = $arv_repo_list->{'serverResponse'}->{'items_available'};
|
421
435
|
if ($n_found > 0) {
|
422
436
|
Log(undef, "Repository '$repo' -> "
|
423
|
-
. join(", ", map { $_->{'uuid'} } @
|
437
|
+
. join(", ", map { $_->{'uuid'} } @repos_found));
|
424
438
|
}
|
425
439
|
if ($n_found != 1) {
|
426
440
|
croak("Error: Found $n_found repositories with name '$repo'.");
|
427
441
|
}
|
428
|
-
$repo = $
|
442
|
+
$repo = $repos_found[0]->{'fetch_url'};
|
429
443
|
$repo_location = 'remote';
|
430
444
|
}
|
431
445
|
Log(undef, "Using $repo_location repository '$repo'");
|
@@ -458,11 +472,12 @@ else {
|
|
458
472
|
# our local cache first, since that's cheaper. (We don't want to
|
459
473
|
# do that with tags/branches though -- those change over time, so
|
460
474
|
# they should always be resolved by the remote repo.)
|
461
|
-
if ($treeish =~ /^[0-9a-f]{
|
475
|
+
if ($treeish =~ /^[0-9a-f]{7,40}$/s) {
|
462
476
|
# Hide stderr because it's normal for this to fail:
|
463
477
|
my $sha1 = `$gitcmd rev-list -n1 ''\Q$treeish\E 2>/dev/null`;
|
464
478
|
if ($? == 0 &&
|
465
|
-
|
479
|
+
# Careful not to resolve a branch named abcdeff to commit 1234567:
|
480
|
+
$sha1 =~ /^$treeish/ &&
|
466
481
|
$sha1 =~ /^([0-9a-f]{40})$/s) {
|
467
482
|
$commit = $1;
|
468
483
|
Log(undef, "Commit $commit already present in $local_repo");
|
@@ -1444,14 +1459,15 @@ sub croak
|
|
1444
1459
|
Log (undef, $message);
|
1445
1460
|
freeze() if @jobstep_todo;
|
1446
1461
|
collate_output() if @jobstep_todo;
|
1447
|
-
cleanup()
|
1448
|
-
save_meta()
|
1462
|
+
cleanup();
|
1463
|
+
save_meta();
|
1449
1464
|
die;
|
1450
1465
|
}
|
1451
1466
|
|
1452
1467
|
|
1453
1468
|
sub cleanup
|
1454
1469
|
{
|
1470
|
+
return unless $Job;
|
1455
1471
|
if ($Job->{'state'} eq 'Cancelled') {
|
1456
1472
|
$Job->update_attributes('finished_at' => scalar gmtime);
|
1457
1473
|
} else {
|
@@ -1464,6 +1480,7 @@ sub save_meta
|
|
1464
1480
|
{
|
1465
1481
|
my $justcheckpoint = shift; # false if this will be the last meta saved
|
1466
1482
|
return if $justcheckpoint; # checkpointing is not relevant post-Warehouse.pm
|
1483
|
+
return unless log_writer_is_active();
|
1467
1484
|
|
1468
1485
|
my $loglocator = log_writer_finish();
|
1469
1486
|
Log (undef, "log manifest is $loglocator");
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: arvados-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.20141007160434
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Arvados Authors
|
@@ -178,7 +178,7 @@ dependencies:
|
|
178
178
|
- - "<"
|
179
179
|
- !ruby/object:Gem::Version
|
180
180
|
version: 1.0.0
|
181
|
-
description: Arvados command line tools, git commit
|
181
|
+
description: Arvados command line tools, git commit dd7bb176c565d0d0718f9b0e59a6d9ee4b8ecbf2
|
182
182
|
email: gem-dev@curoverse.com
|
183
183
|
executables:
|
184
184
|
- arv
|