arvados-cli 0.1.20141104173855 → 0.1.20141105163214
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/arv +4 -6
- data/bin/crunch-job +57 -42
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f8c810a4117e76ddb95388c08ad927b7baeb5392
|
4
|
+
data.tar.gz: b1ff94d1fd2f492ac84b70b38e1996ec851ab1e5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9ab91c019570a4010b692e7fa30e3cf97c9b7645512bf4402e74d8f1de60a9426de18763386e7b1e92d476e1e46aea0f00b8d5c619e3992e9f900fdb94ebba47
|
7
|
+
data.tar.gz: fd906ce3d8705a0bd57091f8b0e0f680b54b6bcd17fd4382cc0d3c4332f5077502fefafcb09d1cfe1f31c43202cc244fa22c2e797fa13ffae141a45ddea36a78
|
data/bin/arv
CHANGED
@@ -113,7 +113,7 @@ def init_config
|
|
113
113
|
end
|
114
114
|
end
|
115
115
|
|
116
|
-
subcommands = %w(create edit keep pipeline tag ws)
|
116
|
+
subcommands = %w(copy create edit keep pipeline tag ws)
|
117
117
|
|
118
118
|
def check_subcommands client, arvados, subcommand, global_opts, remaining_opts
|
119
119
|
case subcommand
|
@@ -121,9 +121,11 @@ def check_subcommands client, arvados, subcommand, global_opts, remaining_opts
|
|
121
121
|
arv_create client, arvados, global_opts, remaining_opts
|
122
122
|
when 'edit'
|
123
123
|
arv_edit client, arvados, global_opts, remaining_opts
|
124
|
+
when 'copy', 'tag', 'ws'
|
125
|
+
exec `which arv-#{subcommand}`.strip, *remaining_opts
|
124
126
|
when 'keep'
|
125
127
|
@sub = remaining_opts.shift
|
126
|
-
if ['get', 'put', 'ls', 'normalize'
|
128
|
+
if ['get', 'put', 'ls', 'normalize'].index @sub then
|
127
129
|
# Native Arvados
|
128
130
|
exec `which arv-#{@sub}`.strip, *remaining_opts
|
129
131
|
elsif ['less', 'check'].index @sub then
|
@@ -147,10 +149,6 @@ def check_subcommands client, arvados, subcommand, global_opts, remaining_opts
|
|
147
149
|
puts "Available methods: run"
|
148
150
|
end
|
149
151
|
abort
|
150
|
-
when 'tag'
|
151
|
-
exec `which arv-tag`.strip, *remaining_opts
|
152
|
-
when 'ws'
|
153
|
-
exec `which arv-ws`.strip, *remaining_opts
|
154
152
|
end
|
155
153
|
end
|
156
154
|
|
data/bin/crunch-job
CHANGED
@@ -157,20 +157,15 @@ my $dbh;
|
|
157
157
|
my $sth;
|
158
158
|
my @jobstep;
|
159
159
|
|
160
|
-
my $User =
|
160
|
+
my $User = api_call("users/current");
|
161
161
|
|
162
162
|
if ($jobspec =~ /^[-a-z\d]+$/)
|
163
163
|
{
|
164
164
|
# $jobspec is an Arvados UUID, not a JSON job specification
|
165
|
-
$Job =
|
166
|
-
$arv->{'jobs'}->{'get'}->execute('uuid' => $jobspec);
|
167
|
-
});
|
165
|
+
$Job = api_call("jobs/get", uuid => $jobspec);
|
168
166
|
if (!$force_unlock) {
|
169
167
|
# Claim this job, and make sure nobody else does
|
170
|
-
eval {
|
171
|
-
# lock() sets is_locked_by_uuid and changes state to Running.
|
172
|
-
$arv->{'jobs'}->{'lock'}->execute('uuid' => $Job->{'uuid'})
|
173
|
-
}); };
|
168
|
+
eval { api_call("jobs/lock", uuid => $Job->{uuid}); };
|
174
169
|
if ($@) {
|
175
170
|
Log(undef, "Error while locking job, exiting ".EX_TEMPFAIL);
|
176
171
|
exit EX_TEMPFAIL;
|
@@ -191,7 +186,7 @@ else
|
|
191
186
|
$Job->{'started_at'} = gmtime;
|
192
187
|
$Job->{'state'} = 'Running';
|
193
188
|
|
194
|
-
$Job =
|
189
|
+
$Job = api_call("jobs/create", job => $Job);
|
195
190
|
}
|
196
191
|
$job_id = $Job->{'uuid'};
|
197
192
|
|
@@ -321,13 +316,11 @@ if (defined $Job->{thawedfromkey})
|
|
321
316
|
}
|
322
317
|
else
|
323
318
|
{
|
324
|
-
my $first_task =
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
'parameters' => {},
|
330
|
-
});
|
319
|
+
my $first_task = api_call("job_tasks/create", job_task => {
|
320
|
+
'job_uuid' => $Job->{'uuid'},
|
321
|
+
'sequence' => 0,
|
322
|
+
'qsequence' => 0,
|
323
|
+
'parameters' => {},
|
331
324
|
});
|
332
325
|
push @jobstep, { 'level' => 0,
|
333
326
|
'failures' => 0,
|
@@ -427,10 +420,8 @@ else {
|
|
427
420
|
} else {
|
428
421
|
# $repo is none of the above. It must be the name of a hosted
|
429
422
|
# repository.
|
430
|
-
my $arv_repo_list =
|
431
|
-
|
432
|
-
'filters' => [['name','=',$repo]]);
|
433
|
-
});
|
423
|
+
my $arv_repo_list = api_call("repositories/list",
|
424
|
+
'filters' => [['name','=',$repo]]);
|
434
425
|
my @repos_found = @{$arv_repo_list->{'items'}};
|
435
426
|
my $n_found = $arv_repo_list->{'serverResponse'}->{'items_available'};
|
436
427
|
if ($n_found > 0) {
|
@@ -935,10 +926,8 @@ else {
|
|
935
926
|
while (my $manifest_line = <$orig_manifest>) {
|
936
927
|
$orig_manifest_text .= $manifest_line;
|
937
928
|
}
|
938
|
-
my $output =
|
939
|
-
|
940
|
-
'collection' => {'manifest_text' => $orig_manifest_text});
|
941
|
-
});
|
929
|
+
my $output = api_call("collections/create", collection => {
|
930
|
+
'manifest_text' => $orig_manifest_text});
|
942
931
|
Log(undef, "output uuid " . $output->{uuid});
|
943
932
|
Log(undef, "output hash " . $output->{portable_data_hash});
|
944
933
|
$Job->update_attributes('output' => $output->{portable_data_hash});
|
@@ -1072,15 +1061,14 @@ sub reapchildren
|
|
1072
1061
|
my $newtask_list = [];
|
1073
1062
|
my $newtask_results;
|
1074
1063
|
do {
|
1075
|
-
$newtask_results =
|
1076
|
-
|
1077
|
-
|
1078
|
-
|
1079
|
-
|
1080
|
-
|
1081
|
-
|
1082
|
-
|
1083
|
-
});
|
1064
|
+
$newtask_results = api_call(
|
1065
|
+
"job_tasks/list",
|
1066
|
+
'where' => {
|
1067
|
+
'created_by_job_task_uuid' => $Jobstep->{'arvados_task'}->{uuid}
|
1068
|
+
},
|
1069
|
+
'order' => 'qsequence',
|
1070
|
+
'offset' => scalar(@$newtask_list),
|
1071
|
+
);
|
1084
1072
|
push(@$newtask_list, @{$newtask_results->{items}});
|
1085
1073
|
} while (@{$newtask_results->{items}});
|
1086
1074
|
foreach my $arvados_task (@$newtask_list) {
|
@@ -1103,9 +1091,7 @@ sub check_refresh_wanted
|
|
1103
1091
|
my @stat = stat $ENV{"CRUNCH_REFRESH_TRIGGER"};
|
1104
1092
|
if (@stat && $stat[9] > $latest_refresh) {
|
1105
1093
|
$latest_refresh = scalar time;
|
1106
|
-
my $Job2 =
|
1107
|
-
$arv->{'jobs'}->{'get'}->execute('uuid' => $jobspec);
|
1108
|
-
});
|
1094
|
+
my $Job2 = api_call("jobs/get", uuid => $jobspec);
|
1109
1095
|
for my $attr ('cancelled_at',
|
1110
1096
|
'cancelled_by_user_uuid',
|
1111
1097
|
'cancelled_by_client_uuid',
|
@@ -1620,9 +1606,7 @@ sub find_docker_image {
|
|
1620
1606
|
# If not, return undef for both values.
|
1621
1607
|
my $locator = shift;
|
1622
1608
|
my ($streamname, $filename);
|
1623
|
-
my $image =
|
1624
|
-
$arv->{collections}->{get}->execute(uuid => $locator);
|
1625
|
-
});
|
1609
|
+
my $image = api_call("collections/get", uuid => $locator);
|
1626
1610
|
if ($image) {
|
1627
1611
|
foreach my $line (split(/\n/, $image->{manifest_text})) {
|
1628
1612
|
my @tokens = split(/\s+/, $line);
|
@@ -1669,10 +1653,14 @@ sub retry_count {
|
|
1669
1653
|
}
|
1670
1654
|
|
1671
1655
|
sub retry_op {
|
1672
|
-
#
|
1673
|
-
#
|
1674
|
-
#
|
1656
|
+
# Pass in two function references.
|
1657
|
+
# This method will be called with the remaining arguments.
|
1658
|
+
# If it dies, retry it with exponential backoff until it succeeds,
|
1659
|
+
# or until the current retry_count is exhausted. After each failure
|
1660
|
+
# that can be retried, the second function will be called with
|
1661
|
+
# the current try count (0-based), next try time, and error message.
|
1675
1662
|
my $operation = shift;
|
1663
|
+
my $retry_callback = shift;
|
1676
1664
|
my $retries = retry_count();
|
1677
1665
|
foreach my $try_count (0..$retries) {
|
1678
1666
|
my $next_try = time + (2 ** $try_count);
|
@@ -1680,6 +1668,7 @@ sub retry_op {
|
|
1680
1668
|
if (!$@) {
|
1681
1669
|
return $result;
|
1682
1670
|
} elsif ($try_count < $retries) {
|
1671
|
+
$retry_callback->($try_count, $next_try, $@);
|
1683
1672
|
my $sleep_time = $next_try - time;
|
1684
1673
|
sleep($sleep_time) if ($sleep_time > 0);
|
1685
1674
|
}
|
@@ -1690,6 +1679,32 @@ sub retry_op {
|
|
1690
1679
|
die($@ . "\n");
|
1691
1680
|
}
|
1692
1681
|
|
1682
|
+
sub api_call {
|
1683
|
+
# Pass in a /-separated API method name, and arguments for it.
|
1684
|
+
# This function will call that method, retrying as needed until
|
1685
|
+
# the current retry_count is exhausted, with a log on the first failure.
|
1686
|
+
my $method_name = shift;
|
1687
|
+
my $log_api_retry = sub {
|
1688
|
+
my ($try_count, $next_try_at, $errmsg) = @_;
|
1689
|
+
$errmsg =~ s/\s*\bat \Q$0\E line \d+\.?\s*//;
|
1690
|
+
$errmsg =~ s/\s/ /g;
|
1691
|
+
$errmsg =~ s/\s+$//;
|
1692
|
+
my $retry_msg;
|
1693
|
+
if ($next_try_at < time) {
|
1694
|
+
$retry_msg = "Retrying.";
|
1695
|
+
} else {
|
1696
|
+
my $next_try_fmt = strftime("%Y-%m-%d %H:%M:%S", $next_try_at);
|
1697
|
+
$retry_msg = "Retrying at $next_try_fmt.";
|
1698
|
+
}
|
1699
|
+
Log(undef, "API method $method_name failed: $errmsg. $retry_msg");
|
1700
|
+
};
|
1701
|
+
my $method = $arv;
|
1702
|
+
foreach my $key (split(/\//, $method_name)) {
|
1703
|
+
$method = $method->{$key};
|
1704
|
+
}
|
1705
|
+
return retry_op(sub { $method->execute(@_); }, $log_api_retry, @_);
|
1706
|
+
}
|
1707
|
+
|
1693
1708
|
sub exit_status_s {
|
1694
1709
|
# Given a $?, return a human-readable exit code string like "0" or
|
1695
1710
|
# "1" or "0 with signal 1" or "1 with signal 11".
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: arvados-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.20141105163214
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Arvados Authors
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-11-
|
11
|
+
date: 2014-11-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: arvados
|
@@ -178,7 +178,7 @@ dependencies:
|
|
178
178
|
- - "<"
|
179
179
|
- !ruby/object:Gem::Version
|
180
180
|
version: 1.0.0
|
181
|
-
description: Arvados command line tools, git commit
|
181
|
+
description: Arvados command line tools, git commit f91dff0f2f49a83047e31b1a56dbddd85be6926a
|
182
182
|
email: gem-dev@curoverse.com
|
183
183
|
executables:
|
184
184
|
- arv
|