arvados-cli 0.1.20141201230227 → 0.1.20141202211726
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/crunch-job +31 -44
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: dabc20ac76a73f07613314a0727aa242db29fb50
|
4
|
+
data.tar.gz: b9664decff16341a3dcbcc184a800f0958184a2e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0e3e255254c7cd13a1f4f3827d3443921dc8e4bf6cb32ebfdf5479ebe323cdad5d24e6bbfc2701c3a9bd9c9d975272912c883ae67c59a8ac15758b8f556e98d4
|
7
|
+
data.tar.gz: 7ded50195dfa5e440817680a6214d7fdad31831e51d7460177c0db63788991b79e178cbcd703d25e623e453703969b5d40055237fde2463237199a841b7d1009
|
data/bin/crunch-job
CHANGED
@@ -305,7 +305,6 @@ my @jobstep_tomerge = ();
|
|
305
305
|
my $jobstep_tomerge_level = 0;
|
306
306
|
my $squeue_checked;
|
307
307
|
my $squeue_kill_checked;
|
308
|
-
my $output_in_keep = 0;
|
309
308
|
my $latest_refresh = scalar time;
|
310
309
|
|
311
310
|
|
@@ -823,7 +822,7 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
|
|
823
822
|
{
|
824
823
|
$main::please_info = 0;
|
825
824
|
freeze();
|
826
|
-
|
825
|
+
create_output_collection();
|
827
826
|
save_meta(1);
|
828
827
|
update_progress_stats();
|
829
828
|
}
|
@@ -885,7 +884,7 @@ while (%proc)
|
|
885
884
|
$main::please_continue = 0;
|
886
885
|
goto THISROUND;
|
887
886
|
}
|
888
|
-
$main::please_info = 0, freeze(),
|
887
|
+
$main::please_info = 0, freeze(), create_output_collection(), save_meta(1) if $main::please_info;
|
889
888
|
readfrompipes ();
|
890
889
|
if (!reapchildren())
|
891
890
|
{
|
@@ -922,28 +921,14 @@ goto ONELEVEL if !defined $main::success;
|
|
922
921
|
|
923
922
|
release_allocation();
|
924
923
|
freeze();
|
925
|
-
my $collated_output = &
|
924
|
+
my $collated_output = &create_output_collection();
|
926
925
|
|
927
926
|
if (!$collated_output) {
|
928
|
-
Log(undef, "output
|
927
|
+
Log (undef, "Failed to write output collection");
|
929
928
|
}
|
930
929
|
else {
|
931
|
-
|
932
|
-
|
933
|
-
or die "failed to get collated manifest: $!";
|
934
|
-
my $orig_manifest_text = '';
|
935
|
-
while (my $manifest_line = <$orig_manifest>) {
|
936
|
-
$orig_manifest_text .= $manifest_line;
|
937
|
-
}
|
938
|
-
my $output = api_call("collections/create", collection => {
|
939
|
-
'manifest_text' => $orig_manifest_text});
|
940
|
-
Log(undef, "output uuid " . $output->{uuid});
|
941
|
-
Log(undef, "output hash " . $output->{portable_data_hash});
|
942
|
-
$Job->update_attributes('output' => $output->{portable_data_hash});
|
943
|
-
};
|
944
|
-
if ($@) {
|
945
|
-
Log (undef, "Failed to register output manifest: $@");
|
946
|
-
}
|
930
|
+
Log(undef, "output hash " . $collated_output);
|
931
|
+
$Job->update_attributes('output' => $collated_output);
|
947
932
|
}
|
948
933
|
|
949
934
|
Log (undef, "finish");
|
@@ -1275,14 +1260,24 @@ sub fetch_block
|
|
1275
1260
|
return $output_block;
|
1276
1261
|
}
|
1277
1262
|
|
1278
|
-
|
1263
|
+
# create_output_collections generates a new collection containing the
|
1264
|
+
# output of each successfully completed task, and returns the
|
1265
|
+
# portable_data_hash for the new collection.
|
1266
|
+
#
|
1267
|
+
sub create_output_collection
|
1279
1268
|
{
|
1280
1269
|
Log (undef, "collate");
|
1281
1270
|
|
1282
1271
|
my ($child_out, $child_in);
|
1283
|
-
my $pid = open2($child_out, $child_in, '
|
1284
|
-
'
|
1285
|
-
|
1272
|
+
my $pid = open2($child_out, $child_in, 'python', '-c',
|
1273
|
+
'import arvados; ' .
|
1274
|
+
'import sys; ' .
|
1275
|
+
'print arvados.api()' .
|
1276
|
+
'.collections()' .
|
1277
|
+
'.create(body={"manifest_text":sys.stdin.read()})' .
|
1278
|
+
'.execute()["portable_data_hash"]'
|
1279
|
+
);
|
1280
|
+
|
1286
1281
|
for (@jobstep)
|
1287
1282
|
{
|
1288
1283
|
next if (!exists $_->{'arvados_task'}->{'output'} ||
|
@@ -1290,17 +1285,10 @@ sub collate_output
|
|
1290
1285
|
my $output = $_->{'arvados_task'}->{output};
|
1291
1286
|
if ($output !~ /^[0-9a-f]{32}(\+\S+)*$/)
|
1292
1287
|
{
|
1293
|
-
$output_in_keep ||= $output =~ / [0-9a-f]{32}\S*\+K/;
|
1294
1288
|
print $child_in $output;
|
1295
1289
|
}
|
1296
|
-
elsif (@jobstep == 1)
|
1297
|
-
{
|
1298
|
-
$joboutput = $output;
|
1299
|
-
last;
|
1300
|
-
}
|
1301
1290
|
elsif (defined (my $outblock = fetch_block ($output)))
|
1302
1291
|
{
|
1303
|
-
$output_in_keep ||= $outblock =~ / [0-9a-f]{32}\S*\+K/;
|
1304
1292
|
print $child_in $outblock;
|
1305
1293
|
}
|
1306
1294
|
else
|
@@ -1311,15 +1299,14 @@ sub collate_output
|
|
1311
1299
|
}
|
1312
1300
|
$child_in->close;
|
1313
1301
|
|
1314
|
-
|
1315
|
-
|
1316
|
-
|
1317
|
-
|
1318
|
-
|
1319
|
-
|
1320
|
-
|
1321
|
-
|
1322
|
-
}
|
1302
|
+
my $joboutput;
|
1303
|
+
my $s = IO::Select->new($child_out);
|
1304
|
+
if ($s->can_read(120)) {
|
1305
|
+
sysread($child_out, $joboutput, 64 * 1024 * 1024);
|
1306
|
+
chomp($joboutput);
|
1307
|
+
# TODO: Ensure exit status == 0.
|
1308
|
+
} else {
|
1309
|
+
Log (undef, "timed out while creating output collection");
|
1323
1310
|
}
|
1324
1311
|
# TODO: kill $pid instead of waiting, now that we've decided to
|
1325
1312
|
# ignore further output.
|
@@ -1469,7 +1456,7 @@ sub croak
|
|
1469
1456
|
my $message = "@_ at $file line $line\n";
|
1470
1457
|
Log (undef, $message);
|
1471
1458
|
freeze() if @jobstep_todo;
|
1472
|
-
|
1459
|
+
create_output_collection() if @jobstep_todo;
|
1473
1460
|
cleanup();
|
1474
1461
|
save_meta();
|
1475
1462
|
die;
|
@@ -1521,7 +1508,7 @@ sub freeze_if_want_freeze
|
|
1521
1508
|
}
|
1522
1509
|
}
|
1523
1510
|
freeze();
|
1524
|
-
|
1511
|
+
create_output_collection();
|
1525
1512
|
cleanup();
|
1526
1513
|
save_meta();
|
1527
1514
|
exit 1;
|
@@ -1702,7 +1689,7 @@ sub api_call {
|
|
1702
1689
|
if ($next_try_at < time) {
|
1703
1690
|
$retry_msg = "Retrying.";
|
1704
1691
|
} else {
|
1705
|
-
my $next_try_fmt = strftime
|
1692
|
+
my $next_try_fmt = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime($next_try_at);
|
1706
1693
|
$retry_msg = "Retrying at $next_try_fmt.";
|
1707
1694
|
}
|
1708
1695
|
Log(undef, "API method $method_name failed: $errmsg. $retry_msg");
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: arvados-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.20141202211726
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Arvados Authors
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-12-
|
11
|
+
date: 2014-12-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: arvados
|
@@ -178,7 +178,7 @@ dependencies:
|
|
178
178
|
- - "<"
|
179
179
|
- !ruby/object:Gem::Version
|
180
180
|
version: 1.0.0
|
181
|
-
description: Arvados command line tools, git commit
|
181
|
+
description: Arvados command line tools, git commit 0ecbb08536d02433272e62cf1878edd7927f66e1
|
182
182
|
email: gem-dev@curoverse.com
|
183
183
|
executables:
|
184
184
|
- arv
|