arvados-cli 0.1.20140403152751 → 0.1.20140403155249
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/crunch-job +49 -89
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5ae48eceb6e1a15ee8b14c99aeb18680a103fe28
|
4
|
+
data.tar.gz: 0ff02e615a2a053a80a9671259e474854fdee58e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 33467b027edfdd6ef6ffb138b0a0868e6bc6a0b7f54909bd09022d10fa4f9f5900d4e70c26360fbcf5fe2e40aa6551165fe12c13d70714c2f3fcbc55017dca4c
|
7
|
+
data.tar.gz: 39fcee1939a25c2522bc27964676f55324a7b3579a702547740a392628b950a37dae06e0843e50dba1423a27c9705f61a7c302cf3158e47f5c67b7d811688e01
|
data/bin/crunch-job
CHANGED
@@ -77,9 +77,9 @@ use POSIX ':sys_wait_h';
|
|
77
77
|
use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
|
78
78
|
use Arvados;
|
79
79
|
use Getopt::Long;
|
80
|
-
use
|
81
|
-
use
|
82
|
-
use
|
80
|
+
use IPC::Open2;
|
81
|
+
use IO::Select;
|
82
|
+
use File::Temp;
|
83
83
|
use Fcntl ':flock';
|
84
84
|
|
85
85
|
$ENV{"TMPDIR"} ||= "/tmp";
|
@@ -175,11 +175,8 @@ else
|
|
175
175
|
}
|
176
176
|
$job_id = $Job->{'uuid'};
|
177
177
|
|
178
|
-
$
|
179
|
-
$
|
180
|
-
$metastream->name('.');
|
181
|
-
$metastream->write_start($job_id . '.log.txt');
|
182
|
-
|
178
|
+
my $keep_logfile = $job_id . '.log.txt';
|
179
|
+
my $local_logfile = File::Temp->new();
|
183
180
|
|
184
181
|
$Job->{'runtime_constraints'} ||= {};
|
185
182
|
$Job->{'runtime_constraints'}->{'max_tasks_per_node'} ||= 0;
|
@@ -761,7 +758,7 @@ if ($job_has_uuid) {
|
|
761
758
|
if ($Job->{'output'})
|
762
759
|
{
|
763
760
|
eval {
|
764
|
-
my $manifest_text =
|
761
|
+
my $manifest_text = `arv keep get \Q$Job->{'output'}\E`;
|
765
762
|
$arv->{'collections'}->{'create'}->execute('collection' => {
|
766
763
|
'uuid' => $Job->{'output'},
|
767
764
|
'manifest_text' => $manifest_text,
|
@@ -1071,12 +1068,24 @@ sub process_stderr
|
|
1071
1068
|
} split ("\n", $jobstep[$job]->{stderr});
|
1072
1069
|
}
|
1073
1070
|
|
1071
|
+
sub fetch_block
|
1072
|
+
{
|
1073
|
+
my $hash = shift;
|
1074
|
+
my ($keep, $child_out, $output_block);
|
1075
|
+
|
1076
|
+
my $cmd = "arv keep get \Q$hash\E";
|
1077
|
+
open($keep, '-|', $cmd) or die "fetch_block: $cmd: $!";
|
1078
|
+
sysread($keep, $output_block, 64 * 1024 * 1024);
|
1079
|
+
close $keep;
|
1080
|
+
return $output_block;
|
1081
|
+
}
|
1074
1082
|
|
1075
1083
|
sub collate_output
|
1076
1084
|
{
|
1077
|
-
my $whc = Warehouse->new;
|
1078
1085
|
Log (undef, "collate");
|
1079
|
-
|
1086
|
+
|
1087
|
+
my ($child_out, $child_in);
|
1088
|
+
my $pid = open2($child_out, $child_in, 'arv', 'keep', 'put', '--raw');
|
1080
1089
|
my $joboutput;
|
1081
1090
|
for (@jobstep)
|
1082
1091
|
{
|
@@ -1087,26 +1096,36 @@ sub collate_output
|
|
1087
1096
|
if ($output !~ /^[0-9a-f]{32}(\+\S+)*$/)
|
1088
1097
|
{
|
1089
1098
|
$output_in_keep ||= $output =~ / [0-9a-f]{32}\S*\+K/;
|
1090
|
-
$
|
1099
|
+
print $child_in $output;
|
1091
1100
|
}
|
1092
1101
|
elsif (@jobstep == 1)
|
1093
1102
|
{
|
1094
1103
|
$joboutput = $output;
|
1095
|
-
|
1104
|
+
last;
|
1096
1105
|
}
|
1097
|
-
elsif (defined (my $outblock =
|
1106
|
+
elsif (defined (my $outblock = fetch_block ($output)))
|
1098
1107
|
{
|
1099
1108
|
$output_in_keep ||= $outblock =~ / [0-9a-f]{32}\S*\+K/;
|
1100
|
-
$
|
1109
|
+
print $child_in $outblock;
|
1101
1110
|
}
|
1102
1111
|
else
|
1103
1112
|
{
|
1104
|
-
|
1105
|
-
$whc->write_data ("XXX fetch_block($output) failed: $errstr XXX\n");
|
1113
|
+
Log (undef, "XXX fetch_block($output) failed XXX");
|
1106
1114
|
$main::success = 0;
|
1107
1115
|
}
|
1108
1116
|
}
|
1109
|
-
$
|
1117
|
+
$child_in->close;
|
1118
|
+
|
1119
|
+
if (!defined $joboutput) {
|
1120
|
+
my $s = IO::Select->new($child_out);
|
1121
|
+
if ($s->can_read(120)) {
|
1122
|
+
sysread($child_out, $joboutput, 64 * 1024 * 1024);
|
1123
|
+
} else {
|
1124
|
+
Log (undef, "timed out reading from 'arv keep put'");
|
1125
|
+
}
|
1126
|
+
}
|
1127
|
+
waitpid($pid, 0);
|
1128
|
+
|
1110
1129
|
if ($joboutput)
|
1111
1130
|
{
|
1112
1131
|
Log (undef, "output $joboutput");
|
@@ -1181,8 +1200,9 @@ sub Log # ($jobstep_id, $logmessage)
|
|
1181
1200
|
}
|
1182
1201
|
print STDERR ((-t STDERR) ? ($datetime." ".$message) : $message);
|
1183
1202
|
|
1184
|
-
|
1185
|
-
|
1203
|
+
if ($metastream) {
|
1204
|
+
print $metastream $datetime . " " . $message;
|
1205
|
+
}
|
1186
1206
|
}
|
1187
1207
|
|
1188
1208
|
|
@@ -1211,16 +1231,15 @@ sub cleanup
|
|
1211
1231
|
sub save_meta
|
1212
1232
|
{
|
1213
1233
|
my $justcheckpoint = shift; # false if this will be the last meta saved
|
1214
|
-
|
1215
|
-
|
1216
|
-
$
|
1217
|
-
my $
|
1218
|
-
|
1219
|
-
$
|
1220
|
-
|
1221
|
-
|
1222
|
-
|
1223
|
-
undef $metastream if !$justcheckpoint; # otherwise Log() will try to use it
|
1234
|
+
return if $justcheckpoint; # checkpointing is not relevant post-Warehouse.pm
|
1235
|
+
|
1236
|
+
$local_logfile->flush;
|
1237
|
+
my $cmd = "arv keep put --filename \Q$keep_logfile\E "
|
1238
|
+
. quotemeta($local_logfile->filename);
|
1239
|
+
my $loglocator = `$cmd`;
|
1240
|
+
die "system $cmd failed: $?" if $?;
|
1241
|
+
|
1242
|
+
$local_logfile = undef; # the temp file is automatically deleted
|
1224
1243
|
Log (undef, "log manifest is $loglocator");
|
1225
1244
|
$Job->{'log'} = $loglocator;
|
1226
1245
|
$Job->update_attributes('log', $loglocator) if $job_has_uuid;
|
@@ -1266,65 +1285,6 @@ sub freeze
|
|
1266
1285
|
sub thaw
|
1267
1286
|
{
|
1268
1287
|
croak ("Thaw not implemented");
|
1269
|
-
|
1270
|
-
my $whc;
|
1271
|
-
my $key = shift;
|
1272
|
-
Log (undef, "thaw from $key");
|
1273
|
-
|
1274
|
-
@jobstep = ();
|
1275
|
-
@jobstep_done = ();
|
1276
|
-
@jobstep_todo = ();
|
1277
|
-
@jobstep_tomerge = ();
|
1278
|
-
$jobstep_tomerge_level = 0;
|
1279
|
-
my $frozenjob = {};
|
1280
|
-
|
1281
|
-
my $stream = new Warehouse::Stream ( whc => $whc,
|
1282
|
-
hash => [split (",", $key)] );
|
1283
|
-
$stream->rewind;
|
1284
|
-
while (my $dataref = $stream->read_until (undef, "\n\n"))
|
1285
|
-
{
|
1286
|
-
if ($$dataref =~ /^job /)
|
1287
|
-
{
|
1288
|
-
foreach (split ("\n", $$dataref))
|
1289
|
-
{
|
1290
|
-
my ($k, $v) = split ("=", $_, 2);
|
1291
|
-
$frozenjob->{$k} = freezeunquote ($v);
|
1292
|
-
}
|
1293
|
-
next;
|
1294
|
-
}
|
1295
|
-
|
1296
|
-
if ($$dataref =~ /^merge (\d+) (.*)/)
|
1297
|
-
{
|
1298
|
-
$jobstep_tomerge_level = $1;
|
1299
|
-
@jobstep_tomerge
|
1300
|
-
= map { freezeunquote ($_) } split ("\n", freezeunquote($2));
|
1301
|
-
next;
|
1302
|
-
}
|
1303
|
-
|
1304
|
-
my $Jobstep = { };
|
1305
|
-
foreach (split ("\n", $$dataref))
|
1306
|
-
{
|
1307
|
-
my ($k, $v) = split ("=", $_, 2);
|
1308
|
-
$Jobstep->{$k} = freezeunquote ($v) if $k;
|
1309
|
-
}
|
1310
|
-
$Jobstep->{'failures'} = 0;
|
1311
|
-
push @jobstep, $Jobstep;
|
1312
|
-
|
1313
|
-
if ($Jobstep->{exitcode} eq "0")
|
1314
|
-
{
|
1315
|
-
push @jobstep_done, $#jobstep;
|
1316
|
-
}
|
1317
|
-
else
|
1318
|
-
{
|
1319
|
-
push @jobstep_todo, $#jobstep;
|
1320
|
-
}
|
1321
|
-
}
|
1322
|
-
|
1323
|
-
foreach (qw (script script_version script_parameters))
|
1324
|
-
{
|
1325
|
-
$Job->{$_} = $frozenjob->{$_};
|
1326
|
-
}
|
1327
|
-
$Job->save if $job_has_uuid;
|
1328
1288
|
}
|
1329
1289
|
|
1330
1290
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: arvados-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.20140403155249
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Arvados Authors
|
@@ -146,7 +146,7 @@ dependencies:
|
|
146
146
|
- - "~>"
|
147
147
|
- !ruby/object:Gem::Version
|
148
148
|
version: '0.8'
|
149
|
-
description: This is the Arvados SDK CLI gem, git revision
|
149
|
+
description: This is the Arvados SDK CLI gem, git revision 39717c4347c00ee0697942d60a50000bfc89fe3c
|
150
150
|
email: gem-dev@curoverse.com
|
151
151
|
executables:
|
152
152
|
- arv
|