arvados-cli 0.1.20141112222031 → 0.1.20141117160514
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/crunch-job +52 -39
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a2ed5674e54a294dc0ddb8a5eafd76fa389b5626
|
4
|
+
data.tar.gz: fa23b195a238b3bbbc5591665b6cc72c8c3f2dff
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ee0649cd95545b5e301e334eb878bc7c2ceb93c3db44f109a974b2bb44f6db057ad18648cf6c9cbb2c4e4f68eb157a9b9bf358aed69b93338c96991ca30bdd70
|
7
|
+
data.tar.gz: 1b62f0ebf2871c79abc820a0270a47a386fbddb67ba47f047a1fcb98ef9f98019cea7bc716b768a5d2a884e62fc2362979e2b525c4f91e9b110bd48a5dac1e1c
|
data/bin/crunch-job
CHANGED
@@ -551,7 +551,9 @@ else {
|
|
551
551
|
freeze_if_want_freeze ($installpid);
|
552
552
|
select (undef, undef, undef, 0.1);
|
553
553
|
}
|
554
|
-
|
554
|
+
my $install_exited = $?;
|
555
|
+
Log (undef, "Install script exited ".exit_status_s($install_exited));
|
556
|
+
exit (1) if $install_exited != 0;
|
555
557
|
}
|
556
558
|
|
557
559
|
if (!$have_slurm)
|
@@ -699,17 +701,10 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
|
|
699
701
|
qw(-n1 -c1 -N1 -D), $ENV{'TMPDIR'},
|
700
702
|
"--job-name=$job_id.$id.$$",
|
701
703
|
);
|
702
|
-
my $build_script_to_send = "";
|
703
704
|
my $command =
|
704
705
|
"if [ -e $ENV{TASK_WORK} ]; then rm -rf $ENV{TASK_WORK}; fi; "
|
705
706
|
."mkdir -p $ENV{CRUNCH_TMP} $ENV{JOB_WORK} $ENV{TASK_WORK} $ENV{TASK_KEEPMOUNT} "
|
706
707
|
."&& cd $ENV{CRUNCH_TMP} ";
|
707
|
-
if ($build_script)
|
708
|
-
{
|
709
|
-
$build_script_to_send = $build_script;
|
710
|
-
$command .=
|
711
|
-
"&& perl -";
|
712
|
-
}
|
713
708
|
$command .= "&& exec arv-mount --by-id --allow-other $ENV{TASK_KEEPMOUNT} --exec ";
|
714
709
|
if ($docker_hash)
|
715
710
|
{
|
@@ -738,18 +733,32 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
|
|
738
733
|
$command .= "--volume=\Q$ENV{TASK_KEEPMOUNT}:/keep:ro\E ";
|
739
734
|
$ENV{TASK_KEEPMOUNT} = "/keep";
|
740
735
|
|
741
|
-
# TASK_WORK is a
|
742
|
-
# is writable, and persists until no
|
743
|
-
# more. We don't use --volumes-from to
|
744
|
-
# containers: it is only accessible to this
|
745
|
-
# away when this task stops.
|
746
|
-
|
747
|
-
|
748
|
-
#
|
749
|
-
#
|
750
|
-
#
|
751
|
-
#
|
752
|
-
|
736
|
+
# TASK_WORK is almost exactly like a docker data volume: it
|
737
|
+
# starts out empty, is writable, and persists until no
|
738
|
+
# containers use it any more. We don't use --volumes-from to
|
739
|
+
# share it with other containers: it is only accessible to this
|
740
|
+
# task, and it goes away when this task stops.
|
741
|
+
#
|
742
|
+
# However, a docker data volume is writable only by root unless
|
743
|
+
# the mount point already happens to exist in the container with
|
744
|
+
# different permissions. Therefore, we [1] assume /tmp already
|
745
|
+
# exists in the image and is writable by the crunch user; [2]
|
746
|
+
# avoid putting TASK_WORK inside CRUNCH_TMP (which won't be
|
747
|
+
# writable if they are created by docker while setting up the
|
748
|
+
# other --volumes); and [3] create $TASK_WORK inside the
|
749
|
+
# container using $build_script.
|
750
|
+
$command .= "--volume=/tmp ";
|
751
|
+
$ENV{"TASK_WORK"} = "/tmp/crunch-job-task-work/$childslotname";
|
752
|
+
$ENV{"HOME"} = $ENV{"TASK_WORK"};
|
753
|
+
$ENV{"TASK_TMPDIR"} = $ENV{"TASK_WORK"}; # deprecated
|
754
|
+
|
755
|
+
# TODO: Share a single JOB_WORK volume across all task
|
756
|
+
# containers on a given worker node, and delete it when the job
|
757
|
+
# ends (and, in case that doesn't work, when the next job
|
758
|
+
# starts).
|
759
|
+
#
|
760
|
+
# For now, use the same approach as TASK_WORK above.
|
761
|
+
$ENV{"JOB_WORK"} = "/tmp/crunch-job-work";
|
753
762
|
|
754
763
|
while (my ($env_key, $env_val) = each %ENV)
|
755
764
|
{
|
@@ -760,16 +769,16 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
|
|
760
769
|
$command .= "--env=\QHOME=$ENV{HOME}\E ";
|
761
770
|
$command .= "\Q$docker_hash\E ";
|
762
771
|
$command .= "stdbuf --output=0 --error=0 ";
|
763
|
-
$command .= "$ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
|
772
|
+
$command .= "perl - $ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
|
764
773
|
} else {
|
765
774
|
# Non-docker run
|
766
775
|
$command .= "crunchstat -cgroup-root=/sys/fs/cgroup -poll=10000 ";
|
767
776
|
$command .= "stdbuf --output=0 --error=0 ";
|
768
|
-
$command .= "$ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
|
777
|
+
$command .= "perl - $ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
|
769
778
|
}
|
770
779
|
|
771
780
|
my @execargs = ('bash', '-c', $command);
|
772
|
-
srun (\@srunargs, \@execargs, undef, $
|
781
|
+
srun (\@srunargs, \@execargs, undef, $build_script);
|
773
782
|
# exec() failed, we assume nothing happened.
|
774
783
|
die "srun() failed on build script\n";
|
775
784
|
}
|
@@ -1730,9 +1739,10 @@ use File::Path qw( make_path remove_tree );
|
|
1730
1739
|
my $destdir = $ENV{"CRUNCH_SRC"};
|
1731
1740
|
my $commit = $ENV{"CRUNCH_SRC_COMMIT"};
|
1732
1741
|
my $repo = $ENV{"CRUNCH_SRC_URL"};
|
1742
|
+
my $job_work = $ENV{"JOB_WORK"};
|
1733
1743
|
my $task_work = $ENV{"TASK_WORK"};
|
1734
1744
|
|
1735
|
-
for my $dir ($destdir, $task_work) {
|
1745
|
+
for my $dir ($destdir, $job_work, $task_work) {
|
1736
1746
|
if ($dir) {
|
1737
1747
|
make_path $dir;
|
1738
1748
|
-e $dir or die "Failed to create temporary directory ($dir): $!";
|
@@ -1743,16 +1753,17 @@ if ($task_work) {
|
|
1743
1753
|
remove_tree($task_work, {keep_root => 1});
|
1744
1754
|
}
|
1745
1755
|
|
1756
|
+
my @git_archive_data = <DATA>;
|
1757
|
+
if (!@git_archive_data) {
|
1758
|
+
# Nothing to extract -> nothing to install.
|
1759
|
+
run_argv_and_exit();
|
1760
|
+
}
|
1746
1761
|
|
1747
1762
|
open L, ">", "$destdir.lock" or die "$destdir.lock: $!";
|
1748
1763
|
flock L, LOCK_EX;
|
1749
1764
|
if (readlink ("$destdir.commit") eq $commit && -d $destdir) {
|
1750
|
-
|
1751
|
-
|
1752
|
-
die "Cannot exec `@ARGV`: $!";
|
1753
|
-
} else {
|
1754
|
-
exit 0;
|
1755
|
-
}
|
1765
|
+
# This version already installed -> nothing to do.
|
1766
|
+
run_argv_and_exit();
|
1756
1767
|
}
|
1757
1768
|
|
1758
1769
|
unlink "$destdir.commit";
|
@@ -1761,13 +1772,10 @@ open STDOUT, ">", "$destdir.log";
|
|
1761
1772
|
open STDERR, ">&STDOUT";
|
1762
1773
|
|
1763
1774
|
mkdir $destdir;
|
1764
|
-
|
1765
|
-
|
1766
|
-
|
1767
|
-
|
1768
|
-
if(!close(TARX)) {
|
1769
|
-
die "'tar -C $destdir -xf -' exited $?: $!";
|
1770
|
-
}
|
1775
|
+
open TARX, "|-", "tar", "-C", $destdir, "-xf", "-";
|
1776
|
+
print TARX @git_archive_data;
|
1777
|
+
if(!close(TARX)) {
|
1778
|
+
die "'tar -C $destdir -xf -' exited $?: $!";
|
1771
1779
|
}
|
1772
1780
|
|
1773
1781
|
my $pwd;
|
@@ -1799,11 +1807,16 @@ if ($commit) {
|
|
1799
1807
|
|
1800
1808
|
close L;
|
1801
1809
|
|
1802
|
-
|
1810
|
+
run_argv_and_exit();
|
1811
|
+
|
1812
|
+
sub run_argv_and_exit
|
1813
|
+
{
|
1814
|
+
if (@ARGV) {
|
1803
1815
|
exec(@ARGV);
|
1804
1816
|
die "Cannot exec `@ARGV`: $!";
|
1805
|
-
} else {
|
1817
|
+
} else {
|
1806
1818
|
exit 0;
|
1819
|
+
}
|
1807
1820
|
}
|
1808
1821
|
|
1809
1822
|
sub shell_or_die
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: arvados-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.20141117160514
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Arvados Authors
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-11-
|
11
|
+
date: 2014-11-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: arvados
|
@@ -178,7 +178,7 @@ dependencies:
|
|
178
178
|
- - "<"
|
179
179
|
- !ruby/object:Gem::Version
|
180
180
|
version: 1.0.0
|
181
|
-
description: Arvados command line tools, git commit
|
181
|
+
description: Arvados command line tools, git commit 664919d58c3689cd9e0a25547ec1e02d9adda38c
|
182
182
|
email: gem-dev@curoverse.com
|
183
183
|
executables:
|
184
184
|
- arv
|