arvados-cli 0.1.20140320160303 → 0.1.20140321150015

Sign up to get free protection for your applications and to get access to all the features.
Files changed (3) hide show
  1. checksums.yaml +4 -4
  2. data/bin/crunch-job +51 -14
  3. metadata +28 -28
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 651a092a5aa772985641a04cbdda40b255f200b5
4
- data.tar.gz: f01664314331a841ba5678962f80c3a95d1a0e7a
3
+ metadata.gz: 788d5ca44ca9c901b6e8f7c477a69cbc2068d378
4
+ data.tar.gz: 92c73dae1010cc1ed2f357f769988f18b566ff37
5
5
  SHA512:
6
- metadata.gz: 8beca4c2358f858685a0b5e6d5d636956b8b345f47befb8be79e99abcca075c1ca910acea8babd87dab8f9c28fb83d45b91c71d47e9be408d61f33df9690ccbc
7
- data.tar.gz: 9efe6117d732d33250851f48a7b08601d355562298cc32024f890c851000d67027ece62bb1fde0df6314259c674ecca71ced0743db60f9a396bccd5516491e38
6
+ metadata.gz: 4f44a6f0f83a3addb71ce85af76b1301824d1a1a7774ce29f6de4784833cc151f751d3cbf0902cc7b9f908a2f5a86388bbc9169f1f8c70c97480e534377d5750
7
+ data.tar.gz: 242989a230f9a58ac29daa3edd0d259125483e0b5c6d10671efba142a79c1a6bd517160a3f91e73e23d77146831a66c54446cc9ff6b17744ff4b4da514b4dea1
data/bin/crunch-job CHANGED
@@ -33,6 +33,12 @@ Path to .git directory where the specified commit is found.
33
33
 
34
34
  Arvados API authorization token to use during the course of the job.
35
35
 
36
+ =item --no-clear-tmp
37
+
38
+ Do not clear per-job/task temporary directories during initial job
39
+ setup. This can speed up development and debugging when running jobs
40
+ locally.
41
+
36
42
  =back
37
43
 
38
44
  =head1 RUNNING JOBS LOCALLY
@@ -74,6 +80,7 @@ use Getopt::Long;
74
80
  use Warehouse;
75
81
  use Warehouse::Stream;
76
82
  use IPC::System::Simple qw(capturex);
83
+ use Fcntl ':flock';
77
84
 
78
85
  $ENV{"TMPDIR"} ||= "/tmp";
79
86
  unless (defined $ENV{"CRUNCH_TMP"}) {
@@ -92,11 +99,13 @@ my $force_unlock;
92
99
  my $git_dir;
93
100
  my $jobspec;
94
101
  my $job_api_token;
102
+ my $no_clear_tmp;
95
103
  my $resume_stash;
96
104
  GetOptions('force-unlock' => \$force_unlock,
97
105
  'git-dir=s' => \$git_dir,
98
106
  'job=s' => \$jobspec,
99
107
  'job-api-token=s' => \$job_api_token,
108
+ 'no-clear-tmp' => \$no_clear_tmp,
100
109
  'resume-stash=s' => \$resume_stash,
101
110
  );
102
111
 
@@ -323,6 +332,12 @@ else
323
332
  }
324
333
 
325
334
 
335
+ if (!$have_slurm)
336
+ {
337
+ must_lock_now("$ENV{CRUNCH_TMP}/.lock", "a job is already running here.");
338
+ }
339
+
340
+
326
341
  my $build_script;
327
342
 
328
343
 
@@ -331,6 +346,11 @@ $ENV{"CRUNCH_SRC_COMMIT"} = $Job->{script_version};
331
346
  my $skip_install = ($local_job && $Job->{script_version} =~ m{^/});
332
347
  if ($skip_install)
333
348
  {
349
+ if (!defined $no_clear_tmp) {
350
+ my $clear_tmp_cmd = 'rm -rf $JOB_WORK $CRUNCH_TMP/opt $CRUNCH_TMP/src*';
351
+ system($clear_tmp_cmd) == 0
352
+ or croak ("`$clear_tmp_cmd` failed: ".($?>>8));
353
+ }
334
354
  $ENV{"CRUNCH_SRC"} = $Job->{script_version};
335
355
  for my $src_path ("$ENV{CRUNCH_SRC}/arvados/sdk/python") {
336
356
  if (-d $src_path) {
@@ -351,22 +371,24 @@ else
351
371
  Log (undef, "Install revision ".$Job->{script_version});
352
372
  my $nodelist = join(",", @node);
353
373
 
354
- # Clean out crunch_tmp/work, crunch_tmp/opt, crunch_tmp/src*
374
+ if (!defined $no_clear_tmp) {
375
+ # Clean out crunch_tmp/work, crunch_tmp/opt, crunch_tmp/src*
355
376
 
356
- my $cleanpid = fork();
357
- if ($cleanpid == 0)
358
- {
359
- srun (["srun", "--nodelist=$nodelist", "-D", $ENV{'TMPDIR'}],
360
- ['bash', '-c', 'if mount | grep -q $JOB_WORK/; then sudo /bin/umount $JOB_WORK/* 2>/dev/null; fi; sleep 1; rm -rf $JOB_WORK $CRUNCH_TMP/opt $CRUNCH_TMP/src*']);
361
- exit (1);
362
- }
363
- while (1)
364
- {
365
- last if $cleanpid == waitpid (-1, WNOHANG);
366
- freeze_if_want_freeze ($cleanpid);
367
- select (undef, undef, undef, 0.1);
377
+ my $cleanpid = fork();
378
+ if ($cleanpid == 0)
379
+ {
380
+ srun (["srun", "--nodelist=$nodelist", "-D", $ENV{'TMPDIR'}],
381
+ ['bash', '-c', 'if mount | grep -q $JOB_WORK/; then sudo /bin/umount $JOB_WORK/* 2>/dev/null; fi; sleep 1; rm -rf $JOB_WORK $CRUNCH_TMP/opt $CRUNCH_TMP/src*']);
382
+ exit (1);
383
+ }
384
+ while (1)
385
+ {
386
+ last if $cleanpid == waitpid (-1, WNOHANG);
387
+ freeze_if_want_freeze ($cleanpid);
388
+ select (undef, undef, undef, 0.1);
389
+ }
390
+ Log (undef, "Clean-work-dir exited $?");
368
391
  }
369
- Log (undef, "Clean-work-dir exited $?");
370
392
 
371
393
  # Install requested code version
372
394
 
@@ -463,6 +485,12 @@ else
463
485
  Log (undef, "Install exited $?");
464
486
  }
465
487
 
488
+ if (!$have_slurm)
489
+ {
490
+ # Grab our lock again (we might have deleted and re-created CRUNCH_TMP above)
491
+ must_lock_now("$ENV{CRUNCH_TMP}/.lock", "a job is already running here.");
492
+ }
493
+
466
494
 
467
495
 
468
496
  foreach (qw (script script_version script_parameters runtime_constraints))
@@ -1346,6 +1374,15 @@ sub ban_node_by_slot {
1346
1374
  Log (undef, "backing off node " . $slot[$slotid]->{node}->{name} . " for 60 seconds");
1347
1375
  }
1348
1376
 
1377
+ sub must_lock_now
1378
+ {
1379
+ my ($lockfile, $error_message) = @_;
1380
+ open L, ">", $lockfile or croak("$lockfile: $!");
1381
+ if (!flock L, LOCK_EX|LOCK_NB) {
1382
+ croak("Can't lock $lockfile: $error_message\n");
1383
+ }
1384
+ }
1385
+
1349
1386
  __DATA__
1350
1387
  #!/usr/bin/perl
1351
1388
 
metadata CHANGED
@@ -1,138 +1,138 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: arvados-cli
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.20140320160303
4
+ version: 0.1.20140321150015
5
5
  platform: ruby
6
6
  authors:
7
7
  - Arvados Authors
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-03-20 00:00:00.000000000 Z
11
+ date: 2014-03-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: google-api-client
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ~>
17
+ - - "~>"
18
18
  - !ruby/object:Gem::Version
19
19
  version: 0.6.3
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ~>
24
+ - - "~>"
25
25
  - !ruby/object:Gem::Version
26
26
  version: 0.6.3
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: activesupport
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - ~>
31
+ - - "~>"
32
32
  - !ruby/object:Gem::Version
33
33
  version: '3.2'
34
- - - '>='
34
+ - - ">="
35
35
  - !ruby/object:Gem::Version
36
36
  version: 3.2.13
37
37
  type: :runtime
38
38
  prerelease: false
39
39
  version_requirements: !ruby/object:Gem::Requirement
40
40
  requirements:
41
- - - ~>
41
+ - - "~>"
42
42
  - !ruby/object:Gem::Version
43
43
  version: '3.2'
44
- - - '>='
44
+ - - ">="
45
45
  - !ruby/object:Gem::Version
46
46
  version: 3.2.13
47
47
  - !ruby/object:Gem::Dependency
48
48
  name: json
49
49
  requirement: !ruby/object:Gem::Requirement
50
50
  requirements:
51
- - - ~>
51
+ - - "~>"
52
52
  - !ruby/object:Gem::Version
53
53
  version: '1.7'
54
- - - '>='
54
+ - - ">="
55
55
  - !ruby/object:Gem::Version
56
56
  version: 1.7.7
57
57
  type: :runtime
58
58
  prerelease: false
59
59
  version_requirements: !ruby/object:Gem::Requirement
60
60
  requirements:
61
- - - ~>
61
+ - - "~>"
62
62
  - !ruby/object:Gem::Version
63
63
  version: '1.7'
64
- - - '>='
64
+ - - ">="
65
65
  - !ruby/object:Gem::Version
66
66
  version: 1.7.7
67
67
  - !ruby/object:Gem::Dependency
68
68
  name: trollop
69
69
  requirement: !ruby/object:Gem::Requirement
70
70
  requirements:
71
- - - ~>
71
+ - - "~>"
72
72
  - !ruby/object:Gem::Version
73
73
  version: '2.0'
74
74
  type: :runtime
75
75
  prerelease: false
76
76
  version_requirements: !ruby/object:Gem::Requirement
77
77
  requirements:
78
- - - ~>
78
+ - - "~>"
79
79
  - !ruby/object:Gem::Version
80
80
  version: '2.0'
81
81
  - !ruby/object:Gem::Dependency
82
82
  name: andand
83
83
  requirement: !ruby/object:Gem::Requirement
84
84
  requirements:
85
- - - ~>
85
+ - - "~>"
86
86
  - !ruby/object:Gem::Version
87
87
  version: '1.3'
88
- - - '>='
88
+ - - ">="
89
89
  - !ruby/object:Gem::Version
90
90
  version: 1.3.3
91
91
  type: :runtime
92
92
  prerelease: false
93
93
  version_requirements: !ruby/object:Gem::Requirement
94
94
  requirements:
95
- - - ~>
95
+ - - "~>"
96
96
  - !ruby/object:Gem::Version
97
97
  version: '1.3'
98
- - - '>='
98
+ - - ">="
99
99
  - !ruby/object:Gem::Version
100
100
  version: 1.3.3
101
101
  - !ruby/object:Gem::Dependency
102
102
  name: oj
103
103
  requirement: !ruby/object:Gem::Requirement
104
104
  requirements:
105
- - - ~>
105
+ - - "~>"
106
106
  - !ruby/object:Gem::Version
107
107
  version: '2.0'
108
- - - '>='
108
+ - - ">="
109
109
  - !ruby/object:Gem::Version
110
110
  version: 2.0.3
111
111
  type: :runtime
112
112
  prerelease: false
113
113
  version_requirements: !ruby/object:Gem::Requirement
114
114
  requirements:
115
- - - ~>
115
+ - - "~>"
116
116
  - !ruby/object:Gem::Version
117
117
  version: '2.0'
118
- - - '>='
118
+ - - ">="
119
119
  - !ruby/object:Gem::Version
120
120
  version: 2.0.3
121
121
  - !ruby/object:Gem::Dependency
122
122
  name: curb
123
123
  requirement: !ruby/object:Gem::Requirement
124
124
  requirements:
125
- - - ~>
125
+ - - "~>"
126
126
  - !ruby/object:Gem::Version
127
127
  version: '0.8'
128
128
  type: :runtime
129
129
  prerelease: false
130
130
  version_requirements: !ruby/object:Gem::Requirement
131
131
  requirements:
132
- - - ~>
132
+ - - "~>"
133
133
  - !ruby/object:Gem::Version
134
134
  version: '0.8'
135
- description: This is the Arvados SDK CLI gem, git revision 061dc0bcb7ced7ad196c6cc71dd25aab59948e46
135
+ description: This is the Arvados SDK CLI gem, git revision 2e2ce40b47b9e248805dcccc59baa77591c13d79
136
136
  email: gem-dev@curoverse.com
137
137
  executables:
138
138
  - arv
@@ -157,17 +157,17 @@ require_paths:
157
157
  - lib
158
158
  required_ruby_version: !ruby/object:Gem::Requirement
159
159
  requirements:
160
- - - '>='
160
+ - - ">="
161
161
  - !ruby/object:Gem::Version
162
162
  version: '0'
163
163
  required_rubygems_version: !ruby/object:Gem::Requirement
164
164
  requirements:
165
- - - '>='
165
+ - - ">="
166
166
  - !ruby/object:Gem::Version
167
167
  version: '0'
168
168
  requirements: []
169
169
  rubyforge_project:
170
- rubygems_version: 2.2.1
170
+ rubygems_version: 2.2.2
171
171
  signing_key:
172
172
  specification_version: 4
173
173
  summary: Arvados SDK CLI