arvados-cli 0.1.20140326173450 → 0.1.20140327092713
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/arv-run-pipeline-instance +23 -80
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: acc87ed620bd3dab59995717a259df4711c97ec4
|
4
|
+
data.tar.gz: 54b3c124bd65e48ab76ca0a43d782ef69b58b5e6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3f9c6d633e1f638ceba03d503e248033504b7de33db85f666cc457d9f8eb836085accf12e8b16837dc2c5b29ce9de5e3dead71bdf1a1b373724218993689471e
|
7
|
+
data.tar.gz: 7ffea4f249a4e93cadfe56cbc286278b2b61daa8343b3607ccf5a7646e282095e77fbc9c66f28ed085479df8e1a06cdaa1624552df798e6a78a0d9d2e23b1b89
|
@@ -28,11 +28,6 @@
|
|
28
28
|
# [--no-wait] Make only as much progress as possible without entering
|
29
29
|
# a sleep/poll loop.
|
30
30
|
#
|
31
|
-
# [--no-reuse-finished] Do not reuse existing outputs to satisfy
|
32
|
-
# pipeline components. Always submit a new job
|
33
|
-
# or use an existing job which has not yet
|
34
|
-
# finished.
|
35
|
-
#
|
36
31
|
# [--no-reuse] Do not reuse existing jobs to satisfy pipeline
|
37
32
|
# components. Submit a new job for every component.
|
38
33
|
#
|
@@ -152,10 +147,6 @@ p = Trollop::Parser.new do
|
|
152
147
|
"Do not wait for jobs to finish. Just look up status, submit new jobs if needed, and exit.",
|
153
148
|
:short => :none,
|
154
149
|
:type => :boolean)
|
155
|
-
opt(:no_reuse_finished,
|
156
|
-
"Do not reuse existing outputs to satisfy pipeline components. Always submit a new job or use an existing job which has not yet finished.",
|
157
|
-
:short => :none,
|
158
|
-
:type => :boolean)
|
159
150
|
opt(:no_reuse,
|
160
151
|
"Do not reuse existing jobs to satisfy pipeline components. Submit a new job for every component.",
|
161
152
|
:short => :none,
|
@@ -331,7 +322,7 @@ class JobCache
|
|
331
322
|
if j.is_a? Hash and j[:uuid]
|
332
323
|
@cache[j[:uuid]] = j
|
333
324
|
else
|
334
|
-
debuglog "create job: #{j[:errors] rescue nil}", 0
|
325
|
+
debuglog "create job: #{j[:errors] rescue nil} with attribute #{attributes}", 0
|
335
326
|
nil
|
336
327
|
end
|
337
328
|
end
|
@@ -433,84 +424,35 @@ class WhRunPipelineInstance
|
|
433
424
|
moretodo = false
|
434
425
|
@components.each do |cname, c|
|
435
426
|
job = nil
|
436
|
-
if !c[:job] and
|
437
|
-
c[:script_parameters].select { |pname, p| p.is_a? Hash }.empty?
|
438
|
-
# Job is fully specified (all parameter values are present) but
|
439
|
-
# no particular job has been found.
|
440
|
-
|
441
|
-
debuglog "component #{cname} ready to satisfy."
|
442
|
-
|
443
|
-
c.delete :wait
|
444
|
-
second_place_job = nil # satisfies component, but not finished yet
|
445
|
-
|
446
|
-
(@options[:no_reuse] ? [] : JobCache.
|
447
|
-
where(script: c[:script],
|
448
|
-
script_parameters: c[:script_parameters],
|
449
|
-
script_version_descends_from: c[:script_version])
|
450
|
-
).each do |candidate_job|
|
451
|
-
candidate_params_downcase = Hash[candidate_job[:script_parameters].
|
452
|
-
map { |k,v| [k.downcase,v] }]
|
453
|
-
c_params_downcase = Hash[c[:script_parameters].
|
454
|
-
map { |k,v| [k.downcase,v] }]
|
455
|
-
|
456
|
-
debuglog "component #{cname} considering job #{candidate_job[:uuid]} version #{candidate_job[:script_version]} parameters #{candidate_params_downcase.inspect}", 3
|
457
|
-
|
458
|
-
unless candidate_params_downcase == c_params_downcase
|
459
|
-
next
|
460
|
-
end
|
461
|
-
|
462
|
-
if c[:script_version] !=
|
463
|
-
candidate_job[:script_version][0,c[:script_version].length]
|
464
|
-
debuglog "component #{cname} would be satisfied by job #{candidate_job[:uuid]} if script_version matched.", 2
|
465
|
-
next
|
466
|
-
end
|
467
427
|
|
468
|
-
|
469
|
-
|
470
|
-
|
471
|
-
|
472
|
-
|
473
|
-
|
474
|
-
|
475
|
-
|
476
|
-
|
477
|
-
|
478
|
-
|
479
|
-
|
480
|
-
|
481
|
-
|
482
|
-
|
483
|
-
break
|
484
|
-
end
|
485
|
-
if not c[:job] and second_place_job
|
486
|
-
job = second_place_job
|
487
|
-
$stderr.puts "using #{job[:uuid]} (running since #{job[:started_at]}) for component #{cname}"
|
428
|
+
if !c[:job] and
|
429
|
+
c[:script_parameters].select { |pname, p| p.is_a? Hash and p[:output_of]}.empty?
|
430
|
+
# No job yet associated with this component and is component inputs
|
431
|
+
# are fully specified (any output_of script_parameters are resolved
|
432
|
+
# to real value)
|
433
|
+
job = JobCache.create({:script => c[:script],
|
434
|
+
:script_parameters => c[:script_parameters],
|
435
|
+
:script_version => c[:script_version],
|
436
|
+
:repository => c[:repository],
|
437
|
+
:minimum_script_version => c[:minimum_script_version],
|
438
|
+
:exclude_script_versions => c[:exclude_minimum_script_versions],
|
439
|
+
:nondeterministic => c[:nondeterministic],
|
440
|
+
:no_reuse => @options[:no_reuse]})
|
441
|
+
if job
|
442
|
+
debuglog "component #{cname} new job #{job[:uuid]}"
|
488
443
|
c[:job] = job
|
444
|
+
else
|
445
|
+
debuglog "component #{cname} new job failed"
|
489
446
|
end
|
490
|
-
if not c[:job]
|
491
|
-
debuglog "component #{cname} not satisfied by any existing job."
|
492
|
-
if !@options[:dry_run]
|
493
|
-
debuglog "component #{cname} new job."
|
494
|
-
job = JobCache.create(:script => c[:script],
|
495
|
-
:script_parameters => c[:script_parameters],
|
496
|
-
:runtime_constraints => c[:runtime_constraints] || {},
|
497
|
-
:script_version => c[:script_version] || 'master')
|
498
|
-
if job
|
499
|
-
debuglog "component #{cname} new job #{job[:uuid]}"
|
500
|
-
c[:job] = job
|
501
|
-
else
|
502
|
-
debuglog "component #{cname} new job failed"
|
503
|
-
end
|
504
|
-
end
|
505
|
-
end
|
506
|
-
else
|
507
|
-
c[:wait] = true
|
508
447
|
end
|
448
|
+
|
509
449
|
if c[:job] and c[:job][:uuid]
|
510
450
|
if (c[:job][:running] or
|
511
451
|
not (c[:job][:finished_at] or c[:job][:cancelled_at]))
|
512
|
-
|
452
|
+
# Job is running so update copy of job record
|
453
|
+
c[:job] = JobCache.get(c[:job][:uuid])
|
513
454
|
end
|
455
|
+
|
514
456
|
if c[:job][:success]
|
515
457
|
# Populate script_parameters of other components waiting for
|
516
458
|
# this job
|
@@ -525,6 +467,7 @@ class WhRunPipelineInstance
|
|
525
467
|
end
|
526
468
|
elsif c[:job][:running] ||
|
527
469
|
(!c[:job][:started_at] && !c[:job][:cancelled_at])
|
470
|
+
# Job is still running
|
528
471
|
moretodo = true
|
529
472
|
elsif c[:job][:cancelled_at]
|
530
473
|
debuglog "component #{cname} job #{c[:job][:uuid]} cancelled."
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: arvados-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.20140327092713
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Arvados Authors
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-03-
|
11
|
+
date: 2014-03-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: google-api-client
|
@@ -132,7 +132,7 @@ dependencies:
|
|
132
132
|
- - "~>"
|
133
133
|
- !ruby/object:Gem::Version
|
134
134
|
version: '0.8'
|
135
|
-
description: This is the Arvados SDK CLI gem, git revision
|
135
|
+
description: This is the Arvados SDK CLI gem, git revision ee86983a93c335413c508c5717a8707bbced93e6
|
136
136
|
email: gem-dev@curoverse.com
|
137
137
|
executables:
|
138
138
|
- arv
|