arvados-cli 0.1.20140917145602 → 0.1.20140917180103
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/arv-run-pipeline-instance +63 -32
- data/bin/crunch-job +7 -6
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bdf4e0ac1c7f1ec18001dcbea5dfcd6f15fc35a2
|
4
|
+
data.tar.gz: 4ac21d246d0674e38dab00e292d8fce3bef04adb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a61045e929a777bca0aae31234348ebfc75fce479ac07d0a86e44abadf6c9941ede8e79df93b6d52ba5e61bf1df1eef369e8007b5877381432cffe7c4e9077cd
|
7
|
+
data.tar.gz: 2e2a4be9ad5c283df2dc5056ad74dd9d0c656e7abfb4cafc69a0130970de6ac624672e36402f426ad63783989d23f7dcdcfc7b9346a614c9e00b0c737ee0cae5
|
@@ -61,8 +61,6 @@
|
|
61
61
|
class WhRunPipelineInstance
|
62
62
|
end
|
63
63
|
|
64
|
-
$application_version = 1.0
|
65
|
-
|
66
64
|
if RUBY_VERSION < '1.9.3' then
|
67
65
|
abort <<-EOS
|
68
66
|
#{$0.gsub(/^\.\//,'')} requires Ruby version 1.9.3 or higher.
|
@@ -111,21 +109,6 @@ if $arvados_api_host.match /local/
|
|
111
109
|
suppress_warnings { OpenSSL::SSL::VERIFY_PEER = OpenSSL::SSL::VERIFY_NONE }
|
112
110
|
end
|
113
111
|
|
114
|
-
class Google::APIClient
|
115
|
-
def discovery_document(api, version)
|
116
|
-
api = api.to_s
|
117
|
-
return @discovery_documents["#{api}:#{version}"] ||=
|
118
|
-
begin
|
119
|
-
response = self.execute!(
|
120
|
-
:http_method => :get,
|
121
|
-
:uri => self.discovery_uri(api, version),
|
122
|
-
:authenticated => false
|
123
|
-
)
|
124
|
-
response.body.class == String ? JSON.parse(response.body) : response.body
|
125
|
-
end
|
126
|
-
end
|
127
|
-
end
|
128
|
-
|
129
112
|
|
130
113
|
# Parse command line options (the kind that control the behavior of
|
131
114
|
# this program, that is, not the pipeline component parameters).
|
@@ -170,11 +153,19 @@ p = Trollop::Parser.new do
|
|
170
153
|
:short => :none,
|
171
154
|
:type => :string)
|
172
155
|
opt(:submit,
|
173
|
-
"
|
156
|
+
"Submit the pipeline instance to the server, and exit. Let the Crunch dispatch service satisfy the components by finding/running jobs.",
|
157
|
+
:short => :none,
|
158
|
+
:type => :boolean)
|
159
|
+
opt(:run_pipeline_here,
|
160
|
+
"Manage the pipeline instance in-process. Submit jobs to Crunch as needed. Do not exit until the pipeline finishes (or fails).",
|
161
|
+
:short => :none,
|
162
|
+
:type => :boolean)
|
163
|
+
opt(:run_jobs_here,
|
164
|
+
"Run jobs in the local terminal session instead of submitting them to Crunch. Implies --run-pipeline-here. Note: this results in a significantly different job execution environment, and some Crunch features are not supported. It can be necessary to modify a pipeline in order to make it run this way.",
|
174
165
|
:short => :none,
|
175
166
|
:type => :boolean)
|
176
167
|
opt(:run_here,
|
177
|
-
"
|
168
|
+
"Synonym for --run-jobs-here.",
|
178
169
|
:short => :none,
|
179
170
|
:type => :boolean)
|
180
171
|
opt(:description,
|
@@ -188,6 +179,9 @@ $options = Trollop::with_standard_exception_handling p do
|
|
188
179
|
end
|
189
180
|
$debuglevel = $options[:debug_level] || ($options[:debug] && 1) || 0
|
190
181
|
|
182
|
+
$options[:run_jobs_here] ||= $options[:run_here] # old flag name
|
183
|
+
$options[:run_pipeline_here] ||= $options[:run_jobs_here] # B requires A
|
184
|
+
|
191
185
|
if $options[:instance]
|
192
186
|
if $options[:template] or $options[:submit]
|
193
187
|
abort "#{$0}: syntax error: --instance cannot be combined with --template or --submit."
|
@@ -198,8 +192,8 @@ elsif not $options[:template]
|
|
198
192
|
abort
|
199
193
|
end
|
200
194
|
|
201
|
-
if $options[:
|
202
|
-
abort "#{$0}:
|
195
|
+
if $options[:run_pipeline_here] == $options[:submit]
|
196
|
+
abort "#{$0}: error: you must supply --run-pipeline-here, --run-jobs-here, or --submit."
|
203
197
|
end
|
204
198
|
|
205
199
|
# Suppress SSL certificate checks if ARVADOS_API_HOST_INSECURE
|
@@ -220,13 +214,9 @@ end
|
|
220
214
|
|
221
215
|
# Set up the API client.
|
222
216
|
|
223
|
-
$client ||= Google::APIClient.
|
224
|
-
new(:host => $arvados_api_host,
|
225
|
-
:application_name => File.split($0).last,
|
226
|
-
:application_version => $application_version.to_s)
|
227
|
-
$arvados = $client.discovered_api('arvados', $arvados_api_version)
|
228
217
|
$arv = Arvados.new api_version: 'v1'
|
229
|
-
|
218
|
+
$client = $arv.client
|
219
|
+
$arvados = $arv.arvados_api
|
230
220
|
|
231
221
|
class PipelineInstance
|
232
222
|
def self.find(uuid)
|
@@ -525,6 +515,7 @@ class WhRunPipelineInstance
|
|
525
515
|
# No job yet associated with this component and is component inputs
|
526
516
|
# are fully specified (any output_of script_parameters are resolved
|
527
517
|
# to real value)
|
518
|
+
my_submit_id = "instance #{@instance[:uuid]} rand #{rand(2**64).to_s(36)}"
|
528
519
|
job = JobCache.create(@instance, cname, {
|
529
520
|
:script => c[:script],
|
530
521
|
:script_parameters => c[:script_parameters],
|
@@ -533,6 +524,8 @@ class WhRunPipelineInstance
|
|
533
524
|
:nondeterministic => c[:nondeterministic],
|
534
525
|
:runtime_constraints => c[:runtime_constraints],
|
535
526
|
:owner_uuid => owner_uuid,
|
527
|
+
:is_locked_by_uuid => (@options[:run_jobs_here] ? owner_uuid : nil),
|
528
|
+
:submit_id => my_submit_id,
|
536
529
|
}, {
|
537
530
|
# This is the right place to put these attributes when
|
538
531
|
# dealing with new API servers.
|
@@ -545,12 +538,50 @@ class WhRunPipelineInstance
|
|
545
538
|
if job
|
546
539
|
debuglog "component #{cname} new job #{job[:uuid]}"
|
547
540
|
c[:job] = job
|
541
|
+
c[:run_in_process] = (@options[:run_jobs_here] and
|
542
|
+
job[:submit_id] == my_submit_id)
|
548
543
|
else
|
549
544
|
debuglog "component #{cname} new job failed", 0
|
550
545
|
job_creation_failed += 1
|
551
546
|
end
|
552
547
|
end
|
553
548
|
|
549
|
+
if c[:job] and c[:run_in_process] and c[:job][:success].nil?
|
550
|
+
report_status
|
551
|
+
begin
|
552
|
+
require 'open3'
|
553
|
+
Open3.popen3("arv-crunch-job", "--force-unlock",
|
554
|
+
"--job", c[:job][:uuid]) do |stdin, stdout, stderr, wait_thr|
|
555
|
+
debuglog "arv-crunch-job pid #{wait_thr.pid} started", 0
|
556
|
+
stdin.close
|
557
|
+
while true
|
558
|
+
rready, wready, = IO.select([stdout, stderr], [])
|
559
|
+
break if !rready[0]
|
560
|
+
begin
|
561
|
+
buf = rready[0].read_nonblock(2**20)
|
562
|
+
rescue EOFError
|
563
|
+
break
|
564
|
+
end
|
565
|
+
(rready[0] == stdout ? $stdout : $stderr).write(buf)
|
566
|
+
end
|
567
|
+
stdout.close
|
568
|
+
stderr.close
|
569
|
+
debuglog "arv-crunch-job pid #{wait_thr.pid} exit #{wait_thr.value.to_i}", 0
|
570
|
+
end
|
571
|
+
if not $arv.job.get(uuid: c[:job][:uuid])[:finished_at]
|
572
|
+
raise Exception.new("arv-crunch-job did not set finished_at.")
|
573
|
+
end
|
574
|
+
rescue Exception => e
|
575
|
+
debuglog "Interrupted (#{e}). Failing job.", 0
|
576
|
+
$arv.job.update(uuid: c[:job][:uuid],
|
577
|
+
job: {
|
578
|
+
finished_at: Time.now,
|
579
|
+
running: false,
|
580
|
+
success: false
|
581
|
+
})
|
582
|
+
end
|
583
|
+
end
|
584
|
+
|
554
585
|
if c[:job] and c[:job][:uuid]
|
555
586
|
if (c[:job][:running] or
|
556
587
|
not (c[:job][:finished_at] or c[:job][:cancelled_at]))
|
@@ -575,14 +606,12 @@ class WhRunPipelineInstance
|
|
575
606
|
# succeeded. (At the top of this loop, I was still
|
576
607
|
# waiting for it to finish.)
|
577
608
|
|
578
|
-
|
579
|
-
if (not @instance[:name].nil?) and (not @instance[:name].empty?)
|
609
|
+
if @instance[:name].andand.length.andand > 0
|
580
610
|
pipeline_name = @instance[:name]
|
581
|
-
elsif @
|
582
|
-
fetch_template(@instance[:pipeline_template_uuid])
|
611
|
+
elsif @template.andand[:name].andand.length.andand > 0
|
583
612
|
pipeline_name = @template[:name]
|
584
613
|
else
|
585
|
-
pipeline_name =
|
614
|
+
pipeline_name = @instance[:uuid]
|
586
615
|
end
|
587
616
|
if c[:output_name] != false
|
588
617
|
# Create a collection located in the same project as the pipeline with the contents of the output.
|
@@ -747,6 +776,8 @@ class WhRunPipelineInstance
|
|
747
776
|
"failed #{c[:job][:finished_at]}"
|
748
777
|
elsif c[:job][:started_at]
|
749
778
|
"started #{c[:job][:started_at]}"
|
779
|
+
elsif c[:job][:is_locked_by_uuid]
|
780
|
+
"starting #{c[:job][:started_at]}"
|
750
781
|
else
|
751
782
|
"queued #{c[:job][:created_at]}"
|
752
783
|
end
|
data/bin/crunch-job
CHANGED
@@ -848,12 +848,6 @@ release_allocation();
|
|
848
848
|
freeze();
|
849
849
|
my $collated_output = &collate_output();
|
850
850
|
|
851
|
-
if ($job_has_uuid) {
|
852
|
-
$Job->update_attributes('running' => 0,
|
853
|
-
'success' => $collated_output && $main::success,
|
854
|
-
'finished_at' => scalar gmtime)
|
855
|
-
}
|
856
|
-
|
857
851
|
if (!$collated_output) {
|
858
852
|
Log(undef, "output undef");
|
859
853
|
}
|
@@ -880,6 +874,13 @@ else {
|
|
880
874
|
Log (undef, "finish");
|
881
875
|
|
882
876
|
save_meta();
|
877
|
+
|
878
|
+
if ($job_has_uuid) {
|
879
|
+
$Job->update_attributes('running' => 0,
|
880
|
+
'success' => $collated_output && $main::success,
|
881
|
+
'finished_at' => scalar gmtime)
|
882
|
+
}
|
883
|
+
|
883
884
|
exit ($Job->{'success'} ? 1 : 0);
|
884
885
|
|
885
886
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: arvados-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.20140917180103
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Arvados Authors
|
@@ -166,7 +166,7 @@ dependencies:
|
|
166
166
|
- - "<"
|
167
167
|
- !ruby/object:Gem::Version
|
168
168
|
version: 1.0.0
|
169
|
-
description: This is the Arvados SDK CLI gem, git revision
|
169
|
+
description: This is the Arvados SDK CLI gem, git revision dce0ccabe3d9fab6943e89dc84050793cca5b553
|
170
170
|
email: gem-dev@curoverse.com
|
171
171
|
executables:
|
172
172
|
- arv
|