arvados-cli 0.1.20140326173450 → 0.1.20140327092713

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5e7ce3e63bcaeac4a73ce1efb484c66756677c8e
-  data.tar.gz: 90c8acaa05d9ca47f517aa8f61e9b79137f2f8fc
+  metadata.gz: acc87ed620bd3dab59995717a259df4711c97ec4
+  data.tar.gz: 54b3c124bd65e48ab76ca0a43d782ef69b58b5e6
 SHA512:
-  metadata.gz: dac0eeb952ef8a5fc8c29ee08e1f345a02443c6615f7f74fb1fa9f7927dac6f1ffe8af05ca61e5750093fbc744533f4795351608fb576dd2ee8c48ff7d9299d8
-  data.tar.gz: 3c65515ac79334a72887ac239263298568b20d84c68ceb58cabddc2cd111d4213f7d4fe18bd7146d379c8d073f2ef18d0dae2be14b4ee573359e58faf1a11382
+  metadata.gz: 3f9c6d633e1f638ceba03d503e248033504b7de33db85f666cc457d9f8eb836085accf12e8b16837dc2c5b29ce9de5e3dead71bdf1a1b373724218993689471e
+  data.tar.gz: 7ffea4f249a4e93cadfe56cbc286278b2b61daa8343b3607ccf5a7646e282095e77fbc9c66f28ed085479df8e1a06cdaa1624552df798e6a78a0d9d2e23b1b89

data/bin/arv-run-pipeline-instance

@@ -28,11 +28,6 @@
 # [--no-wait] Make only as much progress as possible without entering
 #             a sleep/poll loop.
 #
-# [--no-reuse-finished] Do not reuse existing outputs to satisfy
-#                       pipeline components. Always submit a new job
-#                       or use an existing job which has not yet
-#                       finished.
-#
 # [--no-reuse] Do not reuse existing jobs to satisfy pipeline
 #              components. Submit a new job for every component.
 #
@@ -152,10 +147,6 @@ p = Trollop::Parser.new do
       "Do not wait for jobs to finish. Just look up status, submit new jobs if needed, and exit.",
       :short => :none,
       :type => :boolean)
-  opt(:no_reuse_finished,
-      "Do not reuse existing outputs to satisfy pipeline components. Always submit a new job or use an existing job which has not yet finished.",
-      :short => :none,
-      :type => :boolean)
   opt(:no_reuse,
       "Do not reuse existing jobs to satisfy pipeline components. Submit a new job for every component.",
       :short => :none,
@@ -331,7 +322,7 @@ class JobCache
     if j.is_a? Hash and j[:uuid]
       @cache[j[:uuid]] = j
     else
-      debuglog "create job: #{j[:errors] rescue nil}", 0
+      debuglog "create job: #{j[:errors] rescue nil} with attribute #{attributes}", 0
       nil
     end
   end
@@ -433,84 +424,35 @@ class WhRunPipelineInstance
       moretodo = false
       @components.each do |cname, c|
         job = nil
-        if !c[:job] and
-            c[:script_parameters].select { |pname, p| p.is_a? Hash }.empty?
-          # Job is fully specified (all parameter values are present) but
-          # no particular job has been found.
-
-          debuglog "component #{cname} ready to satisfy."
-
-          c.delete :wait
-          second_place_job = nil # satisfies component, but not finished yet
-
-          (@options[:no_reuse] ? [] : JobCache.
-           where(script: c[:script],
-                 script_parameters: c[:script_parameters],
-                 script_version_descends_from: c[:script_version])
-           ).each do |candidate_job|
-            candidate_params_downcase = Hash[candidate_job[:script_parameters].
-                                             map { |k,v| [k.downcase,v] }]
-            c_params_downcase = Hash[c[:script_parameters].
-                                     map { |k,v| [k.downcase,v] }]
-
-            debuglog "component #{cname} considering job #{candidate_job[:uuid]} version #{candidate_job[:script_version]} parameters #{candidate_params_downcase.inspect}", 3
-
-            unless candidate_params_downcase == c_params_downcase
-              next
-            end
-
-            if c[:script_version] !=
-                candidate_job[:script_version][0,c[:script_version].length]
-              debuglog "component #{cname} would be satisfied by job #{candidate_job[:uuid]} if script_version matched.", 2
-              next
-            end
 
-            unless candidate_job[:success] || candidate_job[:running] ||
-                (!candidate_job[:started_at] && !candidate_job[:cancelled_at])
-              debuglog "component #{cname} would be satisfied by job #{candidate_job[:uuid]} if it were running or successful.", 2
-              next
-            end
-
-            if candidate_job[:success]
-              unless @options[:no_reuse_finished]
-                job = candidate_job
-                $stderr.puts "using #{job[:uuid]} (finished at #{job[:finished_at]}) for component #{cname}"
-                c[:job] = job
-              end
-            else
-              second_place_job ||= candidate_job
-            end
-            break
-          end
-          if not c[:job] and second_place_job
-            job = second_place_job
-            $stderr.puts "using #{job[:uuid]} (running since #{job[:started_at]}) for component #{cname}"
+        if !c[:job] and
+            c[:script_parameters].select { |pname, p| p.is_a? Hash and p[:output_of]}.empty?
+          # No job yet associated with this component and is component inputs
+          # are fully specified (any output_of script_parameters are resolved
+          # to real value)
+          job = JobCache.create({:script => c[:script],
+                            :script_parameters => c[:script_parameters],
+                            :script_version => c[:script_version],
+                            :repository => c[:repository],
+                            :minimum_script_version => c[:minimum_script_version],
+                            :exclude_script_versions => c[:exclude_minimum_script_versions],
+                            :nondeterministic => c[:nondeterministic],
+                            :no_reuse => @options[:no_reuse]})
+          if job
+            debuglog "component #{cname} new job #{job[:uuid]}"
             c[:job] = job
+          else
+            debuglog "component #{cname} new job failed"
           end
-          if not c[:job]
-            debuglog "component #{cname} not satisfied by any existing job."
-            if !@options[:dry_run]
-              debuglog "component #{cname} new job."
-              job = JobCache.create(:script => c[:script],
-                                    :script_parameters => c[:script_parameters],
-                                    :runtime_constraints => c[:runtime_constraints] || {},
-                                    :script_version => c[:script_version] || 'master')
-              if job
-                debuglog "component #{cname} new job #{job[:uuid]}"
-                c[:job] = job
-              else
-                debuglog "component #{cname} new job failed"
-              end
-            end
-          end
-        else
-          c[:wait] = true
         end
+
         if c[:job] and c[:job][:uuid]
           if (c[:job][:running] or
               not (c[:job][:finished_at] or c[:job][:cancelled_at]))
-            c[:job] = JobCache.get(c[:job][:uuid])
+            # Job is running so update copy of job record
+            c[:job] = JobCache.get(c[:job][:uuid])            
           end
+
           if c[:job][:success]
             # Populate script_parameters of other components waiting for
             # this job
@@ -525,6 +467,7 @@ class WhRunPipelineInstance
             end
           elsif c[:job][:running] ||
               (!c[:job][:started_at] && !c[:job][:cancelled_at])
+            # Job is still running
             moretodo = true
           elsif c[:job][:cancelled_at]
             debuglog "component #{cname} job #{c[:job][:uuid]} cancelled."

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: arvados-cli
 version: !ruby/object:Gem::Version
-  version: 0.1.20140326173450
+  version: 0.1.20140327092713
 platform: ruby
 authors:
 - Arvados Authors
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-26 00:00:00.000000000 Z
+date: 2014-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-api-client
@@ -132,7 +132,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
-description: This is the Arvados SDK CLI gem, git revision 1013c7cee5478538901fd9ba5fc0fb2ce30f6422
+description: This is the Arvados SDK CLI gem, git revision ee86983a93c335413c508c5717a8707bbced93e6
 email: gem-dev@curoverse.com
 executables:
 - arv