arvados-cli 0.1.20140328145218 → 0.1.20140328152103

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6ef3ba3f60cc01742c9f52bee400efc3e6705767
-  data.tar.gz: 245e35a548db02e6221fd57a56c84617e6894e99
+  metadata.gz: 272f81448344f725c7753fa314ab58ae28ec24c5
+  data.tar.gz: 7a27f7396a43a06a5660fb8c97caa961397a5d34
 SHA512:
-  metadata.gz: 1d95f401575b6d180e088515b4233fa79df2c48588f92524714a82301e7a4dac97584bdfeb88ebcc6d579b12784b493c9ef62d7cca8d501d45bd8fa2f8683960
-  data.tar.gz: a4a31e06ebd662fc54b5fe351b1c857d4c84bdfced15cc12ec260ebff309c483fc04e1b0aa51c4429ff21474071af18950f67bf63c40f172c77d9d90ac242edb
+  metadata.gz: 4f27db74f753bc33b61ebba0e549a0e5db2e8f51a4ffd2c4c6dffed69b03c7ff7d4883827411c0f73ec5fe0ab378b580b62a2abbfe398f82d993969d59ba7836
+  data.tar.gz: 24828a815587cb7888261add2c88046002358aa9c00830227899c1f0165bbaf33f6eff8413dd8e750bcda7a7fa34895fe0754a23f5754b80b6fb5170831d92f7

data/bin/arv-run-pipeline-instance

@@ -74,6 +74,7 @@ $arvados_api_token = ENV['ARVADOS_API_TOKEN'] or
   abort "#{$0}: fatal: ARVADOS_API_TOKEN environment variable not set."
 
 begin
+  require 'arvados'
   require 'rubygems'
   require 'json'
   require 'pp'
@@ -84,7 +85,7 @@ rescue LoadError => l
   abort <<-EOS
 #{$0}: fatal: #{l.message}
 Some runtime dependencies may be missing.
-Try: gem install pp google-api-client json trollop
+Try: gem install arvados pp google-api-client json trollop
   EOS
 end
 
@@ -216,6 +217,7 @@ $client ||= Google::APIClient.
       :application_name => File.split($0).last,
       :application_version => $application_version.to_s)
 $arvados = $client.discovered_api('arvados', $arvados_api_version)
+$arv = Arvados.new api_version: 'v1'
 
 
 class PipelineInstance
@@ -424,7 +426,12 @@ class WhRunPipelineInstance
       moretodo = false
       @components.each do |cname, c|
         job = nil
-
+        # Is the job satisfying this component already known to be
+        # finished? (Already meaning "before we query API server about
+        # the job's current state")
+        c_already_finished = (c[:job] &&
+                              c[:job][:uuid] &&
+                              !c[:job][:success].nil?)
         if !c[:job] and
             c[:script_parameters].select { |pname, p| p.is_a? Hash and p[:output_of]}.empty?
           # No job yet associated with this component and is component inputs
@@ -437,7 +444,8 @@ class WhRunPipelineInstance
                             :minimum_script_version => c[:minimum_script_version],
                             :exclude_script_versions => c[:exclude_minimum_script_versions],
                             :nondeterministic => c[:nondeterministic],
-                            :no_reuse => @options[:no_reuse]})
+                            :no_reuse => @options[:no_reuse],
+                            :output_is_persistent => c[:output_is_persistent] || false})
           if job
             debuglog "component #{cname} new job #{job[:uuid]}"
             c[:job] = job
@@ -450,7 +458,7 @@ class WhRunPipelineInstance
           if (c[:job][:running] or
               not (c[:job][:finished_at] or c[:job][:cancelled_at]))
             # Job is running so update copy of job record
-            c[:job] = JobCache.get(c[:job][:uuid])            
+            c[:job] = JobCache.get(c[:job][:uuid])
           end
 
           if c[:job][:success]
@@ -465,6 +473,43 @@ class WhRunPipelineInstance
                 end
               end
             end
+            unless c_already_finished
+              # This is my first time discovering that the job
+              # succeeded. (At the top of this loop, I was still
+              # waiting for it to finish.)
+              if c[:output_is_persistent]
+                # I need to make sure a resources/wants link is in
+                # place to protect the output from garbage
+                # collection. (Normally Crunch does this for me, but
+                # here I might be reusing the output of someone else's
+                # job and I need to make sure it's understood that the
+                # output is valuable to me, too.)
+                wanted = c[:job][:output]
+                debuglog "checking for existing persistence link for #{wanted}"
+                @my_user_uuid ||= $arv.user.current[:uuid]
+                links = $arv.link.list(limit: 1,
+                                       filters:
+                                       [%w(link_class = resources),
+                                        %w(name = wants),
+                                        %w(tail_uuid =) + [@my_user_uuid],
+                                        %w(head_uuid =) + [wanted]
+                                       ])[:items]
+                if links.any?
+                  debuglog "link already exists, uuid #{links.first[:uuid]}"
+                else
+                  newlink = $arv.link.create link: \
+                  {
+                    link_class: 'resources',
+                    name: 'wants',
+                    tail_kind: 'arvados#user',
+                    tail_uuid: @my_user_uuid,
+                    head_kind: 'arvados#collection',
+                    head_uuid: wanted
+                  }
+                  debuglog "added link, uuid #{newlink[:uuid]}"
+                end
+              end
+            end
           elsif c[:job][:running] ||
               (!c[:job][:started_at] && !c[:job][:cancelled_at])
             # Job is still running
@@ -507,7 +552,7 @@ class WhRunPipelineInstance
         end
       end
     end
-    
+
     if ended == @components.length or failed > 0
       @instance[:active] = false
       @instance[:success] = (succeeded == @components.length)

data/bin/crunch-job

@@ -1,5 +1,5 @@
 #!/usr/bin/perl
-# -*- mode: perl; perl-indent-level: 2; -*-
+# -*- mode: perl; perl-indent-level: 2; indent-tabs-mode: nil; -*-
 
 =head1 NAME
 
@@ -766,6 +766,16 @@ if ($Job->{'output'})
       'uuid' => $Job->{'output'},
       'manifest_text' => $manifest_text,
     });
+    if ($Job->{'output_is_persistent'}) {
+      $arv->{'links'}->{'create'}->execute('link' => {
+        'tail_kind' => 'arvados#user',
+        'tail_uuid' => $User->{'uuid'},
+        'head_kind' => 'arvados#collection',
+        'head_uuid' => $Job->{'output'},
+        'link_class' => 'resources',
+        'name' => 'wants',
+      });
+    }
   };
   if ($@) {
     Log (undef, "Failed to register output manifest: $@");

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: arvados-cli
 version: !ruby/object:Gem::Version
-  version: 0.1.20140328145218
+  version: 0.1.20140328152103
 platform: ruby
 authors:
 - Arvados Authors
@@ -10,6 +10,20 @@ bindir: bin
 cert_chain: []
 date: 2014-03-28 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: arvados
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
 - !ruby/object:Gem::Dependency
   name: google-api-client
   requirement: !ruby/object:Gem::Requirement
@@ -132,7 +146,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
-description: This is the Arvados SDK CLI gem, git revision c6043720763ecad450078728e2a3c815c51148bc
+description: This is the Arvados SDK CLI gem, git revision 64b24877bdb4b35938e8b691463b78073a4fd31d
 email: gem-dev@curoverse.com
 executables:
 - arv