arvados-cli 0.1.20141007134429 → 0.1.20141007160434

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ec72b4c6433914d92d8f712603c11bf014e0dcea
-  data.tar.gz: 5c38938f5a1123631f0b9da225b8e895756788df
+  metadata.gz: 9e07192a0e91503a3e234029df54195bbb24046d
+  data.tar.gz: b565a3c5de08ce709331106b97d03a55c542a8b1
 SHA512:
-  metadata.gz: a234f80b2daf38847e9a655a3ee040e0978e348dff65011c04e1765944376fdd7ad89c46bf7b905ebb07f9afaa85602e527e6d5d8657c79ba88a4e1586936aac
-  data.tar.gz: f0b12cbf890165ed7ef486d6af389f43fb3f55b6cf398b1b5d07abc6c65cd5719e0199780ed9299b53796a608d50f1d9a74d72f3aa4436a2d3bccb8ebd15e9fc
+  metadata.gz: 0616516ef880b98b938eeef26741d0f25e6e116c0eb7fcec03db94b3d2f7139826398127c540d130c2009beb73ecec14db679ada5f82557569b1171860a8b035
+  data.tar.gz: 126b2996d4a63be3cc4cadc02f6277ca4c9e87b554d81dbe62a71d4c6940e448cd48b667ec77934fc93e4848d6baf4124a6bf17be4613f0a41e84f296c2b0605

data/bin/crunch-job

@@ -10,12 +10,14 @@ crunch-job: Execute job steps, save snapshots as requested, collate output.
 Obtain job details from Arvados, run tasks on compute nodes (typically
 invoked by scheduler on controller):
 
- crunch-job --job x-y-z
+ crunch-job --job x-y-z --git-dir /path/to/repo/.git
 
 Obtain job details from command line, run tasks on local machine
 (typically invoked by application or developer on VM):
 
- crunch-job --job '{"script_version":"/path/to/tree","script":"scriptname",...}'
+ crunch-job --job '{"script_version":"/path/to/working/tree","script":"scriptname",...}'
+
+ crunch-job --job '{"repository":"https://github.com/curoverse/arvados.git","script_version":"master","script":"scriptname",...}'
 
 =head1 OPTIONS
 
@@ -27,7 +29,9 @@ If the job is already locked, steal the lock and run it anyway.
 
 =item --git-dir
 
-Path to .git directory where the specified commit is found.
+Path to a .git directory (or a git URL) where the commit given in the
+job's C<script_version> attribute is to be found. If this is I<not>
+given, the job's C<repository> attribute will be used.
 
 =item --job-api-token
 
@@ -39,6 +43,11 @@ Do not clear per-job/task temporary directories during initial job
 setup. This can speed up development and debugging when running jobs
 locally.
 
+=item --job
+
+UUID of the job to run, or a JSON-encoded job resource without a
+UUID. If the latter is given, a new job object will be created.
+
 =back
 
 =head1 RUNNING JOBS LOCALLY
@@ -363,17 +372,21 @@ if (!defined $no_clear_tmp) {
 
 my $git_archive;
 if (!defined $git_dir && $Job->{'script_version'} =~ m{^/}) {
-  # If we're in user-land (i.e., not called from crunch-dispatch)
-  # script_version can be an absolute directory path, signifying we
-  # should work straight out of that directory instead of using a git
-  # commit.
+  # If script_version looks like an absolute path, *and* the --git-dir
+  # argument was not given -- which implies we were not invoked by
+  # crunch-dispatch -- we will use the given path as a working
+  # directory instead of resolving script_version to a git commit (or
+  # doing anything else with git).
   $ENV{"CRUNCH_SRC_COMMIT"} = $Job->{'script_version'};
   $ENV{"CRUNCH_SRC"} = $Job->{'script_version'};
 }
 else {
+  # Resolve the given script_version to a git commit sha1. Also, if
+  # the repository is remote, clone it into our local filesystem: this
+  # ensures "git archive" will work, and is necessary to reliably
+  # resolve a symbolic script_version like "master^".
   $ENV{"CRUNCH_SRC"} = "$ENV{CRUNCH_TMP}/src";
 
-  # Install requested code version
   Log (undef, "Looking for version ".$Job->{script_version}." from repository ".$Job->{repository});
 
   $ENV{"CRUNCH_SRC_COMMIT"} = $Job->{script_version};
@@ -415,17 +428,18 @@ else {
     # repository.
     my $arv_repo_list = retry_op(sub {
       $arv->{'repositories'}->{'list'}->execute(
-        'filters' => [['name','=',$repo]])->{'items'};
+        'filters' => [['name','=',$repo]]);
     });
-    my $n_found = scalar @{$arv_repo_list};
+    my @repos_found = @{$arv_repo_list->{'items'}};
+    my $n_found = $arv_repo_list->{'serverResponse'}->{'items_available'};
     if ($n_found > 0) {
       Log(undef, "Repository '$repo' -> "
-          . join(", ", map { $_->{'uuid'} } @{$arv_repo_list}));
+          . join(", ", map { $_->{'uuid'} } @repos_found));
     }
     if ($n_found != 1) {
       croak("Error: Found $n_found repositories with name '$repo'.");
     }
-    $repo = $arv_repo_list->[0]->{'fetch_url'};
+    $repo = $repos_found[0]->{'fetch_url'};
     $repo_location = 'remote';
   }
   Log(undef, "Using $repo_location repository '$repo'");
@@ -458,11 +472,12 @@ else {
     # our local cache first, since that's cheaper. (We don't want to
     # do that with tags/branches though -- those change over time, so
     # they should always be resolved by the remote repo.)
-    if ($treeish =~ /^[0-9a-f]{3,40}$/s) {
+    if ($treeish =~ /^[0-9a-f]{7,40}$/s) {
       # Hide stderr because it's normal for this to fail:
       my $sha1 = `$gitcmd rev-list -n1 ''\Q$treeish\E 2>/dev/null`;
       if ($? == 0 &&
-          $sha1 =~ /^$treeish/ && # Don't use commit 123 @ branch abc!
+          # Careful not to resolve a branch named abcdeff to commit 1234567:
+          $sha1 =~ /^$treeish/ &&
           $sha1 =~ /^([0-9a-f]{40})$/s) {
         $commit = $1;
         Log(undef, "Commit $commit already present in $local_repo");
@@ -1444,14 +1459,15 @@ sub croak
   Log (undef, $message);
   freeze() if @jobstep_todo;
   collate_output() if @jobstep_todo;
-  cleanup() if $Job;
-  save_meta() if log_writer_is_active();
+  cleanup();
+  save_meta();
   die;
 }
 
 
 sub cleanup
 {
+  return unless $Job;
   if ($Job->{'state'} eq 'Cancelled') {
     $Job->update_attributes('finished_at' => scalar gmtime);
   } else {
@@ -1464,6 +1480,7 @@ sub save_meta
 {
   my $justcheckpoint = shift; # false if this will be the last meta saved
   return if $justcheckpoint;  # checkpointing is not relevant post-Warehouse.pm
+  return unless log_writer_is_active();
 
   my $loglocator = log_writer_finish();
   Log (undef, "log manifest is $loglocator");

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: arvados-cli
 version: !ruby/object:Gem::Version
-  version: 0.1.20141007134429
+  version: 0.1.20141007160434
 platform: ruby
 authors:
 - Arvados Authors
@@ -178,7 +178,7 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
-description: Arvados command line tools, git commit 344c6dcdbae76310879c85a736e4e6cce05d5645
+description: Arvados command line tools, git commit dd7bb176c565d0d0718f9b0e59a6d9ee4b8ecbf2
 email: gem-dev@curoverse.com
 executables:
 - arv