cnvrg 1.5.3 → 1.5.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c00637f5ba127ddb1859179dff0b8ea44ba64ca8102146c36e5cef9d09b9e8bb
-  data.tar.gz: 2036a76187827395553b717c3f5c47a272a5f480141294582c231e81d49b9bad
+  metadata.gz: cfc351ff0936f6dd53e58da8ced925f7000c68d2e5ffa3c687d15fc090faafc1
+  data.tar.gz: 2e32e0decc29ef4b41a08c82c74fdf8b197c8844d1d49f98efcb81284977b967
 SHA512:
-  metadata.gz: dba5b7970a7db3e502d0582d4d9ed6279f444c794a8c08a1a4100f696e6b9cbb69f5f448b616fe9a3f7f451bbc00ece13017002f426ac095e226993bf732a954
-  data.tar.gz: ccf55b4c78e2f2c068618539328786faf868a7a09f55a7b32a6b902f7a5569aabac69512aa76dfd6f168c9920331c121f5f57bea39ecacaa4e8f2ee162d90406
+  metadata.gz: 71928a5eba809f59237be1ff75e58df1ac8b41203705278cf110b96acade3ab04c5073135a1e0eae3c58621d896d13f47fc2be9099a715be0d3e1f1c4a6e7d5f
+  data.tar.gz: 563db133a90bccdca1ca4eec07cf8dc7356a551647ebfd19a9936752ea65097386b8c20f429f690c75525d83a72130b14aa1c5e023961617672e1d4c1fb05ad2

data/lib/cnvrg/cli.rb

@@ -1948,19 +1948,22 @@ module Cnvrg
     method_option :all_files, :type => :boolean, :aliases => ["--all"], :desc => "download specified commit", :default => true
     method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
     method_option :init, :type => :boolean, :aliases => ["--initial"], :desc => "initial sync", :default => false
+    method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
 
-    def sync_data_new(new_branch, force, verbose, commit, all_files, tags ,parallel, chunk_size, init)
+
+    def sync_data_new(new_branch, force, verbose, commit, all_files, tags ,parallel, chunk_size, init, message)
       verify_logged_in(true)
       log_start(__method__, args, options)
       log_message('Syncing dataset', Thor::Shell::Color::BLUE, !options["verbose"])
+
       if !force and !init
         # w(verbose=false, new_branch=false,sync=false, commit=nil,all_files=true)
         invoke :download_data_new,[verbose, new_branch, true, commit, all_files], :new_branch=>new_branch, :direct=>false, :force =>force
       end
 
       # w(new_branch,  verbose,sync,force, tags, chunk_size)
-      invoke :upload_data_new,[new_branch,  verbose,true,force, tags, chunk_size], :new_branch=>new_branch,
-             :direct=>false, :force =>force, :sync =>true, :tags =>tags, :parallel => parallel
+      invoke :upload_data_new,[new_branch, verbose, true, force, tags, chunk_size, message:message], :new_branch=>new_branch,
+             :direct=>false, :force =>force, :sync =>true, :tags =>tags, :parallel => parallel, :message => message
 
     end
     desc 'upload_data_new', 'upload_data_new', :hide => true
@@ -1970,14 +1973,16 @@ module Cnvrg
     method_option :sync, :type => :boolean, :aliases => ["-s","--sync"], :default => false
     method_option :tags, :type => :boolean, :aliases => ["--tags"], :desc => "upload file tags", :default => false
     method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
-    def upload_data_new(new_branch,  verbose,sync,force, tags, chunk_size)
+    method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
+
+    def upload_data_new(new_branch, verbose, sync, force, tags, chunk_size, message:nil)
       begin
-      commit = invoke :start_commit_data,[], :new_branch=> new_branch, :direct=>false, :force =>force, :chunk_size => chunk_size
+      commit = invoke :start_commit_data,[], :new_branch=> new_branch, :direct=>false, :force =>force, :chunk_size => chunk_size, :message => message
       upload_res = invoke :upload_data_files,[commit],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync, :chunk_size => chunk_size
       if upload_res < 0
         return
       end
-      invoke :end_commit_data,[commit, success: true, uploaded_files: upload_res] , :new_branch=>new_branch,  :force =>force
+      invoke :end_commit_data,[commit, success: true, uploaded_files: upload_res], :new_branch=>new_branch, :force =>force
       if tags
         log_message('Uploading Tags', Thor::Shell::Color::BLUE)
         dataset_dir = is_cnvrg_dir(Dir.pwd)
@@ -2017,6 +2022,7 @@ module Cnvrg
     method_option :direct, :type => :boolean, :aliases => ["-d", "--direct"], :desc => "was called directed", :default => true
     method_option :force, :type => :boolean, :aliases => ["-f", "--force"], :default => false
     method_option :chunk_size, :type => :numeric, :aliases => ["-ch"], :default => 0
+    method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
 
     def start_commit_data()
       verify_logged_in(true)
@@ -2026,13 +2032,14 @@ module Cnvrg
       new_branch = options["new_branch"] || false
       force = options["force"] || false
       chunk_size = options["chunk_size"] || false
+      message = options["message"]
       commit_sha1 = nil
       @dataset = Dataset.new(dataset_dir)
       @dataset.backup_idx
       @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
       next_commit = @dataset.get_next_commit #if there was a partial commit..
       chunks = (@dataset.list_all_files.length.to_f / chunk_size).ceil
-      resp = @files.start_commit(new_branch, force, chunks: chunks, dataset: @dataset)
+      resp = @files.start_commit(new_branch, force, chunks: chunks, dataset: @dataset, message: message)
       if !resp['result']['can_commit']
         log_message("Cant upload files because a new version of this dataset exists, please download it or upload with --force", Thor::Shell::Color::RED)
         exit(1)
@@ -2516,6 +2523,73 @@ module Cnvrg
       end
     end
 
+    desc 'commit before termination', 'Commit job code before termination'
+    def commit_before_termination()
+      job_type = ENV['CNVRG_JOB_TYPE']
+      job_id =  ENV['CNVRG_JOB_ID']
+      return unless job_type.present? and job_id.present?
+      invoke :sync, [false], :job_slug => job_id, :job_type => job_type, :new_branch => true, :in_exp=> true
+    rescue => e
+      log_error(e)
+    end
+
+    desc 'update_job_commit', 'Update job with its last commit'
+    def update_job_commit()
+      job_type = ENV['CNVRG_JOB_TYPE']
+      job_id =  ENV['CNVRG_JOB_ID']
+      return unless job_type.present? and job_id.present?
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+      project_home = get_project_home
+      @project = Project.new(project_home)
+      current_commit = @project.get_idx[:commit] rescue nil
+      commit  = @project.get_job_last_commit(job_type, job_id)
+      if commit.present? and commit != current_commit
+        invoke :download, [false, "", true ], :commit => commit
+      end
+    rescue
+    end
+
+    desc 'update running jupyter token to server', 'update running jupyter token to server', :hide =>true
+    def update_jupyter_token()
+      begin
+        job_type = ENV['CNVRG_JOB_TYPE']
+        job_id =  ENV['CNVRG_JOB_ID']
+        return unless job_type.present? and job_id.present?
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+        count = 0
+        token = nil
+        while count < 20
+          res = `jupyter notebook list`
+          match = res.match(/token=(\w+)/)
+          if match.present?
+            token = match[1]
+            break
+          end
+          sleep(0.5)
+        end
+        if token.blank?
+          log_message("Failed to find jupyter token", Thor::Shell::Color::RED)
+          return
+        end
+        log_message("Found token #{token}", Thor::Shell::Color::BLUE)
+        project_home = get_project_home
+        @project = Project.new(project_home)
+        puts(token)
+        resp = @project.update_job_jupyter_token(job_type, job_id, token)
+        if resp["status"] == 200
+          log_message("Updated jupter token successfully", Thor::Shell::Color::BLUE)
+        else
+          log_message("Failed to update jupyter token ", Thor::Shell::Color::RED)
+        end
+      rescue => e
+        log_error(e)
+        log_message("Error while trying to get jupyter token ", Thor::Shell::Color::RED)
+        return
+      end
+    end
+
 
     desc 'download', 'Download updated files'
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"

data/lib/cnvrg/data.rb

@@ -29,6 +29,8 @@ module Cnvrg
     method_option :sync, :type => :boolean, :aliases => ["-s","--sync"], :default => false
     method_option :tags, :type => :boolean, :aliases => ["--tags"], :desc => "upload file tags", :default => false
     method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :desc => "upload file tags", :default => 1000
+    method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
+
     def upload
       cli = Cnvrg::CLI.new()
       verbose = options["verbose"]
@@ -37,7 +39,8 @@ module Cnvrg
       new_branch = options["new_branch"]
       chunk_size = options["chunk_size"]
       tags = options["tags"]
-      cli.upload_data_new(new_branch,  verbose,sync,force, tags, chunk_size)
+      message = options["message"]
+      cli.upload_data_new(new_branch, verbose, sync, force, tags, chunk_size, message:message)
     end
     desc 'data sync', 'Synchronise local dataset directory with remote server'
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
@@ -49,6 +52,7 @@ module Cnvrg
     method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
     method_option :chunk_size, :type => :numeric, :aliases => ["--chunk_size"], :desc => "chunk size to communicate with the server", :default => 1000
     method_option :init, :type => :boolean, :aliases => ["--initial"], :desc => "initial upload of dataset", :default => false
+    method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
 
     def sync_data_new()
       cli = Cnvrg::CLI.new()
@@ -61,7 +65,8 @@ module Cnvrg
       parallel=options["parallel"]
       chunk_size = options["chunk_size"]
       init = options["init"]
-      cli.sync_data_new(new_branch, force, verbose,commit,all_files, tags,parallel, chunk_size, init)
+      message = options["message"]
+      cli.sync_data_new(new_branch, force, verbose, commit, all_files, tags, parallel, chunk_size, init, message)
     end
     desc 'data download', 'Download files from remote server'
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits", :default => false

data/lib/cnvrg/datafiles.rb

@@ -854,7 +854,7 @@ module Cnvrg
       Cnvrg::CLI.is_response_success(response, true)
       return response
     end
-    def start_commit(new_branch,force=false,delete_commit=nil, chunks: 0, dataset: @dataset)
+    def start_commit(new_branch,force=false,delete_commit=nil, chunks: 0, dataset: @dataset, message:nil)
       begin
         #if we are pushing with force or to branch we dont need to send current/next commit cause we want to
         # create a new commit.
@@ -862,7 +862,7 @@ module Cnvrg
         commit = idx[:commit]
         next_commit = idx[:next_commit]
       response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch,force:force,
-                                                                              username: @owner,current_commit: commit, next_commit: next_commit, total_chunks: chunks})
+                                                                              username: @owner,current_commit: commit, next_commit: next_commit, total_chunks: chunks, message: message})
       Cnvrg::CLI.is_response_success(response, true)
       return response
       rescue => e

data/lib/cnvrg/project.rb

@@ -471,6 +471,13 @@ module Cnvrg
       return response
     end
 
+    def get_job_last_commit(job_type, job_id)
+      base_url = "users/#{self.owner}/projects/#{self.slug}/jobs/#{job_type.underscore}/#{job_id}"
+      resp = Cnvrg::API.request("#{base_url}/last_commit", "GET")
+      commit = resp["commit"]
+      return commit
+    end
+
     def compare_idx(new_branch, force:false, deploy: false, in_exp:false, specific_files: [], download: false)
       is_download = download
       if is_download
@@ -638,6 +645,12 @@ module Cnvrg
       @machines || []
     end
 
+
+    def update_job_jupyter_token(job_type, job_id, token)
+      base_url = "users/#{self.owner}/projects/#{self.slug}/jobs/#{job_type.underscore}/#{job_id}"
+      Cnvrg::API.request("#{base_url}/update_jupyter_token", "POST", {token: token})
+    end
+
     def check_machine(machine)
       Cnvrg::Logger.log_info("Check if #{machine} machine exists")
       machines = get_machines

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '1.5.3'
+  VERSION = '1.5.6'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.5.3
+  version: 1.5.6
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-02 00:00:00.000000000 Z
+date: 2019-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -432,7 +432,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubyforge_project: 
+rubygems_version: 2.7.9
 signing_key: 
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.