cnvrg 0.9.9.6 → 1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 062c8902037886f575ccb224b2289e63f42679f0
-  data.tar.gz: d46b2c4dc4fef81a18695d69caf6cf65dde514cd
+  metadata.gz: 12febac69e1c9f8c035af8af4f971130b94de970
+  data.tar.gz: 30935d7322d87135629f944f3d5588bc4f3c9a80
 SHA512:
-  metadata.gz: b95c6d8bae730055f93295441dc04b10a8b97c942c4ce80a5af971e5835bfa33db3dffe420f6ae7457d11b7c7f94a76f3525105ae90525d7e5dd9430becc7788
-  data.tar.gz: efdd40d83a57a95fc4bb6e1ec9f82f880f069cb2ef890e842761ef119081f6fa47b28a53ac229a074386686507c4f3d0ef80c70edd415c863d2cc4e85e2e5d26
+  metadata.gz: 6b0447676f52c0dc2ff437a3d2ce67aee5326485a4ac5dcf81814706713d2ffffcc16ca1520372080d895b73a87ca8fe3a27277870ec7f0ea05a6db27ded7ac8
+  data.tar.gz: c687e4ec459d503675171b0711a56c2cc048675833dbf1db97821b7f456eb05d5eca0772ab0ac89d60534e9d1e2c4db951b0b87b23748f5ddce8e08d5cbb985b

data/lib/cnvrg/api.rb

@@ -54,7 +54,7 @@ module Cnvrg
             conn.headers['Auth-Token'] = @pass
             conn.headers['User-Agent'] = "#{Cnvrg::API::USER_AGENT}"
                 conn.options.timeout = 420
-                conn.options.open_timeout =420
+                conn.options.open_timeout=180
             case method
               when 'GET'
                 retries = 0
@@ -82,7 +82,7 @@ module Cnvrg
                 end
                 when 'POST', 'PUT'
                 conn.options.timeout = 4200
-                conn.options.open_timeout =4200
+                conn.options.open_timeout=180
                 retries = 0
                 success = false
                 while !success and retries < 20

data/lib/cnvrg/cli.rb

@@ -771,21 +771,42 @@ module Cnvrg
       end
     end
 
+    desc 'data verify', 'Verify datasets'
+    method_option :timeout, :type => :numeric, :aliases => ["-t", "--timeout"], :desc => "Time to wait before returning final answer", :default => 15
+
+    def verify_datasets(dataset_titles, timeout=0)
+      begin
+        verify_logged_in(false)
+        log_start(__method__, args, options)
+        log_message("Verifying datasets #{dataset_titles}", Thor::Shell::Color::BLUE)
+        verified = Dataset.verify_datasets(dataset_titles, timeout)
+        log_message("All datasets are verified", Thor::Shell::Color::BLUE) if verified
+        log_message("Failed to verify datasets", Thor::Shell::Color::RED) if !verified
+        exit(1) if !verified
+
+      rescue SignalException
+        say "\nAborting", Thor::Shell::Color::RED
+        exit(1)
+      end
+    end
+
     desc 'data clone', 'Clone dataset'
     method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
     method_option :only_tree, :type => :boolean, :aliases => ["-t", "--tree"], :default => false
     method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => nil
     method_option :read, :type => :boolean, :aliases => ["-r", "--read"], :default => false
+    method_option :remote, :type => :boolean, :aliases => ["-h", "--remote"], :default => false
 
-    def clone_data(dataset_url,only_tree=false,commit=nil,query=nil,read=false)
+    def clone_data(dataset_url,only_tree=false,commit=nil,query=nil,read=false,remote=false)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
         commit = options["commit"] || commit
         only_tree = options["only_tree"] || only_tree
         read = options["read"] || read || false
+        remote = options["remote"] || remote || false
         query = options['query'].presence || query.presence
-        if query.present? 
+        if query.present?
           return clone_data_query(dataset_url, query)
         end
 
@@ -798,7 +819,7 @@ module Cnvrg
         Cnvrg::CLI.is_response_success(response,true)
         dataset_name = response["result"]["name"]
         dataset_home = Dir.pwd+"/"+dataset_name
-        if (Dir.exists? dataset_name)
+        if !remote and (Dir.exists? dataset_name)
           log_message("Error: Conflict with dir #{dataset_name}", Thor::Shell::Color::RED)
           if no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
             log_message("Remove dir in order to clone #{dataset_name}", Thor::Shell::Color::RED)
@@ -806,7 +827,7 @@ module Cnvrg
           end
         end
         check = Helpers.checkmark
-        if Dataset.clone(owner, dataset_name, slug)
+        if Dataset.clone(owner, dataset_name, slug, remote)
           log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
           @files = Cnvrg::Datafiles.new(owner, slug)
           log_message("Downloading files", Thor::Shell::Color::BLUE)
@@ -844,6 +865,7 @@ module Cnvrg
             end
 
             log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
+            @dataset.write_success
           end
         else
           log_message("Error: Couldn't create directory: #{dataset_name}", Thor::Shell::Color::RED)
@@ -923,6 +945,7 @@ module Cnvrg
             dataset.generate_idx()
             check = Helpers.checkmark
             log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
+            dataset.write_success(in_folder=true)
           rescue
               exit(1)
           end
@@ -1874,6 +1897,7 @@ module Cnvrg
         # w(verbose=false, new_branch=false,sync=false, commit=nil,all_files=true)
         invoke :download_data_new,[verbose, new_branch, true, commit, all_files], :new_branch=>new_branch, :direct=>false, :force =>force
       end
+
       # w(new_branch,  verbose,sync,force, tags, chunk_size)
       invoke :upload_data_new,[new_branch,  verbose,true,force, tags, chunk_size], :new_branch=>new_branch,
              :direct=>false, :force =>force, :sync =>true, :tags =>tags, :parallel => parallel
@@ -2126,7 +2150,7 @@ module Cnvrg
             if git_output_dir.ends_with? "/"
               git_output_dir = git_output_dir[0..-2]
             end
-            list = @project.generate_output_dir_tmp(git_output_dir)
+            list = @project.generate_output_dir(git_output_dir)
             spec_files_to_upload = list
             if spec_files_to_upload.blank?
               log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
@@ -2235,6 +2259,7 @@ module Cnvrg
           log_message("Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN)
         else
           if return_id
+            puts "\n"
             print_res = {
                 'success' => "true",
                 'commit_sha1' => res["result"]["commit_id"]
@@ -2263,6 +2288,7 @@ module Cnvrg
             'success' => "false",
             'message' => 'couldn\'t commit changes,  Rolling Back all changes.'
         }
+        puts "\n"
         puts JSON[print_res] if return_id
         return false
       end
@@ -2686,7 +2712,7 @@ module Cnvrg
       end
     end
 
-    desc 'sync', 'Sync with cnvrg'
+    desc 'sync', 'Sync with remote server'
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :default => ""
@@ -2741,6 +2767,7 @@ module Cnvrg
     method_option :image, :type => :string, :aliases => ["--image"], :default => nil
     method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
     method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
+    method_option :datasets, :type => :string, :aliases => ["--datasets"], :desc => "'[{\"id\": \"dataset id\", \"commit\": \"commit id\", \"query\": \"query name\", \"tree_only\": true]'", :default => ""
     method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
     method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :desc => "ignore following files", :default => ""
     method_option :force, :type => :boolean, :aliases => ["-f", "--force"], :default => false
@@ -2758,6 +2785,7 @@ module Cnvrg
     def run(*cmd)
       verify_logged_in(true)
       log_start(__method__, args, options)
+      datasets = options["datasets"]
       sync_before = options["sync_before"]
       sync_after = options["sync_after"]
       log = options["log"]
@@ -2794,7 +2822,6 @@ module Cnvrg
       git_branch = options["git_branch"]
       restart_if_stuck = options["restart_if_stuck"]
 
-
       options_hash = Hash[options]
 
       if local
@@ -2829,7 +2856,8 @@ module Cnvrg
                :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore, :force => force, :sync_before_terminate => sync_before_terminate,
                 :max_time => max_time,
                :periodic_sync => periodic_sync, :dataset_only_tree=> dataset_only_tree,
-               :output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch, :restart_if_stuck =>restart_if_stuck, :local_folders => local_folders
+               :output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch,
+               :restart_if_stuck =>restart_if_stuck, :local_folders => local_folders, :datasets => datasets
         return
       end
 
@@ -3166,6 +3194,7 @@ module Cnvrg
     method_option :git_branch, :type => :string, :aliases => [ "--git_branch"], :default => nil
     method_option :restart_if_stuck, :type => :boolean, :aliases => ["--restart"], :default => nil
     method_option :local_folders, :type => :string, :aliases => ["--local_folders"], :default => nil
+    method_option :datasets, :type => :string, :aliases => ["--datasets"], :default => nil
 
     def exec_remote(*cmd)
 
@@ -3178,6 +3207,7 @@ module Cnvrg
         title = options["title"] || nil
         grid = options["grid"] || nil
         data = options["data"] || nil
+        datasets = options["datasets"] || nil
         data_commit = options["data_commit"] || nil
         data_query = options["data_query"] || nil
         sync_before = options["sync_before"]
@@ -3237,7 +3267,7 @@ module Cnvrg
         local_folders_options = options["local_folders"]
         options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "title",
                              "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree",
-                             "data_query", "git_commit","git_branch", "restart_if_stuck","local_folders", "output_dir", "commit")
+                             "data_query", "git_commit","git_branch", "restart_if_stuck","local_folders","output_dir", "commit", "datasets" )
         exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
         command = "#{exec_options} #{remote}  #{upload_output_option} #{cmd.flatten.join(" ")}"
         commit_to_run = options["commit"] || nil
@@ -3308,7 +3338,7 @@ module Cnvrg
 
         res = exp.exec_remote(command, commit_to_run, instance_type, image, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
                               periodic_sync, sync_before_terminate, max_time, ds_sync_options,output_dir,
-                              data_query, git_commit, git_branch, restart_if_stuck,local_folders_options, title )
+                              data_query, git_commit, git_branch, restart_if_stuck,local_folders_options, title, datasets)
         if Cnvrg::CLI.is_response_success(res)
           check = Helpers.checkmark()
           str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"
@@ -3368,11 +3398,8 @@ module Cnvrg
           project = Project.new(working_dir)
           commit_to_run = options["commit"] || nil
 
-        workers = options["workers"]
+        workers = options["workers"] || nil
         begin
-          if workers.blank?
-            workers = 2
-          end
           num_workers = workers.to_i
         rescue
           log_message("Number of workers should be a number between 1 to 10", Thor::Shell::Color::RED)
@@ -3381,10 +3408,18 @@ module Cnvrg
         file_as_input = options["file_as_input"] || false
 
 
+        image = is_project_with_docker(working_dir)
+        if !image or !image.is_docker
+          image = Images.new(working_dir, "cnvrg")
+          image_slug = image.image_slug
+        else
+          image_slug = image.image_slug
+        end
+
 
         invoke :sync, [false], []
 
-          res = project.deploy(file_to_run, function, nil, commit_to_run, instance_type, "cnvrg", schedule, local_timestamp, num_workers, file_as_input, title)
+          res = project.deploy(file_to_run, function, nil, commit_to_run, instance_type, image_slug, schedule, local_timestamp, num_workers, file_as_input, title)
 
         if Cnvrg::CLI.is_response_success(res)
 
@@ -3416,6 +3451,7 @@ module Cnvrg
     method_option :local, :type => :boolean, :aliases => ["-l"], :default => false
     method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
     method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
+    method_option :datasets, :type => :string, :aliases => ["--datasets"], :desc => "'[{\"id\": \"dataset id\", \"commit\": \"commit id\", \"query\": \"query name\", \"tree_only\": true]'", :default => ""
     method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
     method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
     method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
@@ -3433,6 +3469,7 @@ module Cnvrg
       log_start(__method__, args, options)
       local = options["local"]
       notebook_dir = options["notebook_dir"]
+      datasets = options["datasets"]
       kernel = options["kernel"]
       image = options["image"] || nil
       data = options["data"]
@@ -3457,7 +3494,7 @@ module Cnvrg
         instance_type = get_instance_type(instances)
 
         invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type, :image => image,
-               :data => data, :data_commit => data_commit , :dataset_only_tree => dataset_only_tree, :data_query => data_query
+               :data => data, :data_commit => data_commit , :dataset_only_tree => dataset_only_tree, :data_query => data_query, :datasets => datasets
         return
 
       end
@@ -3562,6 +3599,7 @@ module Cnvrg
     method_option :dataset_only_tree, :type => :boolean, :aliases => [ "--dataset_only_tree"], :default => false
     method_option :data_query, :type => :string, :aliases => ["-q","--data_query"], :default => ""
     method_option :image, :type => :string, :aliases => ["--image"], :default => nil
+    method_option :datasets, :type => :string, :aliases => ["--datasets"], :desc => "'[{\"id\": \"dataset id\", \"commit\": \"commit id\", \"query\": \"query name\", \"tree_only\": true]'", :default => ""
 
     def remote_notebook()
       verify_logged_in(true)
@@ -3569,6 +3607,7 @@ module Cnvrg
 
       working_dir = is_cnvrg_dir()
       instance_type = options["machine_type"] || nil
+      datasets = options["datasets"]
       data = options["data"]
       data_commit = options["data_commit"]
       commit = options["commit"]
@@ -3599,7 +3638,7 @@ module Cnvrg
         end
         invoke :sync, [false], []
         slug = ""
-        res = exp.remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options,data_query, image)
+        res = exp.remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options,data_query, image, datasets)
         if Cnvrg::CLI.is_response_success(res)
           slug = res["result"]["notebook_url"]
           log_message("#{Helpers.checkmark} Notebook is ready: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/notebook_sessions/show/#{slug}", Thor::Shell::Color::GREEN)

data/lib/cnvrg/data.rb

@@ -82,6 +82,7 @@ module Cnvrg
     method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => nil
     method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => nil
     method_option :read, :type => :boolean, :aliases => ["-r", "--read"], :default => false
+    method_option :remote, :type => :boolean, :aliases => ["-h", "--remote"], :default => false
 
     def clone(dataset_url)
       cli = Cnvrg::CLI.new()
@@ -89,7 +90,17 @@ module Cnvrg
       commit =options[:commit]
       query =options[:query]
       read = options[:read]
-      cli.clone_data(dataset_url, only_tree=only_tree,commit=commit, query=query, read=read)
+      remote = options[:remote]
+      cli.clone_data(dataset_url, only_tree=only_tree,commit=commit, query=query, read=read, remote=remote)
+    end
+
+    desc 'data verify', 'verify datasets', :hide => true
+    method_option :timeout, :type => :numeric, :aliases => ["-t", "--timeout"], :desc => "Time to wait before returning final answer", :default => 15
+
+    def verify(*dataset_titles)
+      cli = Cnvrg::CLI.new()
+      timeout =options[:timeout]
+      cli.verify_datasets(dataset_titles, timeout)
     end
 
     desc 'data set --url=dataset_url', 'Set dataset url to other url'

data/lib/cnvrg/dataset.rb

@@ -250,6 +250,28 @@ module Cnvrg
       return true
     end
 
+    def self.verify_datasets(dataset_titles, timeout=100)
+      start_time = Time.now.to_i
+      Cnvrg::Logger.log_info("Verifying datasets #{dataset_titles}")
+      Cnvrg::Logger.log_info("Timeout is  #{timeout}")
+      while true
+        begin
+          current_time = Time.now.to_i
+          return false if timeout < current_time - start_time
+          all_are_ready = dataset_titles.all? do |dataset_title|
+            config = YAML.load_file("/data/#{dataset_title}/.cnvrg/config.yml")
+            config[:success] == true
+          end
+          return true if all_are_ready
+          Cnvrg::Logger.log_info("Sleeping..")
+          sleep 10
+        rescue => e
+          Cnvrg::Logger.log_info("Got error")
+          Cnvrg::Logger.log_error(e)
+          sleep 10
+        end
+      end
+    end
 
     def clone( commit)
 
@@ -257,11 +279,14 @@ module Cnvrg
       return response
     end
 
-    def self.clone(owner, dataset_name, dataset_slug)
+    def self.clone(owner, dataset_name, dataset_slug, remote=false)
 
       begin
-        list_dirs = [ dataset_name, "#{dataset_name}/.cnvrg"
-        ]
+        list_dirs = []
+        if !remote
+          list_dirs <<  dataset_name
+        end
+        list_dirs << "#{dataset_name}/.cnvrg"
         list_files = [
             "#{dataset_name}/.cnvrg/config.yml"
         ]
@@ -281,6 +306,15 @@ module Cnvrg
       return true
     end
 
+    def write_success(in_folder = false)
+      file_path = ".cnvrg/config.yml"
+      file_path = "#{@slug}/" + file_path if !in_folder
+      if File.exist?(file_path)
+        File.open(file_path, "a") { |f| f.puts(":success: true") }
+      end
+    rescue
+    end
+
     def self.init_container(owner, dataset_slug,dataset_name)
 
       cnvrgignore = Helpers.cnvrgignore_content

data/lib/cnvrg/experiment.rb

@@ -90,12 +90,13 @@ module Cnvrg
 
     def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp, grid,path_to_cmd,data, data_commit,periodic_sync,
                     sync_before_terminate, max_time, ds_sync_options=0,output_dir=nil,data_query=nil,
-                    git_commit=nil, git_branch=nil, restart_if_stuck=nil, local_folders=nil,title=nil)
+                    git_commit=nil, git_branch=nil, restart_if_stuck=nil, local_folders=nil,title=nil, datasets=nil)
       response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
                                                                                                             commit_sha1: commit_to_run,
                                                                                                             instance_type: instance_type,
                                                                                                             scheduling_query:scheduling_query,
                                                                                                             local_timestamp:local_timestamp,
+                                                                                                            datasets: datasets,
                                                                                                             grid: grid,
                                                                                                             path_to_cmd:path_to_cmd,dataset_slug:data,
                                                                                                             dataset_commit: data_commit,max_time:max_time,
@@ -106,9 +107,10 @@ module Cnvrg
 
       return response
     end
-    def remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options=0,data_query=nil, image = nil)
+    def remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options=0,data_query=nil, image = nil, datasets = nil)
       response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/notebook/remote", 'POST', {instance_type: instance_type,dataset_slug:data,
                                                                                                           dataset_commit: data_commit,image_slug:image,
+                                                                                                          datasets: datasets,
                                                                                                           commit:commit,notebook_type:notebook_type,dataset_sync_options:ds_sync_options,
                                                                                                           dataset_query:data_query})
       return response

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '0.9.9.6'
+  VERSION = '1.0'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.9.9.6
+  version: '1.0'
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-01-10 00:00:00.000000000 Z
+date: 2019-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler