cnvrg 0.7.3 → 0.7.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ad8377415af407d173ec92d1b9fc7711409cbaaa
-  data.tar.gz: 846a4bcc0e20554272356df964275ec51ef18e28
+  metadata.gz: 34ca8a94802b76adb5ab7b8d2c2d6a2c10ff3824
+  data.tar.gz: 4dc880d0b84ccb072ac878c8c815be43a0689446
 SHA512:
-  metadata.gz: 108d721b0571c652fe7363fa928519a4d36b0685af91966656506a3984cbc25856992d1e479ab4c1d14fbf17148c3ecfff547ff6b002cfb2bd754e2925b1cd76
-  data.tar.gz: f16687af8d0ebc45acb112cdf73c43d599c509cc4af977c61220742ba4444ca5f486a7591a40cfcc1421e2c205d1a7b6f293dec7570eaf069c661869ac5a5f65
+  metadata.gz: 14a5b0e7db3d5bddd7449a0da6583acf5fc069a08443e3889dd754cd9a27fe9b9de10608b1df85f1b8602ac853bcb29e71498f252c9384b515c104227edf4ec0
+  data.tar.gz: 2646237fecf5996079985e1138a493f1dee678423c2ce39455e4e03c3ff52723823ebd79ee9e94b64bafc260b6bb36822852b3cc0e969e7b86a4988822bd3a56

data/cnvrg.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'open4', '~> 1.3', '>= 1.3.4'
     spec.add_runtime_dependency 'highline', '~> 1.7', '>= 1.7.8'
     spec.add_runtime_dependency 'thor', '~> 0.19.0','>=0.19.1'
-    spec.add_runtime_dependency 'aws-sdk', '2.10.19'
+    spec.add_runtime_dependency 'aws-sdk', '~> 2'
     spec.add_runtime_dependency 'sucker_punch', '~> 2.0'
     spec.add_runtime_dependency 'urlcrypt', '~> 0.1.1'
     spec.add_runtime_dependency 'parallel', '~> 1.12.0'

data/lib/cnvrg/cli.rb

@@ -838,6 +838,7 @@ module Cnvrg
         project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
         slug = url_parts[project_index + 1]
         owner = url_parts[project_index - 1]
+        log_message("#{Time.now}: Fetching Dataset files list", Thor::Shell::Color::GREEN)
 
         response = Cnvrg::API.request("users/#{owner}/datasets/#{slug}/clone", 'POST',{ commit: commit, query:query})
         Cnvrg::CLI.is_response_success(response,true)
@@ -872,9 +873,11 @@ module Cnvrg
                                            :starting_at => 0,
                                            :total => files_count,
                                            :autofinish => true)
-          
+
           while files['keys'].length > 0
-            @files.download_multiple_files_s3(files, dataset_home, progressbar: progressbar)
+
+            @files.download_multiple_files_s3(files, dataset_home, progressbar: progressbar, read_only: read)
+
             downloaded_files += files['keys'].length
             files = @files.get_clone_chunk(commit: commit, latest_id: files['latest'])
           end
@@ -884,6 +887,7 @@ module Cnvrg
             if !read
               @dataset.write_idx(nil, commit) #nil means, generate idx
             end
+            log_message("#{Time.now} Clone finished successfully", Thor::Shell::Color::GREEN)
 
             log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
           end
@@ -3356,7 +3360,7 @@ module Cnvrg
     method_option :upload_output, :type => :string, :aliases => ["-uo", "--upload_output"], :default => ""
     method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
     method_option :schedule, :type => :string, :aliases => ["-s", "--schedule"], :default => ""
-    method_option :image, :type => :string, :aliases => ["--image"], :default => ""
+    method_option :image, :type => :string, :aliases => ["--image"], :default => nil
     method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
     method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
     method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
@@ -3385,7 +3389,7 @@ module Cnvrg
       upload_output = options["upload_output"]
       local = options["local"]
       schedule = options["schedule"]
-      image = options["image"]
+      image = options["image"] || nil
       grid = options["grid"]
       data = options["data"]
       data_commit = options["data_commit"]
@@ -3848,7 +3852,6 @@ module Cnvrg
         output_dir = options["output_dir"] || nil
         git_commit = options["git_commit"]
         git_branch = options["git_branch"]
-
         options_hash = Hash[options]
         local_folders_options = options["local_folders"]
         options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit",
@@ -3867,22 +3870,11 @@ module Cnvrg
         if project.is_git and output_dir.blank?
           output_dir  = "output"
         end
-        choose_image = options["image"]
-
-        if !choose_image.nil? and !choose_image.empty?
-          invoke :set_image, [choose_image]
+        image = options["image"] || nil
+        if image.blank?
+          image  = "cnvrg"
         end
-        image = is_project_with_docker(working_dir)
-        if !image or !image.is_docker
-          # say "Couldn't find image related to project", Thor::Shell::Color::RED
 
-          image_slug = "cnvrg"
-          if instance_type.eql? "gpu" or instance_type.eql? "gpuxl"
-            image_slug = "cnvrg_gpu"
-          end
-        else
-          image_slug = image.image_slug
-        end
         forced_commit = nil
         if sync_before and !project.is_git
           if force
@@ -3923,7 +3915,7 @@ module Cnvrg
         end
         commit_to_run = commit_to_run.presence || project.last_local_commit
 
-        res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
+        res = exp.exec_remote(command, commit_to_run, instance_type, image, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
                               periodic_sync, sync_before_terminate, max_time, ds_sync_options,output_dir,
                               data_query, git_commit, git_branch, restart_if_stuck,local_folders_options, title )
         if Cnvrg::CLI.is_response_success(res)
@@ -4042,7 +4034,7 @@ module Cnvrg
     method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
     method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
     method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
-    method_option :image, :type => :string, :aliases => ["-i", "--image"], :default => ""
+    method_option :image, :type => :string, :aliases => ["-i", "--image"], :default => nil
     method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
     method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
     method_option :dataset_only_tree, :type => :boolean, :aliases => [ "--dataset_only_tree"], :default => false
@@ -4056,7 +4048,7 @@ module Cnvrg
       local = options["local"]
       notebook_dir = options["notebook_dir"]
       kernel = options["kernel"]
-      image = options["image"]
+      image = options["image"] || nil
       data = options["data"]
       data_commit = options["data_commit"]
       dataset_only_tree = options["dataset_only_tree"]
@@ -4198,6 +4190,7 @@ module Cnvrg
     method_option :commit, :type => :string, :aliases => ["--commit"], :default => ""
     method_option :dataset_only_tree, :type => :boolean, :aliases => [ "--dataset_only_tree"], :default => false
     method_option :data_query, :type => :string, :aliases => ["-q","--data_query"], :default => ""
+    method_option :image, :type => :string, :aliases => ["--image"], :default => nil
 
     def remote_notebook()
       verify_logged_in(true)
@@ -4210,6 +4203,7 @@ module Cnvrg
       commit = options["commit"]
       notebook_type = options["notebook_type"]
       dataset_only_tree = options["dataset_only_tree"]
+      image = options["image"]
       ds_sync_options = 0
       if dataset_only_tree
         ds_sync_options = 1
@@ -4234,7 +4228,7 @@ module Cnvrg
         end
         invoke :sync, [false], []
         slug = ""
-        res = exp.remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options,data_query)
+        res = exp.remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options,data_query, image)
         if Cnvrg::CLI.is_response_success(res)
           slug = res["result"]["notebook_url"]
           log_message("#{Helpers.checkmark} Notebook is ready: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/notebook_sessions/show/#{slug}", Thor::Shell::Color::GREEN)

data/lib/cnvrg/datafiles.rb

@@ -872,7 +872,7 @@ module Cnvrg
         next unless Cnvrg::CLI.is_response_success(response, false) #trying to api request 5 times.
         files_to_download = response['files']
         data_home = "#{Dir.pwd}/#{response['name']}"
-        res = download_multiple_files_s3(files_to_download, data_home, conflict: conflict)
+        res = download_multiple_files_s3(files_to_download, data_home, conflict: conflict, read_only: false)
         next unless res.is_success? #try again..
         return files_to_download['keys'].length
       end
@@ -890,7 +890,7 @@ module Cnvrg
           data_home = "#{Dir.pwd}/#{response['name']}"
           last_chunk_size = files['keys'].length
           break if last_chunk_size == 0
-          res = download_multiple_files_s3(files, data_home)
+          res = download_multiple_files_s3(files, data_home, read_only: false)
           overall += last_chunk_size
           q[:latest] = files['latest']
         else
@@ -907,7 +907,7 @@ module Cnvrg
       
     end
 
-    def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil)
+    def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false)
       begin
         props = Cnvrg::Helpers.get_s3_props(files)
         client = props[:client]
@@ -915,11 +915,17 @@ module Cnvrg
         key = props[:key]
         bucket = props[:bucket]
         download_succ_count = 0
+        if read_only
+          in_threads = 1000
+        else
+          in_threads = ParallelThreads
+        end
         parallel_options = {
-            in_threads: Cnvrg::Helpers.parallel_threads,
+            in_threads: in_threads,
             isolation: true
         }
         Parallel.map(files["keys"], parallel_options) do |f|
+
           file_path = f["name"]
           progressbar.progress += 1 if progressbar.present?
           if file_path.end_with? "/"
@@ -943,6 +949,7 @@ module Cnvrg
               resp = client.get_object({bucket:bucket,
                                         key:file_key}, target: file)
             end
+
             if resp
               download_succ_count +=1
             else

data/lib/cnvrg/experiment.rb

@@ -106,9 +106,9 @@ module Cnvrg
 
       return response
     end
-    def remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options=0,data_query=nil)
+    def remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options=0,data_query=nil, image = nil)
       response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/notebook/remote", 'POST', {instance_type: instance_type,dataset_slug:data,
-                                                                                                          dataset_commit: data_commit,
+                                                                                                          dataset_commit: data_commit,image_slug:image,
                                                                                                           commit:commit,notebook_type:notebook_type,dataset_sync_options:ds_sync_options,
                                                                                                           dataset_query:data_query})
       return response

data/lib/cnvrg/helpers.rb

@@ -2,7 +2,7 @@ module Cnvrg
   module Helpers
 
     extend self
-    ParallelThreads ||= 10
+    ParallelThreads ||= 20
     def parallel_threads()
       return ParallelThreads
     end

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '0.7.3'
+  VERSION = '0.7.4'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.7.3
+  version: 0.7.4
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-16 00:00:00.000000000 Z
+date: 2018-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -207,16 +207,16 @@ dependencies:
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.10.19
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.10.19
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: sucker_punch
   requirement: !ruby/object:Gem::Requirement