cnvrg 1.6.0.6 → 1.6.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eec6f04b04951d9939b8a6b139fb4566d524b68ca01ace06e7f2be48fa92b6bb
-  data.tar.gz: 1aa365dca2cae40afb8dc1b8bddfd0029e9f376c708c8bed2ba66741094a2016
+  metadata.gz: 74e632845d8a6f6ca39492119e8efa15f90aaa76f66737336da95ef24b89b873
+  data.tar.gz: 79a573e7bb4dcb63dadb89dba6c5bba7c16473e8fcc688ad27c64ff1c0b58dcc
 SHA512:
-  metadata.gz: 6fb5ccb7658268ca0ccfcad00a1e490468f91bf77de5313b61aead3885fe64c4c2f50248a89a24488cf016f8cc53fa53a98a5d3f891deb6740dfecd4c682adec
-  data.tar.gz: 787f8aa03cfc15989ba97ee583144e83070aa31d774886269af1910d15a2bd81338a2ae4e1a44b60a1a8b60de5b2db9cc995df0cda1f12f046af9e7fefa67283
+  metadata.gz: 6308aba4c811a524e84ffa9d1a9519af3fe74561ac1adb20b9ad4db8d78ab91b834549f37a29c258f5775bf59d46328fd956ced36c9877ccf2c678d4641643bd
+  data.tar.gz: a4e5526fa519c995833b52d8ba5147a1cda9373f634dff60aa6435422a6486c7b599f0a27427a68d20166c5dc81f2aa54badc686186baec26483a3fff41f08bc

data/lib/cnvrg/cli.rb

@@ -187,31 +187,6 @@ module Cnvrg
       self.log_handler
     end
 
-    desc "", "", :hide => true
-
-    def test
-      # image_settings = {
-      #     'Image' => "cnvrg:latest",
-      #     'User' => 'ds',
-      #     'Cmd' => '/home/ds/run_ipython.sh',
-      #     'ExposedPorts' => {``
-      #         '80/tcp' => {},
-      #     },
-      #     'HostConfig' => {
-      #         'PortBindings' => {
-      #             '80/tcp' => [
-      #                 {'HostPort' => "7654", 'HostIp' => 'localhost'}
-      #             ],
-      #         },
-      #     },
-      # }
-      # container = Docker::Container.get('b4d64bf83f41')
-      # s = "/leah/1/2/3/4/5"
-      # command = ["/bin/bash","-lc","sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{s}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
-      # puts container.exec(command, tty: true)
-    end
-
-
     desc 'version', 'Prints cnvrg current version'
 
     def version
@@ -810,6 +785,7 @@ module Cnvrg
         exit(1)
       end
     end
+
     desc 'data delete', 'delete dataset', :hide => true
     def delete_data(dataset_slug)
       begin
@@ -1740,7 +1716,7 @@ module Cnvrg
       verify_logged_in(false)
       log_start(__method__, args, options)
       working_dir = is_cnvrg_dir()
-      list_to_del = [working_dir + "/.cnvrgignore", working_dir + "/.cnvrg"]
+      list_to_del = [working_dir + "/.cnvrg"]
       FileUtils.rm_rf list_to_del
     end
 
@@ -1753,6 +1729,7 @@ module Cnvrg
 
       clone_resp = Project.clone_dir_remote(slug, owner, slug,true)
       idx_status = Project.new(get_project_home).generate_idx
+      FileUtils.mkdir_p File.join(get_project_home, ENV['CNVRG_OUTPUT_DIR']) if ENV['CNVRG_OUTPUT_DIR'].present?
       @executer = Cnvrg::Helpers::Executer.get_executer
       if @executer.present?
         @executer.update_git_commit
@@ -2929,6 +2906,8 @@ module Cnvrg
     method_option :git_branch, :type => :string, :aliases => [ "--git_branch"], :default => nil
     method_option :restart_if_stuck, :type => :boolean, :aliases => ["--restart","--restart_if_stuck"], :default => nil
     method_option :local_folders, :type => :string, :aliases => ["--local_folders"], :default => nil
+    method_option :prerun, :type => :boolean, :aliases => ["-p", "--prerun"], :default => true
+    method_option :requirements, :type => :boolean, :aliases => ["-r", "--requirements"], :default => true
 
     def run(*cmd)
       verify_logged_in(true)
@@ -2957,6 +2936,8 @@ module Cnvrg
       output_dir = options["output_dir"]
       data_query = options["data_query"]
       local_folders = options["local_folders"]
+      prerun = options["prerun"]
+      requirements = options["requirements"]
 
       if !data.present? and data_query.present?
         log_message("Please provide data with data_query", Thor::Shell::Color::RED)
@@ -3006,7 +2987,7 @@ module Cnvrg
                 :max_time => max_time,
                :periodic_sync => periodic_sync, :dataset_only_tree=> dataset_only_tree,
                :output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch,
-               :restart_if_stuck =>restart_if_stuck, :local_folders => local_folders, :datasets => datasets
+               :restart_if_stuck =>restart_if_stuck, :local_folders => local_folders, :datasets => datasets, :prerun => prerun, :requirements => requirements
         return
       end
 
@@ -3309,6 +3290,8 @@ module Cnvrg
     method_option :restart_if_stuck, :type => :boolean, :aliases => ["--restart"], :default => nil
     method_option :local_folders, :type => :string, :aliases => ["--local_folders"], :default => nil
     method_option :datasets, :type => :string, :aliases => ["--datasets"], :default => nil
+    method_option :prerun, :type => :boolean, :aliases => ["-p", "--prerun"], :default => true
+    method_option :requirements, :type => :boolean, :aliases => ["-r", "--requirements"], :default => true
 
     def exec_remote(*cmd)
 
@@ -3326,6 +3309,8 @@ module Cnvrg
         data_query = options["data_query"] || nil
         sync_before = options["sync_before"]
         force = options["force"]
+        prerun = options["prerun"]
+        requirements = options["requirements"]
         max_time = options["max_time"]
         if !max_time.nil? and !max_time.empty?
           max_time = max_time.to_i
@@ -3381,7 +3366,7 @@ module Cnvrg
         local_folders_options = options["local_folders"]
         options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "title",
                              "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree",
-                             "data_query", "git_commit","git_branch", "restart_if_stuck","local_folders","output_dir", "commit", "datasets" )
+                             "data_query", "git_commit","git_branch", "restart_if_stuck","local_folders","output_dir", "commit", "datasets", "requirements", "prerun" )
         exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
         command = "#{exec_options} #{remote}  #{upload_output_option} #{cmd.flatten.join(" ")}"
         commit_to_run = options["commit"] || nil
@@ -3452,7 +3437,7 @@ module Cnvrg
 
         res = exp.exec_remote(command, commit_to_run, instance_type, image, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
                               periodic_sync, sync_before_terminate, max_time, ds_sync_options,output_dir,
-                              data_query, git_commit, git_branch, restart_if_stuck,local_folders_options, title, datasets)
+                              data_query, git_commit, git_branch, restart_if_stuck,local_folders_options, title, datasets, prerun: prerun, requirements: requirements)
         if Cnvrg::CLI.is_response_success(res)
           check = Helpers.checkmark()
           str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"

data/lib/cnvrg/data.rb

@@ -13,6 +13,12 @@ end
 module Cnvrg
   class Data < SubCommandBase
     class_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
+
+    module ConfigValidation
+      SUCCESS = "success"
+      FAILED = "failed"
+    end
+
     desc "data init", "Set current directory as dataset directory"
     method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
     method_option :bucket, :type => :string, :aliases => ["-b", "--bucket"], :default => ""
@@ -22,6 +28,44 @@ module Cnvrg
       bucket = options["bucket"]
       cli.init_data(public, bucket: bucket)
     end
+
+    desc "data link", "Set current directory as dataset directory"
+    def link(dataset_slug=nil)
+      begin
+        raise Exception.new("Please enter dataset name") if dataset_slug.blank?
+        cli = Cnvrg::CLI.new()
+        cli.verify_logged_in(false)
+        cli.log_start(__method__, args, options)
+
+        config_validation = Dataset.validate_config
+        if config_validation[:validation] == Data::ConfigValidation::SUCCESS
+          cli.log_message(config_validation[:message])
+          return
+        else
+          Cnvrg::Logger.log_error_message(config_validation)
+          FileUtils.rmtree(".cnvrg")
+        end
+
+        cli.log_message("Linking dataset: #{dataset_slug} to the current directory", Thor::Shell::Color::BLUE)
+        owner = CLI.get_owner
+        success = Dataset.link_dataset(owner: owner, slug: dataset_slug)
+        if success
+          cli.log_message("Dataset: #{dataset_slug} linked successfully to the current directory")
+        else
+          cli.log_message("Linking failed\nAborting", Thor::Shell::Color::RED)
+        end
+      rescue => e
+        Cnvrg::Logger.log_error(e)
+        cli.log_message("Aborting\n#{e.message}", Thor::Shell::Color::RED)
+        exit(1)
+      rescue Exception => e
+        Cnvrg::Logger.log_error(e)
+        cli.log_message("Aborting\n#{e.message}", Thor::Shell::Color::RED)
+        exit(1)
+      end
+    end
+
+
     desc "data upload", "Upload files from local dataset directory to remote server"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"

data/lib/cnvrg/dataset.rb

@@ -311,6 +311,37 @@ module Cnvrg
       return true
     end
 
+    def self.link_dataset(owner: nil, slug: nil)
+      begin
+        return false if owner.blank? or slug.blank?
+
+        response = Cnvrg::API.request("users/#{owner}/datasets/#{slug}", 'GET')
+        success = Cnvrg::CLI.is_response_success(response, false)
+        return unless success
+        result = response["result"]
+
+        config = {dataset_name: result["title"],
+                  dataset_slug: result["slug"],
+                  owner: owner}
+
+        list_dirs = [".cnvrg"]
+        list_files = [".cnvrg/config.yml"]
+
+
+        FileUtils.mkdir_p list_dirs
+        FileUtils.touch list_files
+        File.open(".cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
+
+        # Write init IDX that contain init commit sha1 so the user will be able to doing actions on the dataset
+        # Relevant only for new server
+        dataset = Dataset.new(Dir.pwd)
+        sha1 = result["init_commit_sha1"]
+        dataset.write_idx({}, sha1)
+        true
+      rescue => e
+        raise Exception.new(e)
+      end
+    end
 
     def self.blank_clone(owner, dataset_name, dataset_slug)
       list_dirs = [".cnvrg"
@@ -796,5 +827,28 @@ module Cnvrg
       # CLI.is_response_success(response)
     end
 
+    def self.validate_config
+      ## check that the .cnvrg folder exists:
+      dot_cnvrg_exists = Dir[".cnvrg"].present?
+      return {validation: Data::ConfigValidation::FAILED, message: ".cnvrg folder does not exists"} if not dot_cnvrg_exists
+
+      ## check that the config.yml exists:
+      config_file_exists = Dir[".cnvrg/*"].include? ".cnvrg/config.yml"
+      return {validation: Data::ConfigValidation::FAILED, message: "config.yml exists"} if not config_file_exists
+
+      ## check that the config.yml file not empty:
+      config = YAML.load_file("#{Dir.getwd}/.cnvrg/config.yml")
+      return {validation: Data::ConfigValidation::FAILED, message: "config.yml is empty"} if not config
+
+      ## check that config.yml is valid:
+      title = config[:dataset_name]
+      slug = config[:dataset_slug]
+      owner = config[:owner]
+      return {validation: Data::ConfigValidation::FAILED, message: "config.yml is not valid or some keys are missing"} if title.blank? or slug.blank? or owner.blank?
+
+      ## everything OK:
+      return {validation: Data::ConfigValidation::SUCCESS, message: "Directory is already linked to #{slug}"}
+    end
+
   end
 end

data/lib/cnvrg/experiment.rb

@@ -90,7 +90,7 @@ module Cnvrg
 
     def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp, grid,path_to_cmd,data, data_commit,periodic_sync,
                     sync_before_terminate, max_time, ds_sync_options=0,output_dir=nil,data_query=nil,
-                    git_commit=nil, git_branch=nil, restart_if_stuck=nil, local_folders=nil,title=nil, datasets=nil)
+                    git_commit=nil, git_branch=nil, restart_if_stuck=nil, local_folders=nil,title=nil, datasets=nil, prerun: true, requirements: true)
       response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
                                                                                                             commit_sha1: commit_to_run,
                                                                                                             instance_type: instance_type,
@@ -103,7 +103,8 @@ module Cnvrg
                                                                                                             periodic_sync:periodic_sync, sync_before_terminate:sync_before_terminate,
                                                                                                             dataset_sync_options:ds_sync_options,output_dir:output_dir,
                                                                                                             dataset_query:data_query,git_commit:git_commit,git_branch:git_branch,
-                                                                                                            restart_if_stuck:restart_if_stuck, local_folders: local_folders, title:title })
+                                                                                                            restart_if_stuck:restart_if_stuck, local_folders: local_folders, title:title,
+                                                                                                            prerun: prerun, requirements: requirements})
 
       return response
     end

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '1.6.0.6'
+  VERSION = '1.6.0.7'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.6.0.6
+  version: 1.6.0.7
 platform: ruby
 authors:
 - Yochay Ettun
@@ -476,7 +476,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.