cnvrg 0.3.6 → 0.3.8

data/lib/cnvrg/dataset.rb

@@ -5,7 +5,7 @@ module Cnvrg
 
     RemoteURL ||= "https://cnvrg.io"
     IDXParallelThreads ||= 15
-
+    IDXParallelProcesses ||= Parallel.processor_count
 
     def initialize(project_home)
       begin
@@ -18,7 +18,12 @@ module Cnvrg
       rescue => e
 
       end
+    end
 
+
+    def self.delete(dataset_slug,owner)
+      response = Cnvrg::API.request("users/#{owner}/datasets/#{dataset_slug}/delete", 'DELETE')
+      return response
     end
 
     def last_local_commit
@@ -50,6 +55,26 @@ module Cnvrg
       "#{url}/#{self.owner}/projects/#{self.slug}"
     end
 
+    def self.verify_cnvrgignore_exist(dataset_name,remote)
+      path = ".cnvrgignore"
+      if !File.exist? path
+        path = "#{dataset_name}/.cnvrgignore"
+      end
+      ignore_exits = File.exist? path
+      if !ignore_exits
+        begin
+          list_files = [
+              path
+          ]
+          FileUtils.touch list_files
+          cnvrgignore = Helpers.cnvrgignore_content
+          File.open(path, "w+") { |f| f.write cnvrgignore }
+        rescue => e
+          return false
+        end
+
+      end
+    end
     def update_ignore_list(new_ignore)
 
       if new_ignore.nil? or new_ignore.empty?
@@ -97,9 +122,14 @@ module Cnvrg
         list_dirs = [".cnvrg"
         ]
         list_files = [
-            ".cnvrgignore",
             ".cnvrg/config.yml"
         ]
+        create_ignore = false
+        if !File.exist? ".cnvrgignore"
+          list_files << ".cnvrgignore"
+          create_ignore  = true
+        end
+
 
         cnvrgignore = Helpers.cnvrgignore_content
         begin
@@ -111,22 +141,30 @@ module Cnvrg
           config = {dataset_name: dataset_name,
                     dataset_slug: dataset_slug,
                     owner: owner}
+
           FileUtils.mkdir_p list_dirs
           FileUtils.touch list_files
           File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
-          File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
+          File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless !create_ignore
         rescue => e
           return false
         end
         return true
     end
+
+
+    def clone( commit)
+
+
+      return response
+    end
+
     def self.clone(owner, dataset_name,dataset_slug)
 
       begin
         list_dirs = [ dataset_name, "#{dataset_name}/.cnvrg"
         ]
         list_files = [
-            "#{dataset_name}/.cnvrgignore",
             "#{dataset_name}/.cnvrg/config.yml"
         ]
 
@@ -135,11 +173,9 @@ module Cnvrg
                   owner: owner}
 
 
-        cnvrgignore = Helpers.cnvrgignore_content
         FileUtils.mkdir_p list_dirs
         FileUtils.touch list_files
         File.open("#{dataset_name}/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
-        File.open("#{dataset_name}/.cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
       rescue => e
         puts "Exception in clone request:#{e.message}"
         return false
@@ -189,11 +225,8 @@ module Cnvrg
 
       tree_idx = Hash.new(0)
 
-      list = Dir.glob("#{self.local_path}/**/**", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) }
-    if list_ignore.nil?
+      list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~/^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~/^#{self.local_path}\/\.cnvrgignore/) }
       list_ignore = self.get_ignore_list()
-    end
-
       Parallel.map(list, in_threads: IDXParallelThreads) do |e|
         label = e.gsub(self.local_path + "/", "")
         if File.directory? e
@@ -215,11 +248,16 @@ module Cnvrg
           end
         end
       end
+      if !old_idx.nil? and !old_idx[:next_commit].nil? and  !old_idx[:next_commit].empty?
+        idx = {commit: old_idx.to_h[:commit], tree: tree_idx, next_commit:old_idx[:next_commit] }
+      else
+        idx = {commit: old_idx.to_h[:commit], tree: tree_idx}
 
-      idx = {commit: old_idx.to_h[:commit], tree: tree_idx}
+      end
+      idx_yaml = idx.to_yaml
 
-      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
-      return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_yaml }
+      return idx
     end
     def create_volume
       response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/volumes/create", 'POST')
@@ -233,17 +271,45 @@ module Cnvrg
       CLI.is_response_success(response,false)
       return response
     end
-    def compare_idx(new_branch, commit=last_local_commit,local_idx=nil)
+    def compare_idx(new_branch, commit=last_local_commit,local_idx=nil,force = false, next_commit = nil)
+      if local_idx.nil?
+        local_idx =  self.generate_idx
+      end
+      ignore_list = self.get_ignore_list()
+      if force
+        added = []
+        if local_idx[:tree]
+          added << local_idx[:tree].keys
+          added.flatten!
+        end
 
+        response ={"result"=> {"commit"=>next_commit,"tree"=> {"added"=> added,
+                                                       "updated_on_server"=> [],
+                                                       "updated_on_local"=> [],
+                                                       "deleted"=> [],
+                                                       "conflicts"=> []} } }
+        return response
+
+      end
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status", 'POST', {idx: local_idx, new_branch: new_branch, current_commit: commit, ignore:ignore_list, next_commit: next_commit})
+      CLI.is_response_success(response,false)
+      return response
+    end
+    def compare_idx_download(commit=last_local_commit,local_idx=nil,all=false)
       if local_idx.nil?
         local_idx =  self.generate_idx
       end
       ignore_list = self.send_ignore_list()
-
-      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status", 'POST', {idx: local_idx, new_branch: new_branch, current_commit: commit, ignore:ignore_list})
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/download_status", 'POST', {idx: local_idx,  current_commit: commit, ignore:ignore_list,all_files:all})
       CLI.is_response_success(response,false)
       return response
     end
+    def current_status(new_branch)
+        commit=last_local_commit
+        response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status_current", 'POST', {current_commit: commit,new_branch: new_branch})
+        CLI.is_response_success(response,true)
+        return response
+    end
     def send_ignore_list()
       begin
         ignore_list = []
@@ -277,7 +343,45 @@ module Cnvrg
       CLI.is_response_success(response,false)
       return response
     end
+    def set_next_commit(commit_sha1)
+      if  !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
+        idx_hash = Hash.new()
+        idx_hash[:commit] = ""
+        idx_hash[:tree] = ""
+      else
+        idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
 
+      end
+      idx_hash[:next_commit] = commit_sha1
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
+      return true
+
+    end
+    def get_next_commit()
+      if  !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
+        return nil
+      end
+      idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+      return idx_hash[:next_commit]
+
+    end
+    def remove_next_commit()
+      if  !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
+        return nil
+      end
+      idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+      idx = Hash.new()
+      idx[:commit] = idx_hash[:next_commit]
+      idx[:tree] = idx_hash[:tree]
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
+
+
+    end
+    def list_files(commit, json)
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/list", 'GET', {commit_sha1:commit, as_json:json})
+      CLI.is_response_success(response,true)
+      return response
+    end
     def compare_commit(commit)
         if commit.nil? or commit.empty?
           commit = last_local_commit
@@ -289,11 +393,14 @@ module Cnvrg
       end
 
       def update_idx_with_files_commits!(files, commit_time)
-
+        files.flatten!
         idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+        # idx_hash[:commit] = commit
+
         files.each do |path|
           idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
         end
+        idx_hash[:next_commit] = idx_hash[:next_commit]
         File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
 
         return true

data/lib/cnvrg/experiment.rb

@@ -83,7 +83,7 @@ module Cnvrg
 
     end
 
-    def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp, grid,path_to_cmd,data, data_commit)
+    def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp, grid,path_to_cmd,data, data_commit,periodic_sync, sync_before_terminate, max_time, ds_sync_options=0)
       response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
                                                                                                             commit_sha1: commit_to_run,
                                                                                                             instance_type: instance_type,
@@ -91,12 +91,15 @@ module Cnvrg
                                                                                                             local_timestamp:local_timestamp,
                                                                                                             grid: grid,
                                                                                                             path_to_cmd:path_to_cmd,dataset_slug:data,
-                                                                                                            dataset_commit: data_commit})
+                                                                                                            dataset_commit: data_commit,max_time:max_time,
+                                                                                                            periodic_sync:periodic_sync, sync_before_terminate:sync_before_terminate,
+                                                                                                            dataset_sync_options:ds_sync_options})
       return response
     end
-    def remote_notebook(instance_type, commit, data, data_commit, notebook_type)
+    def remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options=0)
       response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/notebook/remote", 'POST', {instance_type: instance_type,dataset_slug:data,
-                                                                                                          dataset_commit: data_commit,commit:commit,notebook_type:notebook_type})
+                                                                                                          dataset_commit: data_commit,
+                                                                                                          commit:commit,notebook_type:notebook_type,dataset_sync_options:ds_sync_options})
       return response
     end
 

data/lib/cnvrg/files.rb

@@ -59,8 +59,8 @@ module Cnvrg
                                                                                              file_name: file_name, log_date: log_date,
                                                                                              file_size: file_size, file_content_type: content_type})
       if Cnvrg::CLI.is_response_success(upload_resp, false)
-        path = upload_resp["result"]["path"]
-        s3_res = upload_small_files_s3(path, absolute_path, "text/plain")
+       s3_res = upload_large_files_s3(upload_resp, absolute_path)
+
       end
       if s3_res
         return true
@@ -130,8 +130,7 @@ module Cnvrg
                                                                                                   is_base: is_base})
         # puts upload_resp
         if Cnvrg::CLI.is_response_success(upload_resp, false)
-          path = upload_resp["result"]["path"]
-          s3_res = upload_small_files_s3(path, absolute_path, content_type)
+          s3_res = upload_large_files_s3(upload_resp, absolute_path)
           if s3_res
             commit_resp = Cnvrg::API.request("users/#{owner}/images/#{upload_resp["result"]["id"]}/" + "commit", 'GET')
             if Cnvrg::CLI.is_response_success(commit_resp, false)
@@ -161,6 +160,7 @@ module Cnvrg
                                                                                                   })
         # puts upload_resp
         if Cnvrg::CLI.is_response_success(upload_resp, false)
+          path = upload_resp["result"]["path"]
           s3_res = upload_large_files_s3(upload_resp, absolute_path)
           if s3_res
               return true
@@ -254,50 +254,14 @@ module Cnvrg
 
     def upload_large_files_s3(upload_resp, file_path)
       begin
-        # speed_uri = URI.parse("https://s3-us-west-2.amazonaws.com/cnvrgw2-dev/speedtest-cli")
-        # speed_http_object = Net::HTTP.new(speed_uri.host, speed_uri.port)
-        # speed_http_object.use_ssl = true if speed_uri.scheme == 'https'
-        # speed_request = Net::HTTP::Get.new("https://s3-us-west-2.amazonaws.com/cnvrgw2-dev/speedtest-cli")
-        #
-        # speed_body = ""
-        # speed_http_object.start do |http|
-        #   response = http.request speed_request
-        #   speed_body = response.read_body
-        # end
-        # speed_tmp = Tempfile.new('speed')
-        # speed_tmp << speed_body
-        # speed_tmp.flush
-        # speed_tmp.close
-        # count = 0
-        # sum = 0
-        # speed_res = `python #{speed_tmp.path} --json`
-        # if !speed_res.nil? and !speed_res.empty?
-        #   upload = JSON.parse(speed_res)["upload"]
-        #   if !upload.nil?
-        #     up_spped = (upload/1000000).round(2)
-        #   end
-        #
-        #
-        #   file_size = File.size(file_path)/1048576
-        #   est_up = ((file_size*8)/up_spped)/60
-        #   puts est_up
-        # end
-
-
           sts_path = upload_resp["result"]["path_sts"]
 
-          uri = URI.parse(sts_path)
-          http_object = Net::HTTP.new(uri.host, uri.port)
-          http_object.use_ssl = true if uri.scheme == 'https'
-          http_object.ssl_version="TLSv1"
-
-          request = Net::HTTP::Get.new(sts_path)
-
-          body = ""
-          http_object.start do |http|
-            response = http.request request
-            body = response.read_body
+          if !Helpers.is_verify_ssl
+            body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
+          else
+            body = open(sts_path).read
           end
+
           split = body.split("\n")
           key = split[0]
           iv = split[1]
@@ -310,18 +274,30 @@ module Cnvrg
           region =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["region"])
 
           bucket =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["bucket"])
-
+          is_s3 = upload_resp["result"]["is_s3"]
+          if is_s3 or is_s3.nil?
+            use_accelerate_endpoint =true
           client = Aws::S3::Client.new(
                 :access_key_id =>access,
                 :secret_access_key => secret,
                 :session_token => session,
                 :region => region,
-                :http_open_timeout => 60, :retry_limit => 20
-                                         )
+          :http_open_timeout => 60, :retry_limit => 20)
+          else
+            endpoint = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["endpoint"])
+            use_accelerate_endpoint = false
+            client = Aws::S3::Client.new(
+                :access_key_id =>access,
+                :secret_access_key => secret,
+                :region => region,
+                :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
+                :http_open_timeout => 60, :retry_limit => 20)
+          end
+
             s3 = Aws::S3::Resource.new(client: client)
             resp = s3.bucket(bucket).
                 object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
-                upload_file(file_path, {:use_accelerate_endpoint => true, :server_side_encryption => 'AES256'})
+                upload_file(file_path, {:use_accelerate_endpoint => use_accelerate_endpoint, :server_side_encryption => 'AES256'})
 
           return resp
 
@@ -344,6 +320,9 @@ module Cnvrg
         body = file.read
         begin
           Net::HTTP.start(url.host) do |http|
+            if !Helpers.is_verify_ssl
+              http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+            end
             http.send_request("PUT", url.request_uri, body, {
                 "content-type" => content_type,
             })
@@ -394,15 +373,10 @@ module Cnvrg
 
             absolute_path += ".conflict" if conflict
             sts_path = download_resp["result"]["path_sts"]
-            uri = URI.parse(sts_path)
-            http_object = Net::HTTP.new(uri.host, uri.port)
-            http_object.use_ssl = true if uri.scheme == 'https'
-            request = Net::HTTP::Get.new(sts_path)
-
-            body = ""
-            http_object.start do |http|
-              response = http.request request
-              body = response.read_body
+            if !Helpers.is_verify_ssl
+              body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
+            else
+              body = open(sts_path).read
             end
             split = body.split("\n")
             key = split[0]
@@ -416,18 +390,29 @@ module Cnvrg
             region =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["region"])
 
             bucket =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["bucket"])
-            key =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
+            file_key =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
+
+            is_s3 = download_resp["result"]["is_s3"]
+            if is_s3 or is_s3.nil?
+              client = Aws::S3::Client.new(
+                  :access_key_id =>access,
+                  :secret_access_key => secret,
+                  :session_token => session,
+                  :region => region,
+                  :http_open_timeout => 60, :retry_limit => 20)
+            else
+              endpoint = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["endpoint_url"])
+              client = Aws::S3::Client.new(
+                  :access_key_id =>access,
+                  :secret_access_key => secret,
+                  :region => region,
+                  :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
+                  :http_open_timeout => 60, :retry_limit => 20)
+            end
 
-            client = Aws::S3::Client.new(
-                :access_key_id =>access,
-                :secret_access_key => secret,
-                :session_token => session,
-                :region => region,
-                :http_open_timeout => 60, :retry_limit => 20
-            )
             File.open(project_home+"/"+absolute_path, 'w+') do |file|
               resp = client.get_object({bucket:bucket,
-                                    key:key}, target: file)
+                                    key:file_key}, target: file)
             end
             return true
           end