cnvrg 0.3.6 → 0.3.8

data/lib/cnvrg/data.rb

@@ -22,35 +22,69 @@ module Cnvrg
       cli.init_data(public)
     end
     desc "data upload", "upload data folder"
-    method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
-    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
-    method_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
+    method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s","--sync"], :default => false
 
     def upload
       cli = Cnvrg::CLI.new()
-      ignore = options["ignore"]
       verbose = options["verbose"]
       sync = options["sync"]
-      no_compression = options["no_compression"]
-      cli.upload_data_tar(ignore, verbose,sync,no_compression)
+      force = options["force"]
+      new_branch = options["new_branch"]
+      cli.upload_data_new(new_branch,  verbose,sync,force)
+    end
+    desc 'sync', 'sync_data_new'
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
+    method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :commit, :type => :string, :aliases => ["-c"], :desc => "download specified commit", :default => nil
+    method_option :all_files, :type => :boolean, :aliases => ["--all"], :desc => "download specified commit", :default => true
+
+    def sync_data_new()
+      cli = Cnvrg::CLI.new()
+      force = options["force"]
+      new_branch = options["new_branch"]
+      verbose = options["verbose"]
+      commit = options["commit"]
+      all_files = options["all_files"]
+
+      cli.sync_data_new(new_branch, force, verbose,commit,all_files)
     end
     desc 'data download', 'pull data'
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits", :default => false
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+    method_option :commit, :type => :string, :aliases => ["-c"], :desc => "download specified commit", :default => nil
+    method_option :all_files, :type => :boolean, :aliases => ["--all"], :desc => "download specified commit", :default => false
 
     def download()
       cli = Cnvrg::CLI.new()
       verbose = options["verbose"]
       sync = options["sync"]
+      new_branch = options["new_branch"]
+      commit = options["commit"]
+      all_files = options["all_files"]
 
-      cli.download_data(verbose,sync,Dir.pwd)
+      cli.download_data_new(verbose,sync,new_branch, commit,all_files)
 
     end
     desc 'data clone', 'clone datset'
+    method_option :only_tree, :type => :boolean, :aliases => ["-t", "--tree"], :default => false
+    method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => nil
     def clone(dataset_url)
       cli = Cnvrg::CLI.new()
-      cli.clone_data(dataset_url)
+    only_tree =options[:only_tree]
+      commit =options[:commit]
+
+        cli.clone_data(dataset_url, only_tree=only_tree,commit=commit)
+
+    end
+    desc 'data delete', 'delete datset'
+    def delete(dataset_slug)
+      cli = Cnvrg::CLI.new()
+      cli.delete_data(dataset_slug)
 
     end
     desc 'data list', 'list of datasets'

data/lib/cnvrg/datafiles.rb

@@ -1,6 +1,8 @@
 require 'mimemagic'
 require 'aws-sdk'
 require 'URLcrypt'
+require "down"
+
 module Cnvrg
   class Datafiles
 
@@ -16,7 +18,6 @@ module Cnvrg
     end
 
     def upload_file(absolute_path, relative_path, commit_sha1)
-
       file_name = File.basename relative_path
       file_size = File.size(absolute_path).to_f
       mime_type = MimeMagic.by_path(absolute_path)
@@ -26,22 +27,11 @@ module Cnvrg
                                                                                      commit_sha1: commit_sha1, file_name: file_name,
                                                                                      file_size: file_size, file_content_type: content_type, sha1: sha1})
       if Cnvrg::CLI.is_response_success(upload_resp, false)
-        path = upload_resp["result"]["path"]
-        if file_size.to_f>= Cnvrg::Files::LARGE_FILE.to_f
-          s3_res = upload_large_files_s3(upload_resp, absolute_path)
-        else
-          s3_res = upload_small_files_s3(path, absolute_path, content_type)
-        end
-        if s3_res
-          Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
-                                                                    blob_id: upload_resp["result"]["id"]})
-          return true
-        end
-      end
-      return false
-
-    end
+        s3_res = upload_large_files_s3(upload_resp, absolute_path)
 
+        return s3_res
+      end
+      end
     def upload_tar_file(absolute_path, relative_path, commit_sha1)
       begin
         file_name = File.basename relative_path
@@ -82,9 +72,9 @@ module Cnvrg
         puts e.message
         return false
       end
-
     end
 
+
     def upload_log_file(absolute_path, relative_path, log_date)
       file_name = File.basename relative_path
       file_size = File.size(absolute_path).to_f
@@ -251,82 +241,59 @@ module Cnvrg
     def upload_large_files_s3(upload_resp, file_path)
       begin
         sts_path = upload_resp["result"]["path_sts"]
-        # s4cmd_path = upload_resp["result"]["path_s4cmd"]
-
-        uri = URI.parse(sts_path)
-        http_object = Net::HTTP.new(uri.host, uri.port)
-        http_object.use_ssl = true if uri.scheme == 'https'
-        request = Net::HTTP::Get.new(sts_path)
 
-        body = ""
-        http_object.start do |http|
-          response = http.request request
-          body = response.read_body
+        if !Helpers.is_verify_ssl
+          body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
+        else
+          body = open(sts_path).read
         end
         home_dir = File.expand_path('~')
-        log_file = "#{home_dir}/.cnvrg/tmp/upload_#{File.basename(file_path)}.log"
-
 
         split = body.split("\n")
         key = split[0]
         iv = split[1]
+        is_s3 = upload_resp["result"]["is_s3"]
+        access =  Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_a"])
 
-        access =  Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["new_a"])
-
-        secret =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["new_s"])
+        secret =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["sts_s"])
 
+        session =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["sts_st"])
         region =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["region"])
 
         bucket =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["bucket"])
 
+        if is_s3 or is_s3.nil?
+
+          use_accelerate_endpoint =true
+          client = Aws::S3::Client.new(
+              :access_key_id =>access,
+              :secret_access_key => secret,
+              :session_token => session,
+              :region => region,
+              :http_open_timeout => 60, :retry_limit => 20)
+        else
+          endpoint = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["endpoint"])
+          use_accelerate_endpoint = false
+          client = Aws::S3::Client.new(
+              :access_key_id =>access,
+              :secret_access_key => secret,
+              :region => region,
+              :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
+              :http_open_timeout => 60, :retry_limit => 20)
+        end
+
+
+
+
+
+
 
-        client = Aws::S3::Client.new(:access_key_id => access,
-                                     :secret_access_key =>secret,
-                                     :region => region,
-                                     :logger => Logger.new(log_file),
-                                     :http_open_timeout => 60, :retry_limit => 20,
-                                     :http_wire_trace => true)
         s3 = Aws::S3::Resource.new(client: client)
 
         resp = s3.bucket(bucket).
             object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
-            upload_file(file_path, {:use_accelerate_endpoint => true,:server_side_encryption => 'AES256'})
-        # s4cmd_path = upload_resp["result"]["path_s4cmd"]
-        #
-        # s4cmd_uri = URI.parse(s4cmd_path)
-        # s4cmd_http_object = Net::HTTP.new(s4cmd_uri.host, s4cmd_uri.port)
-        # s4cmd_http_object.use_ssl = true if s4cmd_uri.scheme == 'https'
-        # s4cmd_request = Net::HTTP::Get.new(s4cmd_path)
-        #
-        # s4cmd_body = ""
-        # s4cmd_http_object.start do |http|
-        #   response = http.request s4cmd_request
-        #   s4cmd_body = response.read_body
-        # end
-        # s4cmd_new_body = s4cmd_body.gsub(" self.client = self.boto3.client('s3',
-        #                               aws_access_key_id=aws_access_key_id,
-        #                               aws_secret_access_key=aws_secret_access_key)", " self.client = self.boto3.client('s3',
-        #                               aws_access_key_id='#{ URLcrypt.decrypt(upload_resp["result"]["new_a"])}',
-        #                               aws_secret_access_key='#{URLcrypt.decrypt(upload_resp["result"]["new_s"])}')")
-        #
-        #
-        # tmp = Tempfile.new('s4cmd.py')
-        # tmp << s4cmd_new_body
-        # tmp.flush
-        # tmp.close
-        # #
-        # is_success = false
-        # count = 0
-        # while !is_success and count <3
-        #   resp = `python #{tmp.path} --num-threads=8 --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)} > /dev/null 2>&1 `
-        #   is_success =$?.success?
-        #   count +=1
-        #
-        # end
-        # resp= is_success
-        if resp == true
-          FileUtils.rm_rf(log_file)
-        end
+            upload_file(file_path, {:use_accelerate_endpoint => use_accelerate_endpoint,:server_side_encryption => 'AES256'})
+
 
         return resp
 
@@ -380,60 +347,82 @@ module Cnvrg
 
     def create_dir(absolute_path, relative_path, commit_sha1)
       response = Cnvrg::API.request(@base_resource + "create_dir", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
-      puts response
       return Cnvrg::CLI.is_response_success(response, false)
     end
 
-    def download_file_s3(absolute_path, relative_path, project_home, conflict=false, commit_sha1=nil)
+
+    def download_file_s3(absolute_path, relative_path, project_home, conflict=false, commit_sha1=nil, as_link=false)
       begin
-        res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
+        res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1 ,new_version:true, as_link:as_link})
         Cnvrg::CLI.is_response_success(res, false)
-        if res["result"]
-          download_resp = res
-          filename = download_resp["result"]["filename"]
-
-          absolute_path += ".conflict" if conflict
-          sts_path = download_resp["result"]["path_sts"]
-          uri = URI.parse(sts_path)
-          http_object = Net::HTTP.new(uri.host, uri.port)
-          http_object.use_ssl = true if uri.scheme == 'https'
-          request = Net::HTTP::Get.new(sts_path)
-
-          body = ""
-          http_object.start do |http|
-            response = http.request request
-            body = response.read_body
-          end
-          split = body.split("\n")
-          key = split[0]
-          iv = split[1]
-
-          access =  Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
+          if res["result"]
+            file_url = res["result"]["file_url"]
 
-          secret =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_s"])
+            if as_link
+              return res["result"]
+            end
+            # begin
+            #   if !Helpers.is_verify_ssl
+            #     tempfile = Down.download(file_url,open_timeout: 60,ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE )
+            #
+            #   else
+            #     tempfile = Down.download(file_url,open_timeout: 60)
+            #
+            #   end
+            #
+            #   FileUtils.move(tempfile.path, project_home+"/"+ absolute_path)
+            #   return true
+            # rescue
+            #
+            # end
+            download_resp = res
+            filename = download_resp["result"]["filename"]
+
+            absolute_path += ".conflict" if conflict
+            sts_path = download_resp["result"]["path_sts"]
+            if !Helpers.is_verify_ssl
+              body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
+            else
+              body = open(sts_path).read
+            end
+            split = body.split("\n")
+            key = split[0]
+            iv = split[1]
 
-          session =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_st"])
-          region =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["region"])
+            access =  Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
 
-          bucket =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["bucket"])
-          key =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
+            secret =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_s"])
 
+            session =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_st"])
+            region =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["region"])
 
+            bucket =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["bucket"])
+            file_key =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
 
-          client = Aws::S3::Client.new(
-              :access_key_id =>access,
-              :secret_access_key => secret,
-              :session_token => session,
-              :region => region,
-              :http_open_timeout => 60, :retry_limit => 20
-          )
+            is_s3 = download_resp["result"]["is_s3"]
+            if is_s3 or is_s3.nil?
+              client = Aws::S3::Client.new(
+                  :access_key_id =>access,
+                  :secret_access_key => secret,
+                  :session_token => session,
+                  :region => region,
+                  :http_open_timeout => 60, :retry_limit => 20)
+            else
+              endpoint = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["endpoint_url"])
+              client = Aws::S3::Client.new(
+                  :access_key_id =>access,
+                  :secret_access_key => secret,
+                  :region => region,
+                  :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
+                  :http_open_timeout => 60, :retry_limit => 20)
+            end
 
-          File.open(project_home+"/"+absolute_path, 'w+') do |file|
-            resp = client.get_object({bucket:bucket,
-                                  key:key}, target: file)
+            File.open(project_home+"/"+absolute_path, 'w+') do |file|
+              resp = client.get_object({bucket:bucket,
+                                        key:file_key}, target: file)
+            end
+            return true
           end
-          return true
-        end
 
       rescue => e
         return false
@@ -450,15 +439,10 @@ module Cnvrg
           filename = download_resp["result"]["filename"]
 
           sts_path = download_resp["result"]["path_sts"]
-          uri = URI.parse(sts_path)
-          http_object = Net::HTTP.new(uri.host, uri.port)
-          http_object.use_ssl = true if uri.scheme == 'https'
-          request = Net::HTTP::Get.new(sts_path)
-
-          body = ""
-          http_object.start do |http|
-            response = http.request request
-            body = response.read_body
+          if !Helpers.is_verify_ssl
+            body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
+          else
+            body = open(sts_path).read
           end
           split = body.split("\n")
           key = split[0]
@@ -473,21 +457,30 @@ module Cnvrg
           region =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["region"])
 
           bucket =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["bucket"])
-          key =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
-
-
-          client = Aws::S3::Client.new(
-              :access_key_id =>access,
-              :secret_access_key => secret,
-              :session_token => session,
-              :region => region,
-              :http_open_timeout => 60, :retry_limit => 20
-          )
+          file_key =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
+
+          is_s3 = download_resp["result"]["is_s3"]
+          if is_s3 or is_s3.nil?
+            client = Aws::S3::Client.new(
+                :access_key_id =>access,
+                :secret_access_key => secret,
+                :session_token => session,
+                :region => region,
+                :http_open_timeout => 60, :retry_limit => 20)
+          else
+            endpoint = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["endpoint_url"])
+            client = Aws::S3::Client.new(
+                :access_key_id =>access,
+                :secret_access_key => secret,
+                :region => region,
+                :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
+                :http_open_timeout => 60, :retry_limit => 20)
+          end
 
 
           File.open(dataset_home+"/"+filename, 'w+') do |file|
             resp = client.get_object({bucket: bucket,
-                                  key: key }, target: file)
+                                  key: file_key }, target: file)
           end
           return filename
         end
@@ -518,6 +511,17 @@ module Cnvrg
       end
       return true
     end
+    def delete_commit_files_local(deleted)
+      begin
+        FileUtils.rm_rf(deleted) unless (deleted.nil? or deleted.empty?)
+        return true
+      rescue => e
+        return false
+      end
+
+      return true
+
+    end
 
     def download_dir(dataset_home, absolute_path)
       FileUtils.mkdir_p("#{dataset_home}/#{absolute_path}")
@@ -551,18 +555,37 @@ module Cnvrg
       return true
 
     end
-
-    def start_commit(new_branch)
-
-      response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: false,
-                                                                              username: @owner})
-      Cnvrg::CLI.is_response_success(response, false)
+    def delete_commit(commit_sha1)
+      response = Cnvrg::API.request("#{base_resource}/commit/#{commit_sha1}", 'DELETE')
+      Cnvrg::CLI.is_response_success(response, true)
       return response
     end
-
-    def end_commit(commit_sha1)
-      response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1})
+    def get_commit(commit_sha1)
+      response = Cnvrg::API.request("#{base_resource}/commit/#{commit_sha1}", 'GET')
+      Cnvrg::CLI.is_response_success(response, true)
+      return response
+    end
+    def start_commit(new_branch,force=false,delete_commit=nil)
+      begin
+      response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch,force:force,
+                                                                              username: @owner,delete_commit:delete_commit})
+      Cnvrg::CLI.is_response_success(response, true)
       return response
+      rescue => e
+        return false
+      end
+
+    end
+
+    def end_commit(commit_sha1,force)
+      begin
+        response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1,force:force})
+        Cnvrg::CLI.is_response_success(response, true)
+        return response
+      rescue => e
+        return false
+      end
+
     end
 
     def end_commit_tar(commit_sha1, cur_idx)