cnvrg 0.0.148 → 0.0.149

data/lib/cnvrg/data.rb

@@ -1,4 +1,5 @@
 require 'thor'
+require 'cnvrg/cli'
 
 class SubCommandBase < Thor
   def self.banner(command, namespace = nil, subcommand = false)
@@ -11,15 +12,58 @@ class SubCommandBase < Thor
   end
 module Cnvrg
   class Data < SubCommandBase
-    desc "init", "init data folder"
-
+    desc "data init", "init data folder"
+    method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
     def init
-      puts "im in init"
+      cli = Cnvrg::CLI.new()
+      public = options["public"]
+      cli.init_data(public)
     end
-    desc "upload", "upload data folder"
-
+    desc "data upload", "upload data folder"
+    method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
     def upload
-      puts "im in upload"
+      cli = Cnvrg::CLI.new()
+      ignore = options["ignore"]
+      verbose = options["verbose"]
+      sync = options["sync"]
+
+      cli.upload_data_tar(ignore, verbose,sync)
+    end
+    desc 'data download', 'pull data'
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+
+    def download()
+      cli = Cnvrg::CLI.new()
+      verbose = options["verbose"]
+      sync = options["sync"]
+
+      cli.download_data(verbose,sync,Dir.pwd)
+
+    end
+    desc 'data clone', 'clone datset'
+    def clone(dataset_url)
+      cli = Cnvrg::CLI.new()
+      cli.clone_data(dataset_url)
+
+    end
+    desc 'data list', 'list of datasets'
+    def list()
+      cli = Cnvrg::CLI.new()
+      verbose = options["verbose"]
+      sync = options["sync"]
+
+      cli.list_dataset()
+
+    end
+    desc 'data commits', 'pull data'
+
+    def commits()
+      cli = Cnvrg::CLI.new()
+      cli.list_dataset_commits()
+
     end
 
   end

data/lib/cnvrg/datafiles.rb

@@ -6,6 +6,8 @@ module Cnvrg
   class Datafiles
 
     LARGE_FILE=1024*1024*5
+    MULTIPART_SPLIT=10000000
+
     attr_reader :base_resource
 
     def initialize(owner, dataset_slug)
@@ -19,9 +21,11 @@ module Cnvrg
       file_size = File.size(absolute_path).to_f
       mime_type = MimeMagic.by_path(absolute_path)
       content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
+      sha1 = Digest::SHA1.file(absolute_path).hexdigest
+
       upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
                                                                                      commit_sha1: commit_sha1, file_name: file_name,
-                                                                                     file_size: file_size, file_content_type: content_type})
+                                                                                     file_size: file_size, file_content_type: content_type,sha1:sha1})
       if Cnvrg::CLI.is_response_success(upload_resp, false)
         path = upload_resp["result"]["path"]
         if file_size.to_f>= Cnvrg::Files::LARGE_FILE.to_f
@@ -37,6 +41,27 @@ module Cnvrg
       end
       return false
     end
+    def upload_tar_file(absolute_path, relative_path, commit_sha1)
+      file_name = File.basename relative_path
+      file_size = File.size(absolute_path).to_f
+      mime_type = MimeMagic.by_path(absolute_path)
+      content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
+      sha1 = Digest::SHA1.file(absolute_path).hexdigest
+
+      upload_resp = Cnvrg::API.request(@base_resource + "upload_tar_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
+                                                                                     commit_sha1: commit_sha1, file_name: file_name,
+                                                                                     file_size: file_size, file_content_type: content_type,sha1:sha1})
+      if Cnvrg::CLI.is_response_success(upload_resp, false)
+        path = upload_resp["result"]["path"]
+        s3_res = upload_large_files_s3(upload_resp, absolute_path)
+        if s3_res
+          Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
+                                                                    blob_id: upload_resp["result"]["id"]})
+          return true
+        end
+      end
+      return false
+    end
     def upload_log_file(absolute_path, relative_path,log_date)
       file_name = File.basename relative_path
       file_size = File.size(absolute_path).to_f
@@ -199,9 +224,11 @@ module Cnvrg
 
     end
 
-    def upload_large_files_s3(upload_resp, file_path)
+    def   upload_large_files_s3(upload_resp, file_path)
       begin
         sts_path = upload_resp["result"]["path_sts"]
+        s4cmd_path = upload_resp["result"]["path_s4cmd"]
+
         uri = URI.parse(sts_path)
         http_object = Net::HTTP.new(uri.host, uri.port)
         http_object.use_ssl = true if uri.scheme == 'https'
@@ -212,18 +239,72 @@ module Cnvrg
           response = http.request request
           body = response.read_body
         end
+
         URLcrypt::key = [body].pack('H*')
-        s3 = Aws::S3::Resource.new(
-            :access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
-            :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
-            :session_token => URLcrypt.decrypt(upload_resp["result"]["sts_st"]),
-            :region => URLcrypt.decrypt(upload_resp["result"]["region"]))
-        resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
-            object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
-            upload_file(file_path,{:use_accelerate_endpoint=>true})
+
+        python_version=`python --version  > /dev/null 2>&1` ;  is_python=$?.success?
+        if is_python
+
+          s4cmd=`pip freeze |grep s4cmd > /dev/null 2>&1` ;  s4cmd_suc=$?.success?
+          if !s4cmd_suc
+            `pip install s4cmd  > /dev/null 2>&1`
+          end
+
+        end
+
+        if !is_python
+          s3 = Aws::S3::Resource.new(
+              :access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
+              :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
+              :session_token => URLcrypt.decrypt(upload_resp["result"]["sts_st"]),
+              :region => URLcrypt.decrypt(upload_resp["result"]["region"]))
+          resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
+              object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
+              upload_file(file_path,{:use_accelerate_endpoint=>true})
+
+        else
+
+          s4cmd_uri = URI.parse(s4cmd_path)
+          s4cmd_http_object = Net::HTTP.new(s4cmd_uri.host, s4cmd_uri.port)
+          s4cmd_http_object.use_ssl = true if s4cmd_uri.scheme == 'https'
+          s4cmd_request = Net::HTTP::Get.new(s4cmd_path)
+
+          s4cmd_body = ""
+          s4cmd_http_object.start do |http|
+            response = http.request s4cmd_request
+            s4cmd_body = response.read_body
+          end
+
+          s4cmd_new_body = s4cmd_body.gsub(" self.client = self.boto3.client('s3',
+                                      aws_access_key_id=aws_access_key_id,
+                                      aws_secret_access_key=aws_secret_access_key)"," self.client = self.boto3.client('s3',
+                                      aws_access_key_id='#{ URLcrypt.decrypt(upload_resp["result"]["sts_a"])}',
+                                      aws_secret_access_key='#{URLcrypt.decrypt(upload_resp["result"]["sts_s"])}',
+                                      aws_session_token='#{URLcrypt.decrypt(upload_resp["result"]["sts_st"])}')")
+
+          tmp = Tempfile.new('s4cmd.py')
+          tmp << s4cmd_new_body
+          tmp.flush
+          tmp.close
+
+          is_success = false
+          count = 0
+          while !is_success and count <3
+            resp = `python #{tmp.path} --num-threads=128 --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)}  > /dev/null 2>&1`
+            is_success =$?.success?
+            count +=1
+
+          end
+          resp= is_success
+
+        end
+
         return resp
+
       rescue =>e
-        puts e
+        if File.exist? tmp
+          FileUtils.rm_rf [tmp]
+        end
         return false
 
       end
@@ -231,6 +312,7 @@ module Cnvrg
 
     end
 
+
     def upload_small_files_s3(url_path, file_path, content_type)
       url = URI.parse(url_path)
       file = File.open(file_path, "rb")
@@ -273,9 +355,9 @@ module Cnvrg
       response = Cnvrg::API.request(@base_resource + "create_dir", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
       return Cnvrg::CLI.is_response_success(response, false)
     end
-    def download_file_s3(absolute_path, relative_path, project_home, conflict=false)
+    def download_file_s3(absolute_path, relative_path, project_home, conflict=false,commit_sha1=nil)
       begin
-        res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
+        res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path,commit_sha1:commit_sha1})
         Cnvrg::CLI.is_response_success(res, false)
         if res["result"]
           download_resp = res
@@ -311,6 +393,46 @@ module Cnvrg
 
       end
     end
+
+    def download_data_file(commit_sha1,dataset_home)
+      begin
+        res = Cnvrg::API.request(@base_resource + "download_data_file", 'POST', {commit_sha1:commit_sha1})
+        Cnvrg::CLI.is_response_success(res, false)
+        if res["result"]
+          download_resp = res
+          filename = download_resp["result"]["filename"]
+
+          sts_path = download_resp["result"]["path_sts"]
+          uri = URI.parse(sts_path)
+          http_object = Net::HTTP.new(uri.host, uri.port)
+          http_object.use_ssl = true if uri.scheme == 'https'
+          request = Net::HTTP::Get.new(sts_path)
+
+          body = ""
+          http_object.start do |http|
+            response = http.request request
+            body = response.read_body
+          end
+          URLcrypt::key = [body].pack('H*')
+          s3 = Aws::S3::Client.new(
+              :access_key_id => URLcrypt.decrypt(download_resp["result"]["sts_a"]),
+              :secret_access_key => URLcrypt.decrypt(download_resp["result"]["sts_s"]),
+              :session_token => URLcrypt.decrypt(download_resp["result"]["sts_st"]),
+              :region => URLcrypt.decrypt(download_resp["result"]["region"]))
+
+          File.open(dataset_home+"/"+filename, 'wb') do |file|
+            resp = s3.get_object({ bucket:URLcrypt.decrypt(download_resp["result"]["bucket"]),
+                                   key:URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
+          end
+          return filename
+        end
+
+      rescue =>e
+        return false
+
+      end
+    end
+
     def download_file(absolute_path, relative_path, project_home, conflict=false)
       res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
       Cnvrg::CLI.is_response_success(res, false)
@@ -332,14 +454,14 @@ module Cnvrg
       return true
     end
 
-    def download_dir(absolute_path, relative_path, project_home)
-      FileUtils.mkdir_p("#{project_home}/#{absolute_path}")
+    def download_dir(dataset_home, absolute_path)
+      FileUtils.mkdir_p("#{dataset_home}/#{absolute_path}")
     end
-    def revoke_download_dir(absolute_path, relative_path, project_home)
+    def revoke_download_dir(absolute_path)
       puts FileUtils.rmtree("#{absolute_path}")
     end
 
-    def revoke_download_file(project_home,absolute_path,filename,conflict=false)
+    def revoke_download_file(absolute_path,filename,conflict=false)
       begin
       file_location = absolute_path.gsub(/#{filename}\/?$/, "")
 
@@ -363,6 +485,10 @@ module Cnvrg
       response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1})
       return response
     end
+    def end_commit_tar(commit_sha1,cur_idx)
+      response = Cnvrg::API.request("#{base_resource}/commit/end_tar", 'POST', {commit_sha1: commit_sha1,idx: cur_idx})
+      return response
+    end
 
     def rollback_commit(commit_sha1)
       response = Cnvrg::API.request("#{base_resource}/commit/rollback", 'POST', {commit_sha1: commit_sha1})

data/lib/cnvrg/dataset.rb

@@ -14,11 +14,11 @@ module Cnvrg
         @owner = config[:owner]
         @working_dir = project_home
       rescue => e
+
       end
 
     end
 
-
     def last_local_commit
       idx = YAML.load_file(@local_path + "/.cnvrg/idx.yml")
       return idx[:commit]
@@ -36,6 +36,12 @@ module Cnvrg
       CLI.is_response_success(response)
       return response
 
+    end
+    def list_commits
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/list_commits", 'GET')
+      CLI.is_response_success(response)
+      return response
+
     end
     def url
       url = Cnvrg::Helpers.remote_url
@@ -43,11 +49,14 @@ module Cnvrg
     end
 
     def update_ignore_list(new_ignore)
-      if new_ignore.empty?
+
+      if new_ignore.nil? or new_ignore.empty?
         return true
       end
       begin
         File.open(self.local_path+"/.cnvrgignore", "a+") do |f|
+          f.puts("\n")
+
           new_ignore.each do |i|
             f.puts("#{i}\n")
           end
@@ -62,13 +71,19 @@ module Cnvrg
       ignore_list = []
       File.open(self.local_path+"/.cnvrgignore", "r").each_line do |line|
         line = line.strip
-        if line.start_with? "#"
+
+        if line.start_with? "#" or ignore_list.include? line
           next
         end
-        if line.end_with? "/"
-          ignore_list << line.chop
-          sub_dirs = Dir.glob("#{line}/**/*").each { |x| x.gsub!("//", "/") }
-          ignore_list << sub_dirs.flatten
+        if line.end_with? "/" or File.directory?(line)
+          if line.end_with? "/"
+            ignore_list << line.chop
+          else
+            ignore_list << line
+          end
+          all_sub = Dir.glob("#{line}/**/*", File::FNM_DOTMATCH).flatten
+          ignore_list << all_sub.flatten
+          ignore_list << line
         else
           ignore_list << line
         end
@@ -88,8 +103,9 @@ module Cnvrg
 
         cnvrgignore = Helpers.cnvrgignore_content
         begin
-          response = Cnvrg::API.request("cli/create_dataset", 'POST', {title: dataset_name, owner: owner, is_public: is_public})
-          Cnvrg::CLI.is_response_success(response)
+            response = Cnvrg::API.request("cli/create_dataset", 'POST', {title: dataset_name, owner: owner, is_public: is_public})
+
+            Cnvrg::CLI.is_response_success(response)
           response = JSON.parse response["result"]
           dataset_slug = response["slug"]
 
@@ -105,7 +121,53 @@ module Cnvrg
           return false
         end
         return true
+    end
+    def self.clone(owner, dataset_name,dataset_slug)
+
+      begin
+        list_dirs = [ dataset_name, "#{dataset_name}/.cnvrg"
+        ]
+        list_files = [
+            "#{dataset_name}/.cnvrgignore",
+            "#{dataset_name}/.cnvrg/config.yml"
+        ]
+
+        config = {dataset_name: dataset_name,
+                  dataset_slug: dataset_slug,
+                  owner: owner}
+
+
+        cnvrgignore = Helpers.cnvrgignore_content
+        FileUtils.mkdir_p list_dirs
+        FileUtils.touch list_files
+        File.open("#{dataset_name}/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+        File.open("#{dataset_name}/.cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
+      rescue => e
+        return false
       end
+      return true
+    end
+
+    def self.init_container(owner, dataset_slug,dataset_name)
+
+
+      cnvrgignore = Helpers.cnvrgignore_content
+      begin
+
+
+        config = {dataset_name: dataset_name,
+                  dataset_slug: dataset_slug,
+                  owner: owner}
+        File.open("/home/ds/notebooks/data/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+
+        File.open("/home/ds/notebooks/data/.cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
+      rescue => e
+        puts e
+        puts e.backtrace
+        return false
+      end
+      return true
+    end
 
 
       def get_idx
@@ -159,15 +221,29 @@ module Cnvrg
       return response
     end
 
-      def compare_idx(new_branch, commit=last_local_commit)
+    def downlowd_updated_data(current_commit)
 
-        local_idx = self.generate_idx
-        response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status", 'POST', {idx: local_idx, new_branch: new_branch, current_commit: commit})
-        CLI.is_response_success(response)
-        return response
-      end
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/downlowd_updated_data", 'POST', {current_commit: current_commit})
+      CLI.is_response_success(response)
+      return response
+    end
+    def compare_idx(new_branch, commit=last_local_commit)
 
-      def compare_commit(commit)
+      local_idx = self.generate_idx
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status", 'POST', {idx: local_idx, new_branch: new_branch, current_commit: commit})
+      CLI.is_response_success(response)
+      return response
+    end
+
+
+
+    def compare_commits(commit)
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/compare_commits", 'POST', {compare_commit: commit,current_commit:last_local_commit})
+      CLI.is_response_success(response)
+      return response
+    end
+
+    def compare_commit(commit)
         if commit.nil? or commit.empty?
           commit = last_local_commit
         end
@@ -187,6 +263,11 @@ module Cnvrg
 
         return true
       end
+    def update_idx(idx)
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
+
+      return true
+    end
 
 
       def update_idx_with_commit!(commit)