cnvrg 0.0.14 → 0.0.15

data/lib/cnvrg/data.rb

@@ -0,0 +1,72 @@
+require 'cnvrg/cli'
+require 'thor'
+class SubCommandBase < Thor
+ def self.banner(command, namespace = nil, subcommand = false)
+  "#{basename}  #{command.usage}"
+ end
+
+ def self.subcommand_prefix
+  self.name.gsub(%r{.*::}, '').gsub(%r{^[A-Z]}) { |match| match[0].downcase }.gsub(%r{[A-Z]}) { |match| "-#{match[0].downcase}" }
+ end
+end
+
+module Cnvrg
+
+  class Data < SubCommandBase
+    class_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
+    desc "data init", "init data folder"
+    method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
+    def init
+      cli = Cnvrg::CLI.new()
+      public = options["public"]
+      cli.init_data(public)
+    end
+    desc "data upload", "upload data folder"
+    method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+    method_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
+
+    def upload
+      cli = Cnvrg::CLI.new()
+      ignore = options["ignore"]
+      verbose = options["verbose"]
+      sync = options["sync"]
+      no_compression = options["no_compression"]
+      cli.upload_data_tar(ignore, verbose,sync,no_compression)
+    end
+    desc 'data download', 'pull data'
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+
+    def download()
+      cli = Cnvrg::CLI.new()
+      verbose = options["verbose"]
+      sync = options["sync"]
+
+      cli.download_data(verbose,sync,Dir.pwd)
+
+    end
+    desc 'data clone', 'clone datset'
+    def clone(dataset_url)
+      cli = Cnvrg::CLI.new()
+      cli.clone_data(dataset_url)
+
+    end
+    desc 'data list', 'list of datasets'
+    def list()
+      cli = Cnvrg::CLI.new()
+
+      cli.list_dataset()
+
+    end
+    desc 'data commits', 'pull data'
+
+    def commits()
+      cli = Cnvrg::CLI.new()
+      cli.list_dataset_commits()
+
+    end
+
+  end
+end

data/lib/cnvrg/datafiles.rb

@@ -0,0 +1,509 @@
+require 'mimemagic'
+require 'aws-sdk'
+require 'URLcrypt'
+
+module Cnvrg
+  class Datafiles
+
+    LARGE_FILE=1024*1024*5
+    MULTIPART_SPLIT=10000000
+
+    attr_reader :base_resource
+
+    def initialize(owner, dataset_slug)
+      @dataset_slug = dataset_slug
+      @owner = owner
+      @base_resource = "users/#{owner}/datasets/#{dataset_slug}/"
+    end
+
+    def upload_file(absolute_path, relative_path, commit_sha1)
+      file_name = File.basename relative_path
+      file_size = File.size(absolute_path).to_f
+      mime_type = MimeMagic.by_path(absolute_path)
+      content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
+      sha1 = Digest::SHA1.file(absolute_path).hexdigest
+
+      upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
+                                                                                     commit_sha1: commit_sha1, file_name: file_name,
+                                                                                     file_size: file_size, file_content_type: content_type,sha1:sha1})
+      if Cnvrg::CLI.is_response_success(upload_resp, false)
+        path = upload_resp["result"]["path"]
+        if file_size.to_f>= Cnvrg::Files::LARGE_FILE.to_f
+          s3_res = upload_large_files_s3(upload_resp, absolute_path)
+        else
+          s3_res = upload_small_files_s3(path, absolute_path, content_type)
+        end
+        if s3_res
+          Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
+                                                                    blob_id: upload_resp["result"]["id"]})
+          return true
+        end
+      end
+      return false
+    end
+    def upload_tar_file(absolute_path, relative_path, commit_sha1)
+      begin
+      file_name = File.basename relative_path
+      file_size = File.size(absolute_path).to_f
+      mime_type = MimeMagic.by_path(absolute_path)
+      content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
+      sha1 = Digest::SHA1.file(absolute_path).hexdigest
+
+      upload_resp = Cnvrg::API.request(@base_resource + "upload_tar_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
+                                                                                     commit_sha1: commit_sha1, file_name: file_name,
+                                                                                     file_size: file_size, file_content_type: content_type,sha1:sha1})
+      if Cnvrg::CLI.is_response_success(upload_resp, false)
+        path = upload_resp["result"]["path"]
+        s3_res = upload_large_files_s3(upload_resp, absolute_path)
+        if s3_res
+          Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
+                                                                    blob_id: upload_resp["result"]["id"]})
+          return true
+        end
+      else
+        return false
+      end
+      rescue => e
+        puts e.message
+        return false
+      end
+
+    end
+    def upload_log_file(absolute_path, relative_path,log_date)
+      file_name = File.basename relative_path
+      file_size = File.size(absolute_path).to_f
+      content_type = "text/x-log"
+      upload_resp = Cnvrg::API.request("/users/#{@owner}/" + "upload_cli_log", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
+                                                                                     file_name: file_name,log_date:log_date,
+                                                                                     file_size: file_size, file_content_type: content_type})
+      if Cnvrg::CLI.is_response_success(upload_resp, false)
+          path = upload_resp["result"]["path"]
+          s3_res = upload_small_files_s3(path, absolute_path,  "text/plain")
+      end
+      if s3_res
+        return true
+      end
+      return false
+
+    end
+    def upload_exec_file(absolute_path,image_name,commit_id)
+      file_name = File.basename absolute_path
+      file_size = File.size(absolute_path).to_f
+      content_type = "application/zip"
+      begin
+        upload_resp = Cnvrg::API.request("users/#{@owner}/images/" + "upload_config", 'POST_FILE', {relative_path: absolute_path,
+                                                                                                    file_name: file_name,
+                                                                                                    image_name:image_name,
+                                                                                                    file_size: file_size,
+                                                                                                    file_content_type: content_type,
+                                                                                                    project_slug: @project_slug,
+                                                                                                    commit_id:commit_id})
+        # puts upload_resp
+        if Cnvrg::CLI.is_response_success(upload_resp, false)
+          if upload_resp["result"]["image"] == -1
+            return -1
+          end
+          path = upload_resp["result"]["path"]
+          s3_res = upload_small_files_s3(path, absolute_path, content_type)
+
+        end
+      if s3_res
+        return upload_resp["result"]["id"]
+      end
+      return false
+    rescue SignalException
+
+      say "\nAborting"
+      exit(1)
+    end
+
+  end
+
+
+
+  def upload_image(absolute_path, image_name, owner, is_public, is_base,dpkg,libraries,bash,message,commit_id)
+      file_name = File.basename absolute_path
+      file_size = File.size(absolute_path).to_f
+      if is_base
+
+        content_type = "application/zip"
+      else
+        content_type = "application/gzip"
+      end
+      begin
+      upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload_cnvrg", 'POST_FILE', {relative_path: absolute_path,
+                                                                                          file_name: file_name,
+                                                                                          image_name: image_name,
+                                                                                          file_size: file_size,
+                                                                                          file_content_type: content_type,
+                                                                                          is_public: is_public,
+                                                                                          project_slug: @project_slug,
+                                                                                          commit_id:commit_id ,
+                                                                                          dpkg: dpkg,
+                                                                                          py2: libraries,
+                                                                                          py3: libraries,
+
+                                                                                          bash_history: bash,
+                                                                                          commit_message:message,
+                                                                                          is_base: is_base})
+      # puts upload_resp
+      if Cnvrg::CLI.is_response_success(upload_resp, false)
+        path = upload_resp["result"]["path"]
+        s3_res = upload_small_files_s3(path, absolute_path, content_type)
+        if s3_res
+          commit_resp = Cnvrg::API.request("users/#{owner}/images/#{upload_resp["result"]["id"]}/" + "commit", 'GET')
+          if Cnvrg::CLI.is_response_success(commit_resp, false)
+            return commit_resp["result"]["image"]
+          else
+            return false
+          end
+
+        end
+      end
+      return false
+      rescue =>e
+      end
+
+    end
+    def upload_cnvrg_image(absolute_path, image_name, owner, is_public, is_base,dpkg,libraries,bash,message)
+      file_name = File.basename absolute_path
+      file_size = File.size(absolute_path).to_f
+      if is_base
+
+        content_type = "application/zip"
+      else
+        content_type = "application/gzip"
+      end
+      begin
+        upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload_cnvrg", 'POST_FILE', {relative_path: absolute_path,
+                                                                                            file_name: file_name,
+                                                                                            image_name: image_name,
+                                                                                            file_size: file_size,
+                                                                                            file_content_type: content_type,
+                                                                                            is_public: is_public,
+                                                                                            dpkg: dpkg,
+                                                                                            libraries: libraries,
+                                                                                            bash_history: bash,
+                                                                                            commit_message:message,
+                                                                                            is_base: is_base})
+        # puts upload_resp
+        if Cnvrg::CLI.is_response_success(upload_resp, false)
+          path = upload_resp["result"]["path"]
+          s3_res = upload_small_files_s3(path, absolute_path, content_type)
+          if s3_res
+            commit_resp = Cnvrg::API.request("users/#{owner}/images/#{upload_resp["result"]["id"]}/" + "commit", 'GET')
+            if Cnvrg::CLI.is_response_success(commit_resp, false)
+              return commit_resp["result"]["image"]
+            else
+              return false
+            end
+
+          end
+        end
+        return false
+      rescue =>e
+      end
+
+    end
+
+    def download_image(file_path_to_store, image_slug, owner)
+
+
+      download_resp = Cnvrg::API.request("users/#{owner}/images/#{image_slug}/" + "download", 'GET')
+      path = download_resp["result"]["path"]
+
+      if Cnvrg::CLI.is_response_success(download_resp, false)
+        begin
+          open(file_path_to_store, 'wb') do |file|
+            file << open(path).read
+          end
+
+          return true
+        rescue => e
+          return false
+        end
+
+        return true
+      else
+        return false
+      end
+
+
+    end
+
+    def   upload_large_files_s3(upload_resp, file_path)
+      begin
+        sts_path = upload_resp["result"]["path_sts"]
+        # s4cmd_path = upload_resp["result"]["path_s4cmd"]
+
+        uri = URI.parse(sts_path)
+        http_object = Net::HTTP.new(uri.host, uri.port)
+        http_object.use_ssl = true if uri.scheme == 'https'
+        request = Net::HTTP::Get.new(sts_path)
+
+        body = ""
+        http_object.start do |http|
+          response = http.request request
+          body = response.read_body
+        end
+        is_python = false
+        s4cmd_suc = false
+        s4cmd_install_suc = false
+
+        URLcrypt::key = [body].pack('H*')
+
+        # python_version=`python --version  > /dev/null 2>&1` ;  is_python=$?.success?
+        # if is_python
+        #
+        #   s4cmd=`pip freeze 2>/dev/null |grep -e s4cmd -e boto3 > /dev/null 2>&1` ;  s4cmd_suc=$?.success?
+        #   if !s4cmd_suc
+        #     `pip install s4cmd  > /dev/null 2>&1` ; s4cmd_install_suc=$?.success?
+        #   end
+        #
+        # end
+        resp = false
+
+        if  !s4cmd_suc and !s4cmd_install_suc
+          s3 = Aws::S3::Resource.new(
+              :access_key_id => URLcrypt.decrypt(upload_resp["result"]["new_a"]),
+              :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["new_s"]),
+              :region => URLcrypt.decrypt(upload_resp["result"]["region"]))
+          resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
+              object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
+              upload_file(file_path,{:use_accelerate_endpoint=>true})
+
+        # else
+
+          # s4cmd_uri = URI.parse(s4cmd_path)
+          # s4cmd_http_object = Net::HTTP.new(s4cmd_uri.host, s4cmd_uri.port)
+          # s4cmd_http_object.use_ssl = true if s4cmd_uri.scheme == 'https'
+          # s4cmd_request = Net::HTTP::Get.new(s4cmd_path)
+          #
+          # s4cmd_body = ""
+          # s4cmd_http_object.start do |http|
+          #   response = http.request s4cmd_request
+          #   s4cmd_body = response.read_body
+          # end
+          #
+          # s4cmd_new_body = s4cmd_body.gsub(" self.client = self.boto3.client('s3',
+          #                             aws_access_key_id=aws_access_key_id,
+          #                             aws_secret_access_key=aws_secret_access_key)"," self.client = self.boto3.client('s3',
+          #                             aws_access_key_id='#{ URLcrypt.decrypt(upload_resp["result"]["sts_a"])}',
+          #                             aws_secret_access_key='#{URLcrypt.decrypt(upload_resp["result"]["sts_s"])}',
+          #                             aws_session_token='#{URLcrypt.decrypt(upload_resp["result"]["sts_st"])}')")
+          #
+          # tmp = Tempfile.new('s4cmd.py')
+          # tmp << s4cmd_new_body
+          # tmp.flush
+          # tmp.close
+          #
+          #   is_success = false
+          # count = 0
+          # while !is_success and count <3
+          #   resp = `python #{tmp.path} --num-threads=8 --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)}  > /dev/null 2>&1`
+          #   is_success =$?.success?
+          #   count +=1
+          #
+          # end
+          # resp= is_success
+
+        end
+
+        return resp
+
+      rescue => e
+        puts e.message
+        # if File.exist? tmp
+        #   FileUtils.rm_rf [tmp]
+        # end
+        return false
+
+      end
+      return true
+
+    end
+
+
+    def upload_small_files_s3(url_path, file_path, content_type)
+      url = URI.parse(url_path)
+      file = File.open(file_path, "rb")
+      body = file.read
+      begin
+        Net::HTTP.start(url.host) do |http|
+          http.send_request("PUT", url.request_uri, body, {
+              "content-type" => content_type,
+          })
+        end
+        return true
+      rescue Interrupt
+        return false
+      rescue
+        return false
+      end
+    end
+
+    def upload_url(file_path)
+      response = Cnvrg::API.request(@base_resource + "upload_url", 'POST', {file_s3_path: file_path})
+      if Cnvrg::CLI.is_response_success(response, false)
+        return response
+      else
+        return nil
+      end
+
+    end
+
+    def delete_file(absolute_path, relative_path, commit_sha1)
+      response = Cnvrg::API.request(@base_resource + "delete_file", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
+      return Cnvrg::CLI.is_response_success(response, false)
+    end
+
+    def delete_dir(absolute_path, relative_path, commit_sha1)
+      response = Cnvrg::API.request(@base_resource + "delete_dir", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
+      return Cnvrg::CLI.is_response_success(response, false)
+    end
+
+    def create_dir(absolute_path, relative_path, commit_sha1)
+      response = Cnvrg::API.request(@base_resource + "create_dir", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
+      return Cnvrg::CLI.is_response_success(response, false)
+    end
+    def download_file_s3(absolute_path, relative_path, project_home, conflict=false,commit_sha1=nil)
+      begin
+        res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path,commit_sha1:commit_sha1})
+        Cnvrg::CLI.is_response_success(res, false)
+        if res["result"]
+          download_resp = res
+          filename = download_resp["result"]["filename"]
+
+          absolute_path += ".conflict" if conflict
+          sts_path = download_resp["result"]["path_sts"]
+        uri = URI.parse(sts_path)
+        http_object = Net::HTTP.new(uri.host, uri.port)
+        http_object.use_ssl = true if uri.scheme == 'https'
+        request = Net::HTTP::Get.new(sts_path)
+
+        body = ""
+        http_object.start do |http|
+          response = http.request request
+          body = response.read_body
+        end
+        URLcrypt::key = [body].pack('H*')
+        s3 = Aws::S3::Client.new(
+            :access_key_id => URLcrypt.decrypt(download_resp["result"]["sts_a"]),
+            :secret_access_key => URLcrypt.decrypt(download_resp["result"]["sts_s"]),
+            :session_token => URLcrypt.decrypt(download_resp["result"]["sts_st"]),
+            :region => URLcrypt.decrypt(download_resp["result"]["region"]))
+        File.open(project_home+"/"+absolute_path, 'wb') do |file|
+          resp = s3.get_object({ bucket:URLcrypt.decrypt(download_resp["result"]["bucket"]),
+                                 key:URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
+        end
+          return true
+        end
+
+      rescue =>e
+        return false
+
+      end
+    end
+
+    def download_data_file(commit_sha1,dataset_home)
+      begin
+        res = Cnvrg::API.request(@base_resource + "download_data_file", 'POST', {commit_sha1:commit_sha1})
+        Cnvrg::CLI.is_response_success(res, false)
+        if res["result"]
+          download_resp = res
+          filename = download_resp["result"]["filename"]
+
+          sts_path = download_resp["result"]["path_sts"]
+          uri = URI.parse(sts_path)
+          http_object = Net::HTTP.new(uri.host, uri.port)
+          http_object.use_ssl = true if uri.scheme == 'https'
+          request = Net::HTTP::Get.new(sts_path)
+
+          body = ""
+          http_object.start do |http|
+            response = http.request request
+            body = response.read_body
+          end
+          URLcrypt::key = [body].pack('H*')
+          s3 = Aws::S3::Client.new(
+              :access_key_id => URLcrypt.decrypt(download_resp["result"]["sts_a"]),
+              :secret_access_key => URLcrypt.decrypt(download_resp["result"]["sts_s"]),
+              :session_token => URLcrypt.decrypt(download_resp["result"]["sts_st"]),
+              :region => URLcrypt.decrypt(download_resp["result"]["region"]))
+
+          File.open(dataset_home+"/"+filename, 'wb') do |file|
+            resp = s3.get_object({ bucket:URLcrypt.decrypt(download_resp["result"]["bucket"]),
+                                   key:URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
+          end
+          return filename
+        end
+
+      rescue =>e
+        return false
+
+      end
+    end
+
+    def download_file(absolute_path, relative_path, project_home, conflict=false)
+      res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
+      Cnvrg::CLI.is_response_success(res, false)
+      if res["result"]
+        res = res["result"]
+        return false if res["link"].empty? or res["filename"].empty?
+        filename = res["filename"]
+        file_location = absolute_path.gsub(/#{filename}\/?$/, "")
+
+        FileUtils.mkdir_p project_home + "/" + file_location
+        filename += ".conflict" if conflict
+
+        File.open("#{project_home}/#{file_location}/#{filename}", "wb") do |file|
+          file.write open(res["link"]).read
+        end
+      else
+        return false
+      end
+      return true
+    end
+
+    def download_dir(dataset_home, absolute_path)
+      FileUtils.mkdir_p("#{dataset_home}/#{absolute_path}")
+    end
+    def revoke_download_dir(absolute_path)
+      puts FileUtils.rmtree("#{absolute_path}")
+    end
+
+    def revoke_download_file(absolute_path,filename,conflict=false)
+      begin
+      file_location = absolute_path.gsub(/#{filename}\/?$/, "")
+
+      filename += ".conflict" if conflict
+      FileUtils.remove("#{file_location}/#{filename}")
+        return true
+      rescue
+       return false
+      end
+    end
+
+    def start_commit(new_branch)
+
+      response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug,new_branch:false,
+                                                                              username: @owner})
+      Cnvrg::CLI.is_response_success(response,false)
+      return response
+    end
+
+    def end_commit(commit_sha1)
+      response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1})
+      return response
+    end
+    def end_commit_tar(commit_sha1,cur_idx)
+      response = Cnvrg::API.request("#{base_resource}/commit/end_tar", 'POST', {commit_sha1: commit_sha1,idx: cur_idx})
+      return response
+    end
+
+    def rollback_commit(commit_sha1)
+      response = Cnvrg::API.request("#{base_resource}/commit/rollback", 'POST', {commit_sha1: commit_sha1})
+      Cnvrg::CLI.is_response_success(response, false)
+    end
+  end
+end