cnvrg 0.0.145 → 0.0.146

data/lib/cnvrg/data.rb

@@ -0,0 +1,26 @@
+require 'thor'
+
+class SubCommandBase < Thor
+  def self.banner(command, namespace = nil, subcommand = false)
+    "#{basename} #{subcommand_prefix} #{command.usage}"
+  end
+
+  def self.subcommand_prefix
+    self.name.gsub(%r{.*::}, '').gsub(%r{^[A-Z]}) { |match| match[0].downcase }.gsub(%r{[A-Z]}) { |match| "-#{match[0].downcase}" }
+  end
+  end
+module Cnvrg
+  class Data < SubCommandBase
+    desc "init", "init data folder"
+
+    def init
+      puts "im in init"
+    end
+    desc "upload", "upload data folder"
+
+    def upload
+      puts "im in upload"
+    end
+
+  end
+end

data/lib/cnvrg/datafiles.rb

@@ -0,0 +1,376 @@
+require 'mimemagic'
+require 'aws-sdk'
+require 'URLcrypt'
+
+module Cnvrg
+  class Datafiles
+
+    LARGE_FILE=1024*1024*5
+    attr_reader :base_resource
+
+    def initialize(owner, dataset_slug)
+      @dataset_slug = dataset_slug
+      @owner = owner
+      @base_resource = "users/#{owner}/datasets/#{dataset_slug}/"
+    end
+
+    def upload_file(absolute_path, relative_path, commit_sha1)
+      file_name = File.basename relative_path
+      file_size = File.size(absolute_path).to_f
+      mime_type = MimeMagic.by_path(absolute_path)
+      content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
+      upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
+                                                                                     commit_sha1: commit_sha1, file_name: file_name,
+                                                                                     file_size: file_size, file_content_type: content_type})
+      if Cnvrg::CLI.is_response_success(upload_resp, false)
+        path = upload_resp["result"]["path"]
+        if file_size.to_f>= Cnvrg::Files::LARGE_FILE.to_f
+          s3_res = upload_large_files_s3(upload_resp, absolute_path)
+        else
+          s3_res = upload_small_files_s3(path, absolute_path, content_type)
+        end
+        if s3_res
+          Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
+                                                                    blob_id: upload_resp["result"]["id"]})
+          return true
+        end
+      end
+      return false
+    end
+    def upload_log_file(absolute_path, relative_path,log_date)
+      file_name = File.basename relative_path
+      file_size = File.size(absolute_path).to_f
+      content_type = "text/x-log"
+      upload_resp = Cnvrg::API.request("/users/#{@owner}/" + "upload_cli_log", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
+                                                                                     file_name: file_name,log_date:log_date,
+                                                                                     file_size: file_size, file_content_type: content_type})
+      if Cnvrg::CLI.is_response_success(upload_resp, false)
+          path = upload_resp["result"]["path"]
+          s3_res = upload_small_files_s3(path, absolute_path,  "text/plain")
+      end
+      if s3_res
+        return true
+      end
+      return false
+
+    end
+    def upload_exec_file(absolute_path,image_name,commit_id)
+      file_name = File.basename absolute_path
+      file_size = File.size(absolute_path).to_f
+      content_type = "application/zip"
+      begin
+        upload_resp = Cnvrg::API.request("users/#{@owner}/images/" + "upload_config", 'POST_FILE', {relative_path: absolute_path,
+                                                                                                    file_name: file_name,
+                                                                                                    image_name:image_name,
+                                                                                                    file_size: file_size,
+                                                                                                    file_content_type: content_type,
+                                                                                                    project_slug: @project_slug,
+                                                                                                    commit_id:commit_id})
+        # puts upload_resp
+        if Cnvrg::CLI.is_response_success(upload_resp, false)
+          if upload_resp["result"]["image"] == -1
+            return -1
+          end
+          path = upload_resp["result"]["path"]
+          s3_res = upload_small_files_s3(path, absolute_path, content_type)
+
+        end
+      if s3_res
+        return upload_resp["result"]["id"]
+      end
+      return false
+    rescue SignalException
+
+      say "\nAborting"
+      exit(1)
+    end
+
+  end
+
+
+
+  def upload_image(absolute_path, image_name, owner, is_public, is_base,dpkg,libraries,bash,message,commit_id)
+      file_name = File.basename absolute_path
+      file_size = File.size(absolute_path).to_f
+      if is_base
+
+        content_type = "application/zip"
+      else
+        content_type = "application/gzip"
+      end
+      begin
+      upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload_cnvrg", 'POST_FILE', {relative_path: absolute_path,
+                                                                                          file_name: file_name,
+                                                                                          image_name: image_name,
+                                                                                          file_size: file_size,
+                                                                                          file_content_type: content_type,
+                                                                                          is_public: is_public,
+                                                                                          project_slug: @project_slug,
+                                                                                          commit_id:commit_id ,
+                                                                                          dpkg: dpkg,
+                                                                                          py2: libraries,
+                                                                                          py3: libraries,
+
+                                                                                          bash_history: bash,
+                                                                                          commit_message:message,
+                                                                                          is_base: is_base})
+      # puts upload_resp
+      if Cnvrg::CLI.is_response_success(upload_resp, false)
+        path = upload_resp["result"]["path"]
+        s3_res = upload_small_files_s3(path, absolute_path, content_type)
+        if s3_res
+          commit_resp = Cnvrg::API.request("users/#{owner}/images/#{upload_resp["result"]["id"]}/" + "commit", 'GET')
+          if Cnvrg::CLI.is_response_success(commit_resp, false)
+            return commit_resp["result"]["image"]
+          else
+            return false
+          end
+
+        end
+      end
+      return false
+      rescue =>e
+      end
+
+    end
+    def upload_cnvrg_image(absolute_path, image_name, owner, is_public, is_base,dpkg,libraries,bash,message)
+      file_name = File.basename absolute_path
+      file_size = File.size(absolute_path).to_f
+      if is_base
+
+        content_type = "application/zip"
+      else
+        content_type = "application/gzip"
+      end
+      begin
+        upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload_cnvrg", 'POST_FILE', {relative_path: absolute_path,
+                                                                                            file_name: file_name,
+                                                                                            image_name: image_name,
+                                                                                            file_size: file_size,
+                                                                                            file_content_type: content_type,
+                                                                                            is_public: is_public,
+                                                                                            dpkg: dpkg,
+                                                                                            libraries: libraries,
+                                                                                            bash_history: bash,
+                                                                                            commit_message:message,
+                                                                                            is_base: is_base})
+        # puts upload_resp
+        if Cnvrg::CLI.is_response_success(upload_resp, false)
+          path = upload_resp["result"]["path"]
+          s3_res = upload_small_files_s3(path, absolute_path, content_type)
+          if s3_res
+            commit_resp = Cnvrg::API.request("users/#{owner}/images/#{upload_resp["result"]["id"]}/" + "commit", 'GET')
+            if Cnvrg::CLI.is_response_success(commit_resp, false)
+              return commit_resp["result"]["image"]
+            else
+              return false
+            end
+
+          end
+        end
+        return false
+      rescue =>e
+      end
+
+    end
+
+    def download_image(file_path_to_store, image_slug, owner)
+
+
+      download_resp = Cnvrg::API.request("users/#{owner}/images/#{image_slug}/" + "download", 'GET')
+      path = download_resp["result"]["path"]
+
+      if Cnvrg::CLI.is_response_success(download_resp, false)
+        begin
+          open(file_path_to_store, 'wb') do |file|
+            file << open(path).read
+          end
+
+          return true
+        rescue => e
+          return false
+        end
+
+        return true
+      else
+        return false
+      end
+
+
+    end
+
+    def upload_large_files_s3(upload_resp, file_path)
+      begin
+        sts_path = upload_resp["result"]["path_sts"]
+        uri = URI.parse(sts_path)
+        http_object = Net::HTTP.new(uri.host, uri.port)
+        http_object.use_ssl = true if uri.scheme == 'https'
+        request = Net::HTTP::Get.new(sts_path)
+
+        body = ""
+        http_object.start do |http|
+          response = http.request request
+          body = response.read_body
+        end
+        URLcrypt::key = [body].pack('H*')
+        puts  "file_size: #{File.size(file_path).to_f}"
+        puts "start_time: #{Time.now}"
+        s3 = Aws::S3::Resource.new(
+            :access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
+            :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
+            :session_token => URLcrypt.decrypt(upload_resp["result"]["sts_st"]),
+            :region => URLcrypt.decrypt(upload_resp["result"]["region"]))
+        resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
+            object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
+            upload_file(file_path,{:use_accelerate_endpoint=>true})
+        puts "end_time: #{Time.now}"
+
+        return resp
+      rescue =>e
+        puts e
+        return false
+
+      end
+      return true
+
+    end
+
+    def upload_small_files_s3(url_path, file_path, content_type)
+      url = URI.parse(url_path)
+      file = File.open(file_path, "rb")
+      body = file.read
+      begin
+        Net::HTTP.start(url.host) do |http|
+          http.send_request("PUT", url.request_uri, body, {
+              "content-type" => content_type,
+          })
+        end
+        return true
+      rescue Interrupt
+        return false
+      rescue
+        return false
+      end
+    end
+
+    def upload_url(file_path)
+      response = Cnvrg::API.request(@base_resource + "upload_url", 'POST', {file_s3_path: file_path})
+      if Cnvrg::CLI.is_response_success(response, false)
+        return response
+      else
+        return nil
+      end
+
+    end
+
+    def delete_file(absolute_path, relative_path, commit_sha1)
+      response = Cnvrg::API.request(@base_resource + "delete_file", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
+      return Cnvrg::CLI.is_response_success(response, false)
+    end
+
+    def delete_dir(absolute_path, relative_path, commit_sha1)
+      response = Cnvrg::API.request(@base_resource + "delete_dir", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
+      return Cnvrg::CLI.is_response_success(response, false)
+    end
+
+    def create_dir(absolute_path, relative_path, commit_sha1)
+      response = Cnvrg::API.request(@base_resource + "create_dir", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
+      return Cnvrg::CLI.is_response_success(response, false)
+    end
+    def download_file_s3(absolute_path, relative_path, project_home, conflict=false)
+      begin
+        res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
+        Cnvrg::CLI.is_response_success(res, false)
+        if res["result"]
+          download_resp = res
+          filename = download_resp["result"]["filename"]
+
+          absolute_path += ".conflict" if conflict
+          sts_path = download_resp["result"]["path_sts"]
+        uri = URI.parse(sts_path)
+        http_object = Net::HTTP.new(uri.host, uri.port)
+        http_object.use_ssl = true if uri.scheme == 'https'
+        request = Net::HTTP::Get.new(sts_path)
+
+        body = ""
+        http_object.start do |http|
+          response = http.request request
+          body = response.read_body
+        end
+        URLcrypt::key = [body].pack('H*')
+        s3 = Aws::S3::Client.new(
+            :access_key_id => URLcrypt.decrypt(download_resp["result"]["sts_a"]),
+            :secret_access_key => URLcrypt.decrypt(download_resp["result"]["sts_s"]),
+            :session_token => URLcrypt.decrypt(download_resp["result"]["sts_st"]),
+            :region => URLcrypt.decrypt(download_resp["result"]["region"]))
+        File.open(project_home+"/"+absolute_path, 'wb') do |file|
+          resp = s3.get_object({ bucket:URLcrypt.decrypt(download_resp["result"]["bucket"]),
+                                 key:URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
+        end
+          return true
+        end
+
+      rescue =>e
+        return false
+
+      end
+    end
+    def download_file(absolute_path, relative_path, project_home, conflict=false)
+      res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
+      Cnvrg::CLI.is_response_success(res, false)
+      if res["result"]
+        res = res["result"]
+        return false if res["link"].empty? or res["filename"].empty?
+        filename = res["filename"]
+        file_location = absolute_path.gsub(/#{filename}\/?$/, "")
+
+        FileUtils.mkdir_p project_home + "/" + file_location
+        filename += ".conflict" if conflict
+
+        File.open("#{project_home}/#{file_location}/#{filename}", "wb") do |file|
+          file.write open(res["link"]).read
+        end
+      else
+        return false
+      end
+      return true
+    end
+
+    def download_dir(absolute_path, relative_path, project_home)
+      FileUtils.mkdir_p("#{project_home}/#{absolute_path}")
+    end
+    def revoke_download_dir(absolute_path, relative_path, project_home)
+      puts FileUtils.rmtree("#{absolute_path}")
+    end
+
+    def revoke_download_file(project_home,absolute_path,filename,conflict=false)
+      begin
+      file_location = absolute_path.gsub(/#{filename}\/?$/, "")
+
+      filename += ".conflict" if conflict
+      FileUtils.remove("#{file_location}/#{filename}")
+        return true
+      rescue
+       return false
+      end
+    end
+
+    def start_commit(new_branch)
+
+      response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug,new_branch:false,
+                                                                              username: @owner})
+      Cnvrg::CLI.is_response_success(response)
+      return response
+    end
+
+    def end_commit(commit_sha1)
+      response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1})
+      return response
+    end
+
+    def rollback_commit(commit_sha1)
+      response = Cnvrg::API.request("#{base_resource}/commit/rollback", 'POST', {commit_sha1: commit_sha1})
+      Cnvrg::CLI.is_response_success(response, false)
+    end
+  end
+end

data/lib/cnvrg/dataset.rb

@@ -0,0 +1,196 @@
+require 'fileutils'
+module Cnvrg
+  class Dataset
+    attr_reader :slug, :owner, :title, :local_path, :working_dir
+
+    RemoteURL ||= "https://cnvrg.io"
+
+    def initialize(project_home)
+      begin
+        config = YAML.load_file(project_home+"/.cnvrg/config.yml")
+        @local_path = project_home
+        @title = config[:dataset_name]
+        @slug = config[:dataset_slug]
+        @owner = config[:owner]
+        @working_dir = project_home
+      rescue => e
+      end
+
+    end
+
+
+    def last_local_commit
+      idx = YAML.load_file(@local_path + "/.cnvrg/idx.yml")
+      return idx[:commit]
+    end
+
+    def url
+      url = Cnvrg::Helpers.remote_url
+      "#{url}/#{self.owner}/projects/#{self.slug}"
+    end
+
+    def update_ignore_list(new_ignore)
+      if new_ignore.empty?
+        return true
+      end
+      begin
+        File.open(self.local_path+"/.cnvrgignore", "a+") do |f|
+          new_ignore.each do |i|
+            f.puts("#{i}\n")
+          end
+        end
+        return true
+      rescue
+        return false
+      end
+    end
+
+    def get_ignore_list
+      ignore_list = []
+      File.open(self.local_path+"/.cnvrgignore", "r").each_line do |line|
+        line = line.strip
+        if line.start_with? "#"
+          next
+        end
+        if line.end_with? "/"
+          ignore_list << line.chop
+          sub_dirs = Dir.glob("#{line}/**/*").each { |x| x.gsub!("//", "/") }
+          ignore_list << sub_dirs.flatten
+        else
+          ignore_list << line
+        end
+      end
+      return ignore_list.flatten
+
+    end
+
+
+    def self.init(owner, dataset_name, is_public=false)
+        list_dirs = [".cnvrg"
+        ]
+        list_files = [
+            ".cnvrgignore",
+            ".cnvrg/config.yml"
+        ]
+
+        cnvrgignore = Helpers.cnvrgignore_content
+        begin
+          response = Cnvrg::API.request("cli/create_dataset", 'POST', {title: dataset_name, owner: owner, is_public: is_public})
+          Cnvrg::CLI.is_response_success(response)
+          response = JSON.parse response["result"]
+          dataset_slug = response["slug"]
+
+          config = {dataset_name: dataset_name,
+                    dataset_slug: dataset_slug,
+                    owner: owner}
+          FileUtils.mkdir_p list_dirs
+          FileUtils.touch list_files
+          File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+          File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
+        rescue => e
+
+          return false
+        end
+        return true
+      end
+
+
+      def get_idx
+        YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+      end
+    def url
+      url = Cnvrg::Helpers.remote_url
+      "#{url}/#{self.owner}/datsets/#{self.slug}"
+    end
+    def generate_idx
+      if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
+        old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+      else
+        old_idx = nil
+      end
+
+      tree_idx = Hash.new(0)
+
+      list = Dir.glob("#{self.local_path}/**/**", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) }
+      list_ignore = self.get_ignore_list()
+      list.each do |e|
+        label = e.gsub(self.local_path + "/", "")
+        if File.directory? e
+          if list_ignore.include? label
+            next
+          end
+          tree_idx[label+"/"] = nil
+        else
+          if list_ignore.include? label
+            next
+          end
+          sha1 = Digest::SHA1.file(e).hexdigest
+          if old_idx.nil? or old_idx.to_h[:tree].nil?
+            tree_idx[label] = {sha1: sha1, commit_time: nil}
+          elsif old_idx[:tree][label].nil? or old_idx[:tree][label][:sha1] != sha1
+            tree_idx[label] = {sha1: sha1, commit_time: nil}
+          else
+            tree_idx[label] = old_idx[:tree][label]
+          end
+        end
+      end
+
+      idx = {commit: old_idx.to_h[:commit], tree: tree_idx}
+
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
+      return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+    end
+    def create_volume
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/volumes/create", 'POST')
+      CLI.is_response_success(response)
+      return response
+    end
+
+      def compare_idx(new_branch, commit=last_local_commit)
+
+        local_idx = self.generate_idx
+        response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status", 'POST', {idx: local_idx, new_branch: new_branch, current_commit: commit})
+        CLI.is_response_success(response)
+        return response
+      end
+
+      def compare_commit(commit)
+        if commit.nil? or commit.empty?
+          commit = last_local_commit
+        end
+        response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commit/compare", 'POST', {current_commit: commit})
+        CLI.is_response_success(response)
+        update_is_new_branch(response["result"]["new_branch"])
+        return response["result"]["new_branch"]
+      end
+
+      def update_idx_with_files_commits!(files, commit_time)
+
+        idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+        files.each do |path|
+          idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
+        end
+        File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
+
+        return true
+      end
+
+
+      def update_idx_with_commit!(commit)
+        idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+        idx_hash[:commit] = commit
+
+        File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
+        return true
+      end
+
+      def revert(working_dir)
+        FileUtils.rm_rf working_dir
+        # response     = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/revert", 'GET')
+        # CLI.is_response_success(response)
+
+
+      end
+
+    end
+  end