cnvrg 0.0.15 → 0.0.140

data/lib/cnvrg/dataset.rb

@@ -1,296 +0,0 @@
-require 'fileutils'
-module Cnvrg
-  class Dataset
-    attr_reader :slug, :owner, :title, :local_path, :working_dir
-
-    RemoteURL ||= "https://cnvrg.io"
-
-    def initialize(project_home)
-      begin
-        config = YAML.load_file(project_home+"/.cnvrg/config.yml")
-        @local_path = project_home
-        @title = config[:dataset_name]
-        @slug = config[:dataset_slug]
-        @owner = config[:owner]
-        @working_dir = project_home
-      rescue => e
-
-      end
-
-    end
-
-    def last_local_commit
-      idx = YAML.load_file(@local_path + "/.cnvrg/idx.yml")
-      return idx[:commit]
-    end
-
-    def snapshot
-      commit = last_local_commit
-      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/volumes/create", 'POST', {data_commit: commit})
-      CLI.is_response_success(response)
-      return response
-
-    end
-    def list(owner)
-      response = Cnvrg::API.request("users/#{owner}/datasets/list", 'GET')
-      CLI.is_response_success(response)
-      return response
-
-    end
-    def list_commits
-      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/list_commits", 'GET')
-      CLI.is_response_success(response)
-      return response
-
-    end
-    def url
-      url = Cnvrg::Helpers.remote_url
-      "#{url}/#{self.owner}/projects/#{self.slug}"
-    end
-
-    def update_ignore_list(new_ignore)
-
-      if new_ignore.nil? or new_ignore.empty?
-        return true
-      end
-      begin
-        File.open(self.local_path+"/.cnvrgignore", "a+") do |f|
-          f.puts("\n")
-
-          new_ignore.each do |i|
-            f.puts("#{i}\n")
-          end
-        end
-        return true
-      rescue
-        return false
-      end
-    end
-
-    def get_ignore_list
-      ignore_list = []
-      File.open(self.local_path+"/.cnvrgignore", "r").each_line do |line|
-        line = line.strip
-
-        if line.start_with? "#" or ignore_list.include? line
-          next
-        end
-        if line.end_with? "/" or File.directory?(line)
-          if line.end_with? "/"
-            ignore_list << line.chop
-          else
-            ignore_list << line
-          end
-          all_sub = Dir.glob("#{line}/**/*", File::FNM_DOTMATCH).flatten
-          ignore_list << all_sub.flatten
-          ignore_list << line
-        else
-          ignore_list << line
-        end
-      end
-      return ignore_list.flatten
-
-    end
-
-
-    def self.init(owner, dataset_name, is_public=false)
-        list_dirs = [".cnvrg"
-        ]
-        list_files = [
-            ".cnvrgignore",
-            ".cnvrg/config.yml"
-        ]
-
-        cnvrgignore = Helpers.cnvrgignore_content
-        begin
-            response = Cnvrg::API.request("cli/create_dataset", 'POST', {title: dataset_name, owner: owner, is_public: is_public})
-            Cnvrg::CLI.is_response_success(response)
-          response = JSON.parse response["result"]
-          dataset_slug = response["slug"]
-
-          config = {dataset_name: dataset_name,
-                    dataset_slug: dataset_slug,
-                    owner: owner}
-          FileUtils.mkdir_p list_dirs
-          FileUtils.touch list_files
-          File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
-          File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
-        rescue => e
-          "Exception in init data: #{e.message}"
-          return false
-        end
-        return true
-    end
-    def self.clone(owner, dataset_name,dataset_slug)
-
-      begin
-        list_dirs = [ dataset_name, "#{dataset_name}/.cnvrg"
-        ]
-        list_files = [
-            "#{dataset_name}/.cnvrgignore",
-            "#{dataset_name}/.cnvrg/config.yml"
-        ]
-
-        config = {dataset_name: dataset_name,
-                  dataset_slug: dataset_slug,
-                  owner: owner}
-
-
-        cnvrgignore = Helpers.cnvrgignore_content
-        FileUtils.mkdir_p list_dirs
-        FileUtils.touch list_files
-        File.open("#{dataset_name}/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
-        File.open("#{dataset_name}/.cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
-      rescue => e
-        puts "Exception in clone request:#{e.message}"
-        return false
-      end
-      return true
-    end
-
-    def self.init_container(owner, dataset_slug,dataset_name)
-
-      cnvrgignore = Helpers.cnvrgignore_content
-      begin
-        list_dirs = [ ".cnvrg"
-        ]
-        list_files = [
-            ".cnvrgignore",
-            ".cnvrg/config.yml"
-        ]
-        FileUtils.mkdir_p list_dirs
-        FileUtils.touch list_files
-
-        config = {dataset_name: dataset_name,
-                  dataset_slug: dataset_slug,
-                  owner: owner}
-        File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
-
-        File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
-      rescue => e
-        return false
-      end
-      return true
-    end
-
-
-      def get_idx
-        YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
-      end
-    def url
-      url = Cnvrg::Helpers.remote_url
-      "#{url}/#{self.owner}/datasets/#{self.slug}"
-    end
-    def generate_idx
-      if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
-        old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
-      else
-        old_idx = nil
-      end
-
-      tree_idx = Hash.new(0)
-
-      list = Dir.glob("#{self.local_path}/**/**", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) }
-      list_ignore = self.get_ignore_list()
-      list.each do |e|
-        label = e.gsub(self.local_path + "/", "")
-        if File.directory? e
-          if list_ignore.include? label
-            next
-          end
-          tree_idx[label+"/"] = nil
-        else
-          if list_ignore.include? label
-            next
-          end
-          sha1 = Digest::SHA1.file(e).hexdigest
-          if old_idx.nil? or old_idx.to_h["tree"].nil?
-            tree_idx[label] = {sha1: sha1, commit_time: nil}
-          elsif old_idx["tree"][label].nil? or old_idx["tree"][label]["sha1"] != sha1
-            tree_idx[label] = {sha1: sha1, commit_time: nil}
-          else
-            tree_idx[label] = old_idx["tree"][label]
-          end
-        end
-      end
-
-      idx = {commit: old_idx.to_h[:commit], tree: tree_idx}
-
-      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
-      return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
-    end
-    def create_volume
-      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/volumes/create", 'POST')
-      CLI.is_response_success(response)
-      return response
-    end
-
-    def downlowd_updated_data(current_commit)
-
-      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/downlowd_updated_data", 'POST', {current_commit: current_commit})
-      CLI.is_response_success(response,false)
-      return response
-    end
-    def compare_idx(new_branch, commit=last_local_commit,local_idx=nil)
-      if local_idx.nil?
-        local_idx =  self.generate_idx
-      end
-
-      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status", 'POST', {idx: local_idx, new_branch: new_branch, current_commit: commit})
-      CLI.is_response_success(response,false)
-      return response
-    end
-
-
-
-    def compare_commits(commit)
-      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/compare_commits", 'POST', {compare_commit: commit,current_commit:last_local_commit})
-      CLI.is_response_success(response,false)
-      return response
-    end
-
-    def compare_commit(commit)
-        if commit.nil? or commit.empty?
-          commit = last_local_commit
-        end
-        response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commit/compare", 'POST', {current_commit: commit})
-        CLI.is_response_success(response,false)
-        update_is_new_branch(response["result"]["new_branch"])
-        return response["result"]["new_branch"]
-      end
-
-      def update_idx_with_files_commits!(files, commit_time)
-
-        idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
-        files.each do |path|
-          idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
-        end
-        File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
-
-        return true
-      end
-    def update_idx(idx)
-      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
-
-      return true
-    end
-
-
-      def update_idx_with_commit!(commit)
-        idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
-        idx_hash[:commit] = commit
-
-        File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
-        return true
-      end
-
-      def revert(working_dir)
-        FileUtils.rm_rf working_dir
-        # response     = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/revert", 'GET')
-        # CLI.is_response_success(response)
-
-
-      end
-
-    end
-  end

data/lib/cnvrg/ssh.rb

@@ -1,95 +0,0 @@
-require 'fileutils'
-require 'cnvrg/files'
-require 'docker'
-require 'net/ssh'
-
-
-module Cnvrg
-  class Ssh
-    attr_reader :is_ssh
-
-
-    def initialize(resp)
-      begin
-        @is_ssh = false
-        sts_path = resp["result"]["sts_path"]
-
-        uri = URI.parse(sts_path)
-
-        http_object = Net::HTTP.new(uri.host, uri.port)
-        http_object.use_ssl = true if uri.scheme == 'https'
-        request = Net::HTTP::Get.new(sts_path)
-
-        body = ""
-        http_object.start do |http|
-          response = http.request request
-          body = response.read_body
-        end
-
-        URLcrypt::key = [body].pack('H*')
-        ip = URLcrypt.decrypt(resp["result"]["machine_i"])
-
-        @user = URLcrypt.decrypt(resp["result"]["machine_u"])
-        key = URLcrypt.decrypt(resp["result"]["machine_k"])
-        @container = URLcrypt.decrypt(resp["result"]["machine_c"])
-
-        tempssh = Tempfile.new "sshkey"
-        tempssh.write open(key).read
-        tempssh.rewind
-        key_path = tempssh.path
-        count = 0
-        while count < 5
-
-          begin
-            @ssh = Net::SSH.start(ip, user=@user, :keys => key_path, :timeout => 10)
-            if !@ssh.nil?
-              @is_ssh = true
-              return
-            else
-              count+=1
-              sleep(2)
-
-            end
-          rescue
-            count+=1
-            sleep(2)
-
-
-          end
-        end
-        if tempssh
-          tempssh.close
-          tempssh.unlink
-        end
-        return false
-      rescue => e
-
-        puts e
-      end
-    end
-
-
-
-    def exec_command(command)
-      exec_command = "sudo -i -u #{@user} cnvrg exec_container #{@container} \"#{command}\" "
-      return @ssh.exec!(exec_command)
-    end
-
-
-    def close_ssh()
-
-
-      begin
-
-        @ssh.close
-      rescue => e
-
-      end
-
-
-    end
-  end
-
-
-
-end