cnvrg 0.0.14.0 → 0.0.15

data/lib/cnvrg/dataset.rb

@@ -0,0 +1,296 @@
+require 'fileutils'
+module Cnvrg
+  class Dataset
+    attr_reader :slug, :owner, :title, :local_path, :working_dir
+
+    RemoteURL ||= "https://cnvrg.io"
+
+    def initialize(project_home)
+      begin
+        config = YAML.load_file(project_home+"/.cnvrg/config.yml")
+        @local_path = project_home
+        @title = config[:dataset_name]
+        @slug = config[:dataset_slug]
+        @owner = config[:owner]
+        @working_dir = project_home
+      rescue => e
+
+      end
+
+    end
+
+    def last_local_commit
+      idx = YAML.load_file(@local_path + "/.cnvrg/idx.yml")
+      return idx[:commit]
+    end
+
+    def snapshot
+      commit = last_local_commit
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/volumes/create", 'POST', {data_commit: commit})
+      CLI.is_response_success(response)
+      return response
+
+    end
+    def list(owner)
+      response = Cnvrg::API.request("users/#{owner}/datasets/list", 'GET')
+      CLI.is_response_success(response)
+      return response
+
+    end
+    def list_commits
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/list_commits", 'GET')
+      CLI.is_response_success(response)
+      return response
+
+    end
+    def url
+      url = Cnvrg::Helpers.remote_url
+      "#{url}/#{self.owner}/projects/#{self.slug}"
+    end
+
+    def update_ignore_list(new_ignore)
+
+      if new_ignore.nil? or new_ignore.empty?
+        return true
+      end
+      begin
+        File.open(self.local_path+"/.cnvrgignore", "a+") do |f|
+          f.puts("\n")
+
+          new_ignore.each do |i|
+            f.puts("#{i}\n")
+          end
+        end
+        return true
+      rescue
+        return false
+      end
+    end
+
+    def get_ignore_list
+      ignore_list = []
+      File.open(self.local_path+"/.cnvrgignore", "r").each_line do |line|
+        line = line.strip
+
+        if line.start_with? "#" or ignore_list.include? line
+          next
+        end
+        if line.end_with? "/" or File.directory?(line)
+          if line.end_with? "/"
+            ignore_list << line.chop
+          else
+            ignore_list << line
+          end
+          all_sub = Dir.glob("#{line}/**/*", File::FNM_DOTMATCH).flatten
+          ignore_list << all_sub.flatten
+          ignore_list << line
+        else
+          ignore_list << line
+        end
+      end
+      return ignore_list.flatten
+
+    end
+
+
+    def self.init(owner, dataset_name, is_public=false)
+        list_dirs = [".cnvrg"
+        ]
+        list_files = [
+            ".cnvrgignore",
+            ".cnvrg/config.yml"
+        ]
+
+        cnvrgignore = Helpers.cnvrgignore_content
+        begin
+            response = Cnvrg::API.request("cli/create_dataset", 'POST', {title: dataset_name, owner: owner, is_public: is_public})
+            Cnvrg::CLI.is_response_success(response)
+          response = JSON.parse response["result"]
+          dataset_slug = response["slug"]
+
+          config = {dataset_name: dataset_name,
+                    dataset_slug: dataset_slug,
+                    owner: owner}
+          FileUtils.mkdir_p list_dirs
+          FileUtils.touch list_files
+          File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+          File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
+        rescue => e
+          "Exception in init data: #{e.message}"
+          return false
+        end
+        return true
+    end
+    def self.clone(owner, dataset_name,dataset_slug)
+
+      begin
+        list_dirs = [ dataset_name, "#{dataset_name}/.cnvrg"
+        ]
+        list_files = [
+            "#{dataset_name}/.cnvrgignore",
+            "#{dataset_name}/.cnvrg/config.yml"
+        ]
+
+        config = {dataset_name: dataset_name,
+                  dataset_slug: dataset_slug,
+                  owner: owner}
+
+
+        cnvrgignore = Helpers.cnvrgignore_content
+        FileUtils.mkdir_p list_dirs
+        FileUtils.touch list_files
+        File.open("#{dataset_name}/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+        File.open("#{dataset_name}/.cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
+      rescue => e
+        puts "Exception in clone request:#{e.message}"
+        return false
+      end
+      return true
+    end
+
+    def self.init_container(owner, dataset_slug,dataset_name)
+
+      cnvrgignore = Helpers.cnvrgignore_content
+      begin
+        list_dirs = [ ".cnvrg"
+        ]
+        list_files = [
+            ".cnvrgignore",
+            ".cnvrg/config.yml"
+        ]
+        FileUtils.mkdir_p list_dirs
+        FileUtils.touch list_files
+
+        config = {dataset_name: dataset_name,
+                  dataset_slug: dataset_slug,
+                  owner: owner}
+        File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+
+        File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
+      rescue => e
+        return false
+      end
+      return true
+    end
+
+
+      def get_idx
+        YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+      end
+    def url
+      url = Cnvrg::Helpers.remote_url
+      "#{url}/#{self.owner}/datasets/#{self.slug}"
+    end
+    def generate_idx
+      if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
+        old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+      else
+        old_idx = nil
+      end
+
+      tree_idx = Hash.new(0)
+
+      list = Dir.glob("#{self.local_path}/**/**", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) }
+      list_ignore = self.get_ignore_list()
+      list.each do |e|
+        label = e.gsub(self.local_path + "/", "")
+        if File.directory? e
+          if list_ignore.include? label
+            next
+          end
+          tree_idx[label+"/"] = nil
+        else
+          if list_ignore.include? label
+            next
+          end
+          sha1 = Digest::SHA1.file(e).hexdigest
+          if old_idx.nil? or old_idx.to_h["tree"].nil?
+            tree_idx[label] = {sha1: sha1, commit_time: nil}
+          elsif old_idx["tree"][label].nil? or old_idx["tree"][label]["sha1"] != sha1
+            tree_idx[label] = {sha1: sha1, commit_time: nil}
+          else
+            tree_idx[label] = old_idx["tree"][label]
+          end
+        end
+      end
+
+      idx = {commit: old_idx.to_h[:commit], tree: tree_idx}
+
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
+      return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+    end
+    def create_volume
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/volumes/create", 'POST')
+      CLI.is_response_success(response)
+      return response
+    end
+
+    def downlowd_updated_data(current_commit)
+
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/downlowd_updated_data", 'POST', {current_commit: current_commit})
+      CLI.is_response_success(response,false)
+      return response
+    end
+    def compare_idx(new_branch, commit=last_local_commit,local_idx=nil)
+      if local_idx.nil?
+        local_idx =  self.generate_idx
+      end
+
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status", 'POST', {idx: local_idx, new_branch: new_branch, current_commit: commit})
+      CLI.is_response_success(response,false)
+      return response
+    end
+
+
+
+    def compare_commits(commit)
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/compare_commits", 'POST', {compare_commit: commit,current_commit:last_local_commit})
+      CLI.is_response_success(response,false)
+      return response
+    end
+
+    def compare_commit(commit)
+        if commit.nil? or commit.empty?
+          commit = last_local_commit
+        end
+        response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commit/compare", 'POST', {current_commit: commit})
+        CLI.is_response_success(response,false)
+        update_is_new_branch(response["result"]["new_branch"])
+        return response["result"]["new_branch"]
+      end
+
+      def update_idx_with_files_commits!(files, commit_time)
+
+        idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+        files.each do |path|
+          idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
+        end
+        File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
+
+        return true
+      end
+    def update_idx(idx)
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
+
+      return true
+    end
+
+
+      def update_idx_with_commit!(commit)
+        idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+        idx_hash[:commit] = commit
+
+        File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
+        return true
+      end
+
+      def revert(working_dir)
+        FileUtils.rm_rf working_dir
+        # response     = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/revert", 'GET')
+        # CLI.is_response_success(response)
+
+
+      end
+
+    end
+  end

data/lib/cnvrg/experiment.rb

@@ -9,11 +9,11 @@ module Cnvrg
       @slug = nil
     end
 
-    def start(input, platform, machine_name, start_commit, name, email_notification, machine_activity)
+    def start(input, platform, machine_name, start_commit, name, email_notification, machine_activity,script_path)
       res = Cnvrg::API.request(@base_resource + "experiment/start", 'POST',
                                {input: input, platform: platform, machine_name: machine_name, start_commit: start_commit,
-                                title: name, email_notification: email_notification, machine_activity: machine_activity})
-      Cnvrg::CLI.is_response_success(res)
+                                title: name, email_notification: email_notification, machine_activity: machine_activity,script_path:script_path})
+      Cnvrg::CLI.is_response_success(res,false)
 
       @slug = res.to_h["result"].to_h["slug"]
 
@@ -37,7 +37,7 @@ module Cnvrg
     def end_notebook_session(notebook_slug, end_commit)
       res = Cnvrg::API.request(@base_resource + "notebook/end_session", 'POST',
                                {notebook_slug: notebook_slug, end_commit: end_commit})
-      Cnvrg::CLI.is_response_success(res)
+      Cnvrg::CLI.is_response_success(res,false)
 
       return res
 
@@ -78,27 +78,36 @@ module Cnvrg
 
     end
 
-    def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp)
+    def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp, grid,path_to_cmd,data, data_commit)
       response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
                                                                                                             commit_sha1: commit_to_run,
                                                                                                             instance_type: instance_type,
                                                                                                             scheduling_query:scheduling_query,
-                                                                                                            local_timestamp:local_timestamp})
+                                                                                                            local_timestamp:local_timestamp,
+                                                                                                            grid: grid,
+                                                                                                            path_to_cmd:path_to_cmd,dataset_slug:data,
+                                                                                                            dataset_commit: data_commit})
       return response
     end
 
     def upload_temp_log(temp_log, cpu_average, memory_average)
       response = Cnvrg::API.request(@base_resource + "experiment/upload_temp_log", 'POST', {output: temp_log,
-                                                                                            exp_slug: @slug, cpu_average: cpu_average,
-                                                                                            memory_average: memory_average})
-      Cnvrg::CLI.is_response_success(response)
+                                                                                            exp_slug: @slug})
+      Cnvrg::CLI.is_response_success(response,false)
     end
 
     def end(output, exit_status, end_commit, cpu_average, memory_average)
+      #if remote try to remove
       response = Cnvrg::API.request(@base_resource + "experiment/end", 'POST', {output: output, exp_slug: @slug,
                                                                                 exit_status: exit_status, end_commit: end_commit,
                                                                                 cpu_average: cpu_average, memory_average: memory_average})
-      Cnvrg::CLI.is_response_success(response)
+      Cnvrg::CLI.is_response_success(response,false)
+
+      begin
+        FileUtils.rm_rf(["/home/ds/.cnvrg/tmp/exec.log"])
+      rescue
+
+      end
     end
   end
 end