cnvrg 0.5.0 → 0.5.5

data/lib/cnvrg/data.rb

@@ -26,6 +26,7 @@ module Cnvrg
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s","--sync"], :default => false
+    method_option :tags, :type => :boolean, :aliases => ["--tags"], :desc => "upload file tags", :default => false
 
     def upload
       cli = Cnvrg::CLI.new()
@@ -33,14 +34,17 @@ module Cnvrg
       sync = options["sync"]
       force = options["force"]
       new_branch = options["new_branch"]
-      cli.upload_data_new(new_branch,  verbose,sync,force)
+      tags = options["tags"]
+      cli.upload_data_new(new_branch,  verbose,sync,force, tags)
     end
     desc 'sync', 'sync_data_new'
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :commit, :type => :string, :aliases => ["-c"], :desc => "download specified commit", :default => nil
-    method_option :all_files, :type => :boolean, :aliases => ["--all"], :desc => "download specified commit", :default => true
+    method_option :all_files, :type => :boolean, :aliases => ["--all"], :desc => "download specified commit", :default => false
+    method_option :tags, :type => :boolean, :aliases => ["--tags"], :desc => "upload file tags", :default => false
+    method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
 
     def sync_data_new()
       cli = Cnvrg::CLI.new()
@@ -49,8 +53,11 @@ module Cnvrg
       verbose = options["verbose"]
       commit = options["commit"]
       all_files = options["all_files"]
+      tags = options["tags"]
+      parallel=options["parallel"]
+
 
-      cli.sync_data_new(new_branch, force, verbose,commit,all_files)
+      cli.sync_data_new(new_branch, force, verbose,commit,all_files, tags,parallel)
     end
     desc 'data download', 'pull data'
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits", :default => false
@@ -73,14 +80,26 @@ module Cnvrg
     desc 'data clone', 'clone datset'
     method_option :only_tree, :type => :boolean, :aliases => ["-t", "--tree"], :default => false
     method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => nil
+    method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => nil
+
     def clone(dataset_url)
       cli = Cnvrg::CLI.new()
-    only_tree =options[:only_tree]
+      only_tree =options[:only_tree]
       commit =options[:commit]
+      query =options[:query]
+
+        cli.clone_data(dataset_url, only_tree=only_tree,commit=commit, query=query)
 
-        cli.clone_data(dataset_url, only_tree=only_tree,commit=commit)
+    end
 
+    desc 'data clone_query', 'clone query datset'
+    method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => nil
+    def clone_query(dataset_url)
+      cli = Cnvrg::CLI.new()
+      query =options[:query]
+      cli.clone_data_query(dataset_url,query=query)
     end
+
     desc 'data delete', 'delete datset'
     def delete(dataset_slug)
       cli = Cnvrg::CLI.new()
@@ -102,5 +121,23 @@ module Cnvrg
 
     end
 
+    desc 'data queries', 'list of all queries'
+    def queries()
+      cli = Cnvrg::CLI.new()
+      cli.queries()
+    end
+
+    desc 'data query_files', 'list of all queries'
+    def query_files(query)
+      cli = Cnvrg::CLI.new()
+      cli.query_files(query)
+    end
+
+    desc 'data download_tags_yaml', 'Download dataset tags yml file in current directory'
+    def download_tags_yaml
+      cli = Cnvrg::CLI.new()
+      cli.download_tags_yaml()
+    end
+
   end
 end

data/lib/cnvrg/datafiles.rb

@@ -1,10 +1,13 @@
 require 'mimemagic'
 require 'aws-sdk'
 require 'URLcrypt'
-require "down"
+require 'parallel'
+require 'fileutils'
+
 
 module Cnvrg
   class Datafiles
+    ParallelThreads ||= 15
 
     LARGE_FILE=1024*1024*5
     MULTIPART_SPLIT=10000000
@@ -23,9 +26,44 @@ module Cnvrg
       mime_type = MimeMagic.by_path(absolute_path)
       content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
       sha1 =  OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
-      upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
-                                                                                     commit_sha1: commit_sha1, file_name: file_name,
-                                                                                     file_size: file_size, file_content_type: content_type, sha1: sha1})
+      if (absolute_path.include? "_tags.yml" or absolute_path.include? "_tags.yaml")
+        is_valid = false
+        begin
+        content = open(absolute_path).read()
+        hash = YAML.load(open(absolute_path).read())
+          # if level 1 keys count is 1
+          if hash.keys.count == 1
+            if hash["tags"].present?
+              is_valid = true
+            elsif hash[hash.keys.first].class != Hash
+              is_valid = true
+            end
+          # if level 1 keys count is greater than 1
+          elsif hash.keys.count > 1
+            if hash["tags"].present? and hash["tags"].class == Hash
+              is_valid = false
+            else
+              is_valid = true
+            end
+          end
+        rescue
+          is_valid = false
+        end
+
+        if is_valid
+          upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
+                                                                                         commit_sha1: commit_sha1, file_name: file_name,
+                                                                                         file_size: file_size, file_content_type: content_type, sha1: sha1, content: content})
+        else
+          puts("#{absolute_path} is invalid")
+          puts("Please check yaml structure.")
+        end
+      else
+        upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
+                                                                                       commit_sha1: commit_sha1, file_name: file_name,
+                                                                                       file_size: file_size, file_content_type: content_type, sha1: sha1})
+      end
+
       if Cnvrg::CLI.is_response_success(upload_resp, false)
         s3_res = upload_large_files_s3(upload_resp, absolute_path)
 
@@ -359,6 +397,82 @@ module Cnvrg
     def create_dir(absolute_path, relative_path, commit_sha1)
       response = Cnvrg::API.request(@base_resource + "create_dir", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
       return Cnvrg::CLI.is_response_success(response, false)
+    end
+    def download_list_files_in_query(response, dataset_home)
+      sts_path = response["path_sts"]
+      if !Helpers.is_verify_ssl
+        body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
+      else
+        body = open(sts_path).read
+      end
+      split = body.split("\n")
+      key = split[0]
+      iv = split[1]
+
+      access =  Cnvrg::Helpers.decrypt(key, iv, response["sts_a"])
+
+      secret =  Cnvrg::Helpers.decrypt(key,iv, response["sts_s"])
+
+      session =  Cnvrg::Helpers.decrypt(key,iv, response["sts_st"])
+      region =  Cnvrg::Helpers.decrypt(key,iv, response["region"])
+
+      bucket =  Cnvrg::Helpers.decrypt(key,iv, response["bucket"])
+      is_s3 = response["is_s3"]
+      if is_s3 or is_s3.nil?
+        client = Aws::S3::Client.new(
+            :access_key_id =>access,
+            :secret_access_key => secret,
+            :session_token => session,
+            :region => region,
+            :http_open_timeout => 60, :retry_limit => 20)
+      else
+        endpoint = Cnvrg::Helpers.decrypt(key,iv, response["endpoint_url"])
+        client = Aws::S3::Client.new(
+            :access_key_id =>access,
+            :secret_access_key => secret,
+            :region => region,
+            :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
+            :http_open_timeout => 60, :retry_limit => 20)
+      end
+      list_files = response["files"]
+      parallel_options = {
+          :progress => {
+              :title => "Download Progress",
+              :progress_mark => '=',
+              :format => "%b>>%i| %p%% %t",
+              :starting_at => 0,
+              :total => list_files.size,
+              :autofinish => true
+          },
+          in_threads: ParallelThreads
+      }
+      download_count = 0
+      Parallel.map((list_files), parallel_options) do |f|
+        file_key =  Cnvrg::Helpers.decrypt(key,iv, f["path"])
+        begin
+        begin
+          dir = File.dirname f["fullpath"]
+          FileUtils.mkdir_p(dataset_home+"/"+ dir) unless File.exist? (dataset_home+"/"+ dir)
+        end
+
+        File.open(dataset_home+"/"+f["fullpath"], 'w+') do |file|
+          resp = client.get_object({bucket:bucket,
+                                    key:file_key}, target: file)
+        end
+        download_count += 1
+        rescue
+        end
+
+      end
+      if download_count == list_files.size
+        return true
+      else
+        return false
+      end
+
+
+
+
     end
 
 

data/lib/cnvrg/dataset.rb

@@ -44,12 +44,57 @@ module Cnvrg
       return response
 
     end
+    def search_queries
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/queries/list", 'GET')
+      CLI.is_response_success(response)
+      row = [["name","id", "created_at", "username"]]
+      response["results"]["queries"].each do |query|
+        row << [query["name"],query["slug"], query["created_at"].in_time_zone.to_s, query["username"]]
+      end
+      return row
+
+    end
+    def get_query_file(query_slug)
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/search/#{query_slug}", 'GET')
+      CLI.is_response_success(response)
+      row = [["Name","Full path","URL"]]
+      response["results"]["query_files"].each do |file|
+        row << [file["name"],file["fullpath"],file["s3_url"]]
+      end
+      return row
+
+    end
+
+    def download_tags_yaml
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/tags_yml", 'GET')
+      CLI.is_response_success(response)
+      begin
+        path = self.working_dir
+        File.open("#{path}/#{response["results"]["filename"]}", "w+") { |f| f.write response["results"]["file_content"] }
+        return true
+      rescue
+        return false
+      end
+    end
+
     def list_commits
       response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/list_commits", 'GET')
       CLI.is_response_success(response)
       return response
 
     end
+
+    def upload_tags_via_yml(tag_file=nil)
+      records_yml = YAML.load_file(tag_file)
+      tag_file.close
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/data_tags_create", 'POST', {records_yml: records_yml})
+      if response["status"] == 200
+        return true
+      else
+        return false
+      end
+    end
+
     def url
       url = Cnvrg::Helpers.remote_url
       "#{url}/#{self.owner}/projects/#{self.slug}"
@@ -153,13 +198,44 @@ module Cnvrg
     end
 
 
+    def self.blank_clone(owner, dataset_name, dataset_slug)
+      list_dirs = [".cnvrg"
+      ]
+      list_files = [
+          ".cnvrg/config.yml"
+      ]
+      create_ignore = false
+      if !File.exist? ".cnvrgignore"
+        list_files << ".cnvrgignore"
+        create_ignore  = true
+      end
+
+
+      cnvrgignore = Helpers.cnvrgignore_content
+      begin
+
+        config = {dataset_name: dataset_name,
+                  dataset_slug: dataset_slug,
+                  owner: owner}
+
+        FileUtils.mkdir_p list_dirs
+        FileUtils.touch list_files
+        File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+        File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless !create_ignore
+      rescue => e
+        return false
+      end
+      return true
+    end
+
+
     def clone( commit)
 
 
       return response
     end
 
-    def self.clone(owner, dataset_name,dataset_slug)
+    def self.clone(owner, dataset_name, dataset_slug)
 
       begin
         list_dirs = [ dataset_name, "#{dataset_name}/.cnvrg"
@@ -216,7 +292,7 @@ module Cnvrg
       url = Cnvrg::Helpers.remote_url
       "#{url}/#{self.owner}/datasets/#{self.slug}"
     end
-    def generate_idx(list_ignore=nil)
+    def generate_idx(show_progress=false)
       if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
         old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
       else
@@ -227,7 +303,27 @@ module Cnvrg
 
       list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~/^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~/^#{self.local_path}\/\.cnvrgignore/) }
       list_ignore = self.get_ignore_list()
-      Parallel.map(list, in_threads: IDXParallelThreads) do |e|
+      if show_progress
+        parallel_options = {
+            :progress => {
+                :title => "Checking Dataset",
+                :progress_mark => '=',
+                :format => "%b>>%i| %p%% %t",
+                :starting_at => 0,
+                :total => (list).size,
+                :autofinish => true
+            },
+            in_threads: IDXParallelThreads,
+            isolation: true
+        }
+      else
+        parallel_options = {
+            in_threads: IDXParallelThreads,
+            isolation: true
+        }
+      end
+
+      Parallel.map(list, parallel_options ) do |e|
         label = e.gsub(self.local_path + "/", "")
         if File.directory? e
           if list_ignore.include? label
@@ -393,7 +489,7 @@ module Cnvrg
       end
 
       def update_idx_with_files_commits!(files, commit_time)
-        files.flatten!
+        # files.flatten!
         idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
         # idx_hash[:commit] = commit
 

data/lib/cnvrg/experiment.rb

@@ -4,6 +4,7 @@ module Cnvrg
     attr_reader :slug
     attr_reader :sync_before_terminate
     attr_reader :sync_delay_time
+    attr_reader :output_dir
 
     def initialize(owner, project_slug)
       @project_slug = project_slug
@@ -12,17 +13,21 @@ module Cnvrg
       @slug = nil
       @sync_before_terminate = nil
       @sync_delay_time = nil
+      @output_dir = nil
     end
 
-    def start(input, platform, machine_name, start_commit, name, email_notification, machine_activity,script_path,sync_before_terminate, periodic_sync)
+    def start(input, platform, machine_name, start_commit, name, email_notification, machine_activity,script_path,
+              sync_before_terminate, periodic_sync)
 
       res = Cnvrg::API.request(@base_resource + "experiment/start", 'POST',
-                               {input: input, platform: platform, machine_name: machine_name, start_commit: start_commit,sync_before_terminate:sync_before_terminate, periodic_sync:periodic_sync,
+                               {input: input, platform: platform, machine_name: machine_name, start_commit: start_commit,
                                 title: name, email_notification: email_notification, machine_activity: machine_activity,script_path:script_path})
       Cnvrg::CLI.is_response_success(res,false)
+
       @slug = res.to_h["result"].to_h["slug"]
       @sync_before_terminate = res.to_h["result"].to_h["sync_before_terminate"]
       @sync_delay_time = res.to_h["result"].to_h["sync_delay_time"]
+      @output_dir = res.to_h["result"].to_h["output_dir"]
       return res
 
     end
@@ -83,7 +88,8 @@ module Cnvrg
 
     end
 
-    def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp, grid,path_to_cmd,data, data_commit,periodic_sync, sync_before_terminate, max_time, ds_sync_options=0)
+    def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp, grid,path_to_cmd,data, data_commit,periodic_sync,
+                    sync_before_terminate, max_time, ds_sync_options=0,output_dir=nil,data_query=nil, git_commit=nil, git_branch=nil)
       response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
                                                                                                             commit_sha1: commit_to_run,
                                                                                                             instance_type: instance_type,
@@ -93,13 +99,15 @@ module Cnvrg
                                                                                                             path_to_cmd:path_to_cmd,dataset_slug:data,
                                                                                                             dataset_commit: data_commit,max_time:max_time,
                                                                                                             periodic_sync:periodic_sync, sync_before_terminate:sync_before_terminate,
-                                                                                                            dataset_sync_options:ds_sync_options})
+                                                                                                            dataset_sync_options:ds_sync_options,output_dir:output_dir,
+                                                                                                            dataset_query:data_query,git_commit:git_commit,git_branch:git_branch })
       return response
     end
-    def remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options=0)
+    def remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options=0,data_query=nil)
       response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/notebook/remote", 'POST', {instance_type: instance_type,dataset_slug:data,
                                                                                                           dataset_commit: data_commit,
-                                                                                                          commit:commit,notebook_type:notebook_type,dataset_sync_options:ds_sync_options})
+                                                                                                          commit:commit,notebook_type:notebook_type,dataset_sync_options:ds_sync_options,
+                                                                                                          dataset_query:data_query})
       return response
     end
 
@@ -130,6 +138,14 @@ module Cnvrg
 
     def restart_spot_instance
       restart = false
+      #TODO: remove this later
+      fall_number = rand(1..8)
+      if fall_number == 5
+        return true
+      else
+        return false
+      end
+
       begin
       url = URI.parse('http://169.254.169.254/latest/meta-data/spot/termination-time')
       req = Net::HTTP::Get.new(url.to_s)
@@ -143,14 +159,16 @@ module Cnvrg
         restart = false
       end
       rescue
+        restart = false
+
       end
 
       return restart
 
     end
 
-    def send_restart_request()
-      response = Cnvrg::API.request(@base_resource + "experiment/check_spot_instance", 'POST', {exp_slug: @slug})
+    def send_restart_request(sha1=nil)
+      Cnvrg::API.request(@base_resource + "experiment/check_spot_instance", 'POST', {exp_slug: @slug, end_commit: sha1})
     end
   end
 end