cnvrg 1.2.5 → 1.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 6c380da2980fbcbf88cc7dd13b618a339b858f1e
-  data.tar.gz: 7622da00f49f89325195bd03caaa23920e7f9f0d
+SHA256:
+  metadata.gz: 8e33d716e3fce0ef29d6684c29dec59f6edd04173d0330343d0510ec11bd087a
+  data.tar.gz: 9a019b36e6611ffd4b627c7fb9b9edecbde44be360f26034b668c52da804da8d
 SHA512:
-  metadata.gz: ff8016c9f8e7b66b7818292e69f86971249ba225e3178f297c7f93b0bec1a8a85e14439cca76fdaf70afd1793d6eee1d083df3162af11d5c8f7e9de7b9b6aceb
-  data.tar.gz: e3f764b61ce441484238d01f42cebef55aedb00c2d93a23e663ba3f705551ad5c8aa8c7ef37a6486cfdffee938d2b21e5111638115db42c8ba3efe7ab19b8eec
+  metadata.gz: ad76658b08c85dcab2cade07327e5e6c7f4843e9eecbc6d2c768d4e3d166723e2f007d8260d83fd6d2f3cabbb25daf85bf7b247fe7b0bd45445f8af7cae2b141
+  data.tar.gz: 22da11b1b04ec3e55fbb9a3586651783411daa1fc1c5d06548890f71770b0e152740589b883394f84172f3e5d0f0519001abaa4416031bfa0856cd553be685c3

data/lib/cnvrg/Images.rb

@@ -20,7 +20,6 @@ module Cnvrg
         @owner = ""
         @username =  ""
         Cnvrg::Logger.log_info("cnvrg is not configured")
-
       end
 
     end

data/lib/cnvrg/cli.rb

@@ -449,6 +449,22 @@ module Cnvrg
     end
 
 
+    desc 'auth', "hidden..", hide: true
+    def auth
+      @token = ENV['CNVRG_TOKEN']
+      owner = ENV['CNVRG_OWNER']
+      user = ENV['CNVRG_USER']
+      api = ENV['CNVRG_API']
+      @email = ENV['CNVRG_EMAIL']
+
+      netrc = Netrc.read
+      netrc[Cnvrg::Helpers.netrc_domain] = @email, @token
+      netrc.save
+
+      set_owner(owner, user, api)
+    end
+
+
     desc 'login', 'Authenticate with cnvrg.io platform'
 
     def login
@@ -488,39 +504,6 @@ module Cnvrg
             choose_owner = owners[0]
           end
 
-          # if owners.empty?
-          # else
-          #   owners << choose_owner
-          #   chosen = false
-          #   while !chosen
-          #     owners_id = owners.each_with_index.map { |x, i| "#{i+1}. #{x}" }
-          #     choose_owner = ask("Choose default owner:\n"+owners_id.join("\n")+"\n")
-          #
-          #     if choose_owner =~ /[[:digit:]]/
-          #       ow_index = choose_owner.to_i-1
-          #       if ow_index<0 or ow_index >= owners.size
-          #         say "No such owner, please choose again", Thor::Shell::Color::BLUE
-          #         chosen = false
-          #         next
-          #       end
-          #       choose_owner = owners[choose_owner.to_i-1]
-          #       chosen = true
-          #
-          #     else
-          #
-          #       owners_lower = owners.map { |o| o.downcase }
-          #       ow_index = owners_lower.index(choose_owner.downcase)
-          #       if ow_index.nil?
-          #         say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
-          #       else
-          #         chosen = true
-          #       end
-          #     end
-          #
-          #   end
-
-
-          # end
 
           if set_owner(choose_owner, result["username"], urls[ow_index])
             log_message("Setting default owner: #{choose_owner}", Thor::Shell::Color::GREEN)
@@ -831,13 +814,10 @@ module Cnvrg
           log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
           @files = Cnvrg::Datafiles.new(owner, slug)
           log_message("Downloading files", Thor::Shell::Color::BLUE)
-          # if not(query.nil? or query.empty?)
-          #   @files.download_list_files_in_query(response["result"]["list_files"],dataset_home)
-
-          #   log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
-          #   return
-
-          # end
+          if only_tree
+            success = Dataset.clone_tree(commit: commit, dataset_home: dataset_home)
+            return if success
+          end
           commit = response["result"]["commit"]
           files_count = response["result"]["file_count"]
           files = @files.get_clone_chunk(commit: commit)
@@ -1688,7 +1668,7 @@ module Cnvrg
     end
 
 
-    desc 'link git project', 'link git project'
+    desc 'link_git CNVRG_PROJECT_URL', 'Link git project to existed cnvrg project'
     method_option :git, :type => :boolean, :aliases => ["-g", "--git"], :default => false
     def link_git(project_url)
       begin
@@ -3069,9 +3049,6 @@ module Cnvrg
                   exp_success = false
                   log_message("command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED)
                   log_error(e)
-                # rescue Open4::ChildExited
-                #   exp_success = false
-                #   log_message("The process exited!", Thor::Shell::Color::RED)
                 rescue => e
                   res = @exp.end(log, 1, start_commit, 0, 0)
                   log_message("Error occurred,aborting", Thor::Shell::Color::RED)
@@ -3080,6 +3057,7 @@ module Cnvrg
                 end
                 ::Process.wait pid
               end
+              end_time = Time.now
               process_running = false
 
               if !log.empty?
@@ -3116,7 +3094,7 @@ module Cnvrg
                 # log_thread.join
                 stats_thread.join
 
-                res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
+                res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average, end_time: end_time)
 
 
                 if !exp_success
@@ -3410,12 +3388,7 @@ module Cnvrg
 
 
         image = is_project_with_docker(working_dir)
-        if !image or !image.is_docker
-          image = Images.new(working_dir, "cnvrg")
-          image_slug = image.image_slug
-        else
-          image_slug = image.image_slug
-        end
+        image_slug = 'cnvrg'
 
 
         invoke :sync, [false], []
@@ -5961,7 +5934,7 @@ module Cnvrg
 
 
       def parse_io(block_io_str)
-        block_io = block_io_str.gsub!(" ", "").split('/')
+        block_io = block_io_str.gsub(" ", "").split('/')
         input = block_io[0]
         output = block_io[1]
         r = Regexp.new('(\d+(\.\d+)?)([A-Za-z]+)')

data/lib/cnvrg/data.rb

@@ -114,7 +114,7 @@ module Cnvrg
       end
     end
 
-    desc 'date put dataset_url', 'clone datset'
+    desc 'data put dataset_url', 'clone datset'
     method_option :dir, :type => :string, :aliases => ["-d", "--dir"], :default => ''
     def put(dataset_url, *files)
       cli = Cnvrg::CLI.new()
@@ -151,6 +151,24 @@ module Cnvrg
 
     end
 
+    desc 'data files', 'list dataset files'
+    method_option :offset, :type => :numeric, :aliases => ["-o", "--offset"], :default => 0
+    method_option :limit, :type => :numeric, :aliases => ["-l", "--limit"], :default => 1000
+    method_option :expires, :type => :numeric, :aliases => ["-ex", "--expires"], :default => 3600
+    method_option :commit_sha1, :type => :string, :aliases => ["-c", "--commit"], :default => nil
+    def files(dataset_url)
+      cli = Cnvrg::CLI.new()
+      cli.verify_logged_in(false)
+      cli.log_start(__method__, args, options)
+      @dataset = Dataset.new(dataset_url: dataset_url)
+      files = @dataset.list_files(
+          commit_sha1: options[:commit_sha1],
+          limit: options[:limit],
+          expires: options[:expires],
+          offset: options[:offset])
+      cli.log_message(files)
+    end
+
     desc 'data queries', 'list of all queries'
     def queries()
       cli = Cnvrg::CLI.new()

data/lib/cnvrg/datafiles.rb

@@ -875,6 +875,12 @@ module Cnvrg
       end
     end
 
+    def get_trees(commit: "latest")
+      response = Cnvrg::API.request("#{@base_resource}/clone_trees", 'POST',{commit: commit})
+      return nil unless Cnvrg::CLI.is_response_success(response, false)
+      response['result']['files']
+    end
+
     def get_clone_chunk(latest_id: nil, chunk_size: 1000, commit: 'latest')
       response = Cnvrg::API.request("#{@base_resource}/clone_chunk", 'POST',{commit: commit, chunk_size: chunk_size, latest_id: latest_id})
       return nil unless Cnvrg::CLI.is_response_success(response, false)

data/lib/cnvrg/dataset.rb

@@ -7,14 +7,21 @@ module Cnvrg
     IDXParallelThreads ||= 15
     IDXParallelProcesses ||= Parallel.processor_count
 
-    def initialize(project_home)
+    def initialize(project_home = '', dataset_url: '')
       begin
-        @local_path = project_home
-        @working_dir = project_home
-        config = YAML.load_file(project_home+"/.cnvrg/config.yml")
-        @title = config[:dataset_name]
-        @slug = config[:dataset_slug]
-        @owner = config[:owner]
+        if project_home.present?
+          @local_path = project_home
+          @working_dir = project_home
+          config = YAML.load_file(project_home+"/.cnvrg/config.yml")
+          @title = config[:dataset_name]
+          @slug = config[:dataset_slug]
+          @owner = config[:owner]
+        else
+          owner, slug = Cnvrg::Helpers.extract_owner_slug_from_url(dataset_url, 'datasets')
+          @title = slug
+          @slug = slug
+          @owner = owner
+        end
       rescue => e
 
       end
@@ -306,6 +313,32 @@ module Cnvrg
       return true
     end
 
+    def list_files(commit_sha1: "latest", limit: 1000, offset: 0, expires: 3600)
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/list", 'GET', {commit_sha1: commit_sha1, limit: limit, offset: offset, expires: expires})
+      return nil if response.blank?
+      response.to_json
+    end
+
+    def self.clone_tree(commit: 'latest', dataset_home: nil)
+      @dataset = Cnvrg::Dataset.new(dataset_home)
+      @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
+      trees = @files.get_trees(commit: commit)
+      return false if trees.nil?
+      pb = ProgressBar.create(:title => "Download Progress",
+                         :progress_mark => '=',
+                         :format => "%b>>%i| %p%% %t",
+                         :starting_at => 0,
+                         :total => trees.size,
+                         :autofinish => true)
+      trees.each do |tree|
+        pb.progress += 1
+        @files.download_dir(dataset_home, tree)
+      end
+      pb.finish
+      @dataset.write_success
+      true
+    end
+
     def write_success(in_folder = false)
       file_path = ".cnvrg/config.yml"
       file_path = "#{@slug}/" + file_path if !in_folder
@@ -623,11 +656,7 @@ module Cnvrg
       idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
       return idx_hash[:commit]
     end
-    def list_files(commit, json)
-      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/list", 'GET', {commit_sha1:commit, as_json:json})
-      CLI.is_response_success(response,true)
-      return response
-    end
+
     def compare_commit(commit)
         if commit.nil? or commit.empty?
           commit = last_local_commit

data/lib/cnvrg/experiment.rb

@@ -127,12 +127,21 @@ module Cnvrg
       Cnvrg::CLI.is_response_success(response,false)
     end
 
-    def end(output, exit_status, end_commit, cpu_average, memory_average)
+    def end(output, exit_status, end_commit, cpu_average, memory_average, end_time: nil)
       #if remote try to remove
-      response = Cnvrg::API.request(@base_resource + "experiment/end", 'POST', {output: output, exp_slug: @slug,
+      tries = 0
+      success = false
+      end_time ||= Time.now
+      while tries < 10 and success.blank?
+        sleep (tries*rand) ** 2 ### exponential backoff
+                                ## this call is super important so we cant let it crash.
+
+        tries += 1
+        response = Cnvrg::API.request(@base_resource + "experiment/end", 'POST', {output: output, exp_slug: @slug,
                                                                                 exit_status: exit_status, end_commit: end_commit,
-                                                                                cpu_average: cpu_average, memory_average: memory_average})
-      Cnvrg::CLI.is_response_success(response,false)
+                                                                                cpu_average: cpu_average, memory_average: memory_average, end_time: end_time})
+        success = Cnvrg::CLI.is_response_success(response,false)
+      end
 
       begin
         FileUtils.rm_rf(["/home/ds/.cnvrg/tmp/exec.log"])

data/lib/cnvrg/files.rb

@@ -584,6 +584,7 @@ module Cnvrg
       #TODO: parallel
       files_list.map do |file|
         next [file, nil] unless File.exists? file
+        next [file, nil] if File.directory? file
         sha1 = OpenSSL::Digest::SHA1.file(file).hexdigest
         [file.gsub("#{@project_home}/", ""), sha1]
       end.to_h

data/lib/cnvrg/helpers.rb

@@ -238,6 +238,14 @@ parameters:
       return -1
     end
 
+    def extract_owner_slug_from_url(url, breaker)
+      url_parts = url.split("/")
+      project_index = Cnvrg::Helpers.look_for_in_path(url, breaker)
+      slug = url_parts[project_index + 1]
+      owner = url_parts[project_index - 1]
+      return owner, slug
+    end
+
     # cpu
 
     def cpu_time

data/lib/cnvrg/logger.rb

@@ -92,6 +92,11 @@ module Cnvrg
       $log.info message: msg, type: "info"
     end
 
+    def log_json(json, msg: '')
+      Cnvrg::Logger.log_handler if $log.blank?
+      $log.info message: msg, type: "info", data: json
+    end
+
     def jsonify_message(msg: '', success: true)
       puts JSON[{
           "msg": msg,

data/lib/cnvrg/project.rb

@@ -504,6 +504,7 @@ module Cnvrg
         response['result']['tree']['added'] -= downloaded_files
         response['result']['tree']['updated_on_server'] -= downloaded_files
       end
+      Cnvrg::Logger.log_json(response, msg: "Comparing IDX response(tree)")
       return response
     end
 

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '1.2.5'
+  VERSION = '1.2.6'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.2.5
+  version: 1.2.6
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-05 00:00:00.000000000 Z
+date: 2019-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -412,7 +412,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 2.7.9
 signing_key: 
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.