cnvrg 1.5.9.1 → 1.5.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a983ee8a4491253e98fa4704ba878788639792c5975b8cd4efbaf2b7292f978a
-  data.tar.gz: 23aaaa8dcd6b9d476c4585a9240106a7460035e9d735297b60160d9784247870
+  metadata.gz: 15c9e9bdcf068b1bcba64ac65c5589ea53d39478cb50e251f4d956eab7e5a40a
+  data.tar.gz: d4b7bd60ace357d13f502771c260c039de02bf4c6b7894510e2db52b6026dc38
 SHA512:
-  metadata.gz: 169f0fe0cfc5d26ca600c4d07a0efb6df74490b1ac89a2c0ff3a9dd385e0639cf82d9ece70705383a66a9af0cd8b245d8a6981b493aa126a4f299a9778ff3762
-  data.tar.gz: 8eaecd67497635a7e6ee1aff41f22a2b2f66faf7a211feed8d90d71c9e9d822a9dc7466042082a31f770dd3b6b743c237d41f7b6c650fc3fd0388d068e26a882
+  metadata.gz: fb8813704368baf2e80422248d948b29bc4189dd703c77f670b3056b446cffdf8867edcd552003eb40cc21fac0b80257faded1e18faeca912733f50bf1aa711b
+  data.tar.gz: f51367399df0f4a8faa3a1d6e7f160b77050c6f4a9eba88ea5963baeaae7eb7f07dde7e06c297c8003c0b0114dcfe6fcce4a3ff8011e5db319188f2789d7a13e

data/lib/cnvrg/cli.rb

@@ -29,6 +29,7 @@ require 'cnvrg/project'
 require 'cnvrg/files'
 require 'cnvrg/experiment'
 require 'cnvrg/Images'
+require 'cnvrg/image'
 require 'cnvrg/dataset'
 require 'cnvrg/datafiles'
 require 'cnvrg/data'
@@ -40,6 +41,7 @@ require 'cnvrg/org_helpers'
 require 'cnvrg/cli/subcommand'
 require 'cnvrg/cli/flow'
 require 'cnvrg/cli/task'
+require 'cnvrg/image_cli'
 require 'cnvrg/helpers/executer'
 require 'cnvrg/downloader/client'
 require 'cnvrg/downloader/clients/s3_client'
@@ -160,7 +162,7 @@ module Cnvrg
 
     class << self
       # Hackery.Take the run method away from Thor so that we can redefine it.
-  
+
       def is_thor_reserved_word?(word, type)
         return false if word == "run"
         super
@@ -172,7 +174,11 @@ module Cnvrg
     desc "job", "manage running jobs", :hide => false
     subcommand "job", JobCli
 
-     desc "flow", "mange project flows", :hide => true
+    desc "image [COMMAND]", "build existing images", :hide => true
+    subcommand "image", ImageCli
+
+
+    desc "flow", "mange project flows", :hide => true
     subcommand "flow", Cnvrg::Commands::Flow
 
 
@@ -855,7 +861,7 @@ module Cnvrg
     method_option :read, :type => :boolean, :aliases => ["-r", "--read"], :default => false
     method_option :remote, :type => :boolean, :aliases => ["-h", "--remote"], :default => false
 
-    def clone_data(dataset_url,only_tree=false,commit=nil,query=nil,read=false,remote=false)
+    def clone_data(dataset_url,only_tree=false,commit=nil,query=nil,read=false,remote=false, relative: false)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
@@ -885,8 +891,9 @@ module Cnvrg
           @files = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
           log_message("Downloading files", Thor::Shell::Color::BLUE)
           if @dataset.softlinked?
-            @files.cp_ds
-            Cnvrg::CLI.log_message("Clone finished successfully", Thor::Shell::Color::GREEN)
+            @files.cp_ds(relative)
+            @executer.set_dataset_status(dataset: @dataset.slug, status: "cloned") if @executer
+            log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
             @dataset.write_success
             return
           end
@@ -907,7 +914,7 @@ module Cnvrg
                                            :total => files_count,
                                            :autofinish => true)
 
-          
+
           while files['keys'].length > 0
             Cnvrg::Logger.log_info("download multiple files, #{downloaded_files.size} files downloaded")
             @files.download_multiple_files_s3(files, @dataset.local_path, progressbar: progressbar, read_only: read)
@@ -1745,6 +1752,10 @@ module Cnvrg
 
       clone_resp = Project.clone_dir_remote(slug, owner, slug,true)
       idx_status = Project.new(get_project_home).generate_idx
+      @executer = Cnvrg::Helpers::Executer.get_executer
+      if @executer.present?
+        @executer.update_git_commit
+      end
     end
 
 
@@ -2342,17 +2353,18 @@ module Cnvrg
           end
         end
       rescue => e
+        error_message = "Error occured, #{e.message}\nAborting"
         if e.is_a? SignalException
           say "\nAborting", Thor::Shell::Color::BLUE
           say "\nRolling back all changes", Thor::Shell::Color::BLUE
         else
-          log_message("Error occurred, \nAborting", Thor::Shell::Color::RED)
+          log_message(error_message, Thor::Shell::Color::RED)
           log_error(e)
         end
         @files.rollback_commit(commit_sha1) unless commit_sha1.nil?
         print_res = {
             'success' => "false",
-            'message' => 'couldn\'t commit changes,  Rolling Back all changes.'
+            'message' => error_message
         }
         puts "\n"
         puts JSON[print_res] if return_id
@@ -2465,7 +2477,7 @@ module Cnvrg
                                            :total => update_total,
                                            :autofinish => true)
         conflicts = @files.mark_conflicts(result)
-        
+
         log_message("Found some conflicts, check .conflict files.", Thor::Shell::Color::BLUE) if conflicts > 0
         update_res = @files.download_files_in_chunks(result["updated_on_server"], progress: progressbar) if result["updated_on_server"].present?
         added_res = @files.download_files_in_chunks(result["added"], progress: progressbar) if result["added"].present?
@@ -2492,7 +2504,7 @@ module Cnvrg
           else
             log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN, ((sync or options["sync"]) ? false : true))
           end
-          return true          
+          return true
         end
       rescue SignalException => e
         Cnvrg::Logger.log_error(e)

data/lib/cnvrg/data.rb

@@ -90,15 +90,17 @@ module Cnvrg
     method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => nil
     method_option :read, :type => :boolean, :aliases => ["-r", "--read"], :default => false
     method_option :remote, :type => :boolean, :aliases => ["-h", "--remote"], :default => false
+    method_option :relative, :type => :boolean, :aliases => ["-rel", "--relative"], :default => false
 
     def clone(dataset_url)
+      #test
       cli = Cnvrg::CLI.new()
       only_tree =options[:only_tree]
       commit =options[:commit]
       query =options[:query]
       read = options[:read]
       remote = options[:remote]
-      cli.clone_data(dataset_url, only_tree=only_tree,commit=commit, query=query, read=read, remote=remote)
+      cli.clone_data(dataset_url, only_tree=only_tree,commit=commit, query=query, read=read, remote=remote, relative: options[:relative])
     end
 
     desc 'data verify DATASETS_TITLES', 'verify datasets', :hide => true

data/lib/cnvrg/datafiles.rb

@@ -70,9 +70,9 @@ module Cnvrg
       end
     end
 
-    def cp_ds
+    def cp_ds(relative: false)
       prefix = @dataset.get_dataset["bucket_prefix"]
-      current_batch = @downloader.fetch_files(prefix: prefix, limit: 500)
+      batch_size = 10000
       pbar = ProgressBar.create(:title => "Download Progress",
                                         :progress_mark => '=',
                                         :format => "%b%i| %c Files downloaded",
@@ -83,19 +83,21 @@ module Cnvrg
           in_threads: ParallelThreads,
           in_processes: Cnvrg::CLI::ParallelProcesses,
           isolation: true,
-          finish: ->(*args) {pbar.progress += 1}
+          finish: ->(*args) { pbar.progress += 1 }
       }
-      while current_batch.size > 0
+      finished = false
+      while not finished
+        current_batch, marker = @downloader.fetch_files(prefix: prefix, marker: marker, limit: batch_size)
+        if marker.blank?
+          finished = true
+        end
         Parallel.map(current_batch, parallel_options) do |file|
-          #current_batch.map do |file|
           next if file.end_with? "/"
-          cutted_key = Cnvrg::Downloader::Clients::S3Client.cut_prefix(prefix, file)
+          cutted_key = relative ? @downloader.cut_prefix(prefix, file) : file
           dest_path = File.join(@dataset.local_path, cutted_key)
           @downloader.download(file, dest_path, decrypt: false)
           file
         end
-        marker = current_batch.last
-        current_batch = @downloader.fetch_files(prefix: prefix, marker: marker, limit: 500)
       end
     end
 

data/lib/cnvrg/dataset.rb

@@ -9,6 +9,7 @@ module Cnvrg
 
     def initialize(project_home = '', dataset_url: '')
       begin
+        @info = {}
         if project_home.present?
           @local_path = project_home
           @working_dir = project_home
@@ -28,6 +29,10 @@ module Cnvrg
       end
     end
 
+    def soft_linked?
+      @dataset_call["dataset_type"] == "soft_link_dataset"
+    end
+
     def init_home(remote: false)
       dataset_home = File.join(Dir.pwd, @slug)
       if Dir.exists? dataset_home

data/lib/cnvrg/downloader/client.rb

@@ -17,6 +17,10 @@ module Cnvrg
         sts.split("\n")
       end
 
+      def cut_prefix(prefix, file)
+        file.gsub(prefix, '').gsub(/^\/*/, '')
+      end
+
       def download(storage_path, local_path)
         ### need to be implemented..
       end

data/lib/cnvrg/downloader/clients/azure_client.rb

@@ -23,6 +23,13 @@ module Cnvrg
           client.create_block_blob(@container, storage_path, File.open(local_path, "rb"))
         end
 
+        def fetch_files(prefix: nil, marker: nil, limit: 10000)
+          blobs = client.list_blobs(@container, prefix: prefix, max_results: limit, marker: marker)
+          next_marker = blobs.continuation_token
+          files = blobs.map{|x| x.name}
+          [files, next_marker]
+        end
+
 
         private
         def client

data/lib/cnvrg/downloader/clients/s3_client.rb

@@ -61,11 +61,6 @@ module Cnvrg
           batch_files.to_a.map(&:key)
         end
 
-        def self.cut_prefix(prefix, file)
-          file.gsub(prefix, '').gsub(/^\/*/, '')
-        end
-
-
         private
         def aws_client
           Aws::S3::Client.new(@options)

data/lib/cnvrg/files.rb

@@ -88,16 +88,22 @@ module Cnvrg
       # resolve bucket
       res = resp['result']
       files = res['files']
+
       #upload files
-      # files.keys.map do |file|
-      Parallel.map(files.keys, self.get_upload_options) do |file|
-        resp = Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]))}
-        raise SignalException.new("Cant upload #{file}") unless resp
+      blob_ids = Parallel.map(files.keys, self.get_upload_options) do |file|
+        begin
+          Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]))}
+        rescue => e
+          Cnvrg::CLI.log_message("Failed to upload #{file}: #{e.message}", 'red')
+          Cnvrg::Logger.log_error(e)
+          Cnvrg::Logger.log_method(bind: binding)
+          raise e
+        end
         progress.progress += 1 if progress.present?
+        files[file]["bv_id"]
       end
 
       #save files on the server.
-      blob_ids = files.values.map {|f| f['bv_id']}
       resp = Cnvrg::API.request(@base_resource + "upload_files_save", 'POST', {blob_ids: blob_ids, commit: commit_sha1})
       unless Cnvrg::CLI.is_response_success(resp, false)
         raise SignalException.new("Cant save uploaded files to the server.")

data/lib/cnvrg/helpers.rb

@@ -33,22 +33,20 @@ module Cnvrg
       end
     end
 
-    def try_until_success(tries: 5)
-      (0..tries).each do |i|
+    def try_until_success(tries: 3)
+      exception = nil
+      tries.times do |i|
         begin
           yield
           return true
-        rescue Exception => e
-          Cnvrg::Logger.log_info("Error while trying for the #{i} time")
-          Cnvrg::Logger.log_error(e)
-          sleep(5)
         rescue => e
           Cnvrg::Logger.log_info("Error while trying for the #{i} time")
           Cnvrg::Logger.log_error(e)
-          sleep(5)
+          sleep(1)
+          exception = e
         end
       end
-      raise StandardError.new("Failed")
+      raise exception
     end
 
     def get_config

data/lib/cnvrg/helpers/executer.rb

@@ -1,5 +1,6 @@
 class Cnvrg::Helpers::Executer
-  def initialize(project: nil, job_type: nil, job_id: nil)
+  def initialize(project: nil, job_type: nil, job_id: nil, image: nil)
+    @image = image
     @project = project || Cnvrg::Project.new(owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
     @job_type = job_type || ENV['CNVRG_JOB_TYPE']
     @job_id = job_id || ENV['CNVRG_JOB_ID']
@@ -69,7 +70,7 @@ class Cnvrg::Helpers::Executer
       if command[:type] == "file_exists"
         puts "Looking for file #{command[:file]}"
       else
-        puts "Execute #{command[:command]}"
+        puts "Execute #{command[:command]}" unless command[:no_stdout]
       end
       execute(command)
     end
@@ -81,6 +82,13 @@ class Cnvrg::Helpers::Executer
     commands.map{|k| k.with_indifferent_access}
   end
 
+
+  def update_git_commit
+    git_commit = `git rev-parse --verify HEAD`
+    return if git_commit.blank?
+    Cnvrg::API.request("#{base_url}/update_git_commit", "POST", {git_commit: git_commit.strip!})
+  end
+
   def set_dataset_status(dataset: nil, status: nil)
     Cnvrg::API.request("#{base_url}/datasets/#{dataset}", "PUT", {status: status})
   end
@@ -131,12 +139,15 @@ class Cnvrg::Helpers::Executer
     output = []
     start_time = Time.now
     timeout = cmd[:timeout] || 5*60
+    exit_status = nil
     t = Thread.new do
       PTY.spawn(cmd[:command]) do |stdout, stdin, pid, stderr|
         begin
-          stdout.each do |line|
-            puts line
-            output << {log: line.strip, timestamp: Time.now}
+          if stdout.present?
+            stdout.each do |line|
+              puts line
+              output << {log: line.strip, timestamp: Time.now}
+            end
           end
           if stderr.present?
             stderr.each do |line|
@@ -148,16 +159,17 @@ class Cnvrg::Helpers::Executer
         end
         ::Process.wait pid
       end
+      exit_status = $?.exitstatus
     end
     while t.status
       if Time.now - start_time > timeout
         puts "Kill thread because of timeout..."
         errors << {log: "Timeout", timestamp: Time.now}
         Thread.kill(t)
+        exit_status = -100 ##killed
       end
       sleep 1
     end
-    exit_status = $?.exitstatus
     end_time = Time.now
     [exit_status, merge_log_block(output), merge_log_block(errors), start_time, end_time]
   end
@@ -185,7 +197,11 @@ class Cnvrg::Helpers::Executer
 
 
   def job_log(logs, level: 'info', step: nil)
-    @project.job_log(logs, level: level, step: step, job_type: @job_type, job_id: @job_id)
+    if @job_type == "image"
+      @image.job_log(logs, level: level, step: step)
+    else
+      @project.job_log(logs, level: level, step: step, job_type: @job_type, job_id: @job_id)
+    end
   end
 
 end

data/lib/cnvrg/image.rb

@@ -0,0 +1,100 @@
+module Cnvrg
+  class Image
+    # attr_reader :image_name, :image_tag, :is_docker, :project_slug, :commit_id, :owner, :port, :image_slug
+
+
+    def initialize(image_id)
+      begin
+        @cli = Cnvrg::CLI.new
+        home_dir = File.expand_path('~')
+        config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
+        @owner = config.to_h[:owner]
+        @username =  config.to_h[:username]
+        @image_id = image_id
+      rescue => e
+        @owner = ""
+        @username =  ""
+        @cli.log_message("cnvrg is not configured")
+      end
+    end
+
+    def build
+      image_data = get_image_data
+      file_name = "Dockerfile-#{@image_id}"
+      File.new(file_name, "w+")
+      if image_data["reqs_file_path"].present? and image_data["from_image_name"]
+        File.open(file_name, "w+") do |i|
+          i.write("FROM #{image_data["from_image_name"]}")
+          i.write("ADD requirements.txt requirements.txt")
+          i.write("RUN pip3 install -r requirements.txt")
+        end
+      else
+        open(file_name, 'wb') do |file|
+          file << open(image_data["docker_file_path"]).read
+        end
+      end
+      command = {:type=>"notify",
+                   :title=>"docker build",
+                   :logs=>true,
+                   :command=>"sudo docker build . -t #{image_data["docker_name"]} -f #{file_name}"}
+
+      @executer = Helpers::Executer.new(project: @project, job_type: "image", job_id: @image_id, image: self)
+      exit_status, output, errors, _, _ = @executer.execute(command)
+      all_logs = join_logs(output, errors)
+      if exit_status > 0
+        raise StandardError.new(all_logs)
+      end
+      if ENV["CNVRG_IMAGE_BUILD_USERNAME"].present? and ENV["CNVRG_IMAGE_BUILD_PASSWORD"].present?
+        command = {:type=>"notify",
+                    :no_stdout => true,
+                    :title=>"docker login",
+                    :logs=>true,
+                    :command=>"sudo docker login --username=#{ENV["CNVRG_IMAGE_BUILD_USERNAME"]} --password=\"#{ENV["CNVRG_IMAGE_BUILD_PASSWORD"]}\""}
+        exit_status, output, errors, _, _ = @executer.execute(command)
+        all_logs = join_logs(output, errors)
+        if exit_status > 0
+          raise StandardError.new(all_logs)
+        end
+      end
+      command = {:type=>"notify",
+                  :title=>"docker push",
+                  :logs=>true,
+                  :command=>"sudo docker push #{image_data["docker_name"]}"}
+      exit_status, output, errors, _, _ = @executer.execute(command)
+      all_logs = join_logs(output, errors)
+      if exit_status > 0
+        raise StandardError.new(all_logs)
+      end
+      post_build_update(true)
+    rescue => e
+      post_build_update(false, e.message)
+    end
+
+    def get_image_data
+      response = Cnvrg::API.request("users/#{@owner}/images/#{@image_id}/image_start_build", 'GET')
+      CLI.is_response_success(response)
+      return response["image"]
+    end
+
+    def post_build_update(success, message = "")
+      response = Cnvrg::API.request("users/#{@owner}/images/#{@image_id}/image_end_build", 'POST', {success: success, message: message})
+      CLI.is_response_success(response)
+      return response["image"]
+    end
+
+
+    def job_log(logs, level: 'info', step: nil, job_type: "image", job_id: @image_id)
+      logs = [logs].flatten
+      logs.each_slice(10).each do |temp_logs|
+        Cnvrg::API.request("users/#{@owner}/images/#{@image_id}/log", "POST", {job_type: job_type, job_id: job_id, logs: temp_logs, log_level: level, step: step, timestamp: Time.now})
+        sleep(1)
+      end
+    end
+
+
+    def join_logs(output, errors)
+      output.map{ |o| o[:logs]}.join(" ") + " " + errors.map{ |o| o[:logs]}.join(" ")
+    end
+
+  end
+end

data/lib/cnvrg/image_cli.rb

@@ -0,0 +1,25 @@
+
+module Cnvrg
+  class ImageCli < SubCommandBase
+
+    desc 'image build --image=IMAGE_ID', 'Build image job', :hide => true
+    method_option :image_id, :type => :string, :aliases => ["--image"]
+    def build_image_job()
+      begin
+        @cli = Cnvrg::CLI.new
+        @cli.verify_logged_in(false)
+        @cli.log_start(__method__, args, options)
+        @cli.log_message("build image started", Thor::Shell::Color::BLUE)
+        image = Image.new(options["image_id"])
+        image.build
+        @cli.log_message("Image build completed successfully", Thor::Shell::Color::BLUE)
+      rescue => e
+        @cli.log_message("Image build completed with an error", Thor::Shell::Color::RED)
+        @cli.log_error(e)
+        exit(1)
+      end
+    end
+
+
+  end
+end

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '1.5.9.1'
+  VERSION = '1.5.9.2'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.5.9.1
+  version: 1.5.9.2
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-21 00:00:00.000000000 Z
+date: 2019-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -418,6 +418,8 @@ files:
 - lib/cnvrg/helpers.rb
 - lib/cnvrg/helpers/executer.rb
 - lib/cnvrg/hyper.rb
+- lib/cnvrg/image.rb
+- lib/cnvrg/image_cli.rb
 - lib/cnvrg/job_cli.rb
 - lib/cnvrg/logger.rb
 - lib/cnvrg/org_helpers.rb