cnvrg 0.9.3 → 0.9.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d2e26104d7aabe95db17799925bdfdfe35659256
-  data.tar.gz: 9dab8f4d8bac50ba1683e1a52435cea57bccca30
+  metadata.gz: 04d14b6b659ad1bdc05347a594f1a4447334af4f
+  data.tar.gz: 5d12cbd7b69bc517cfe06f8f920fce3edcf8fc37
 SHA512:
-  metadata.gz: a62e0a0a4656ef1edf1d829e6e9fd97062dcf6e69c9a67394cc883b14b59c9dc8a645affa6d4baa90777b3379deb8d849084e0724a88f7e3a6207e63f08dc277
-  data.tar.gz: be977bf88bb480097737919977e75adda5b2eb88454d3e8fefc5e4482622b1783dae265013e6dd7f12056b8e35ac9107d32f0b2adc90b4a802247562591b1b66
+  metadata.gz: 15526296c7f0a868506e787c4fb8f81753ce9159a9257dbd5e1c423ce80448a8b5eeb5e30df0acc88f4643c3ca2240fb8cbb2f53049244fdc1be0f76b7248144
+  data.tar.gz: d5d8953e5d2667e30770e3b48a89b2b9d7482d0acc920a9139b710e178dfc6d5b3484ff47243c222ea3c779e60716e721c8deb3919f8356a2deff0c00c114421

data/lib/cnvrg/Images.rb

@@ -67,14 +67,18 @@ module Cnvrg
       client = props[:client]
       upload_options = props[:upload_options]
       bucket = Aws::S3::Resource.new(client: client).bucket(props[:bucket])
+
       resp = bucket.object(path).
           upload_file(image_path, upload_options)
+
       unless resp
         raise Exception.new("Cant upload #{image_path}")
         return false
       end
       return save_docker_image(image_id)
       rescue => e
+        Cnvrg::Logger.log_error(e)
+
         return false
       end
 
@@ -84,7 +88,8 @@ module Cnvrg
 
     def save_docker_image(image_id)
       image_res = Cnvrg::API.request("users/#{@owner}/images/#{image_id}/save" , 'POST', {})
-      return Cnvrg::CLI.is_response_success(image_res, true)
+       Cnvrg::CLI.is_response_success(image_res, true)
+      return image_res
     end
 
     def is_container_exist()

data/lib/cnvrg/cli.rb

@@ -24,6 +24,7 @@ require 'cnvrg/data'
 require 'cnvrg/ssh'
 require 'cnvrg/result'
 require 'cnvrg/logger'
+require 'cnvrg/org_helpers'
 require 'cnvrg/cli/flow'
 require 'cnvrg/cli/task'
 require 'logstash-logger'
@@ -155,6 +156,7 @@ module Cnvrg
 
     class << self
       # Hackery.Take the run method away from Thor so that we can redefine it.
+  
       def is_thor_reserved_word?(word, type)
         return false if word == "run"
         super
@@ -163,8 +165,8 @@ module Cnvrg
     desc "data", "upload and manage datasets", :hide => false
     subcommand "data", Data
 
-    # desc "flow", "mange project flows", :hide => true
-    # subcommand "flow", Cnvrg::Commands::Flow
+     desc "flow", "mange project flows", :hide => true
+    subcommand "flow", Cnvrg::Commands::Flow
 
 
     def initialize(*args)
@@ -618,39 +620,6 @@ module Cnvrg
 	       path = Dir.pwd + "/" + project_name
           @project = Project.new(path)
           @project.generate_idx
-          if docker
-            local_images = Docker::Image.all
-            docker_image_local = local_images.map {|x| x.info["RepoTags"]}.flatten.select {|y| y.include? docker_image}.flatten
-            if docker_image_local.size == 0
-
-              if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
-                image = pull(docker_image)
-                if image
-                  log_message("downloaded image: #{docker_image}", Thor::Shell::Color::BLUE)
-                  @image = Images.new(working_dir, docker_image)
-                else
-                  log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
-                  @project.revert(working_dir)
-                  exit(1)
-                end
-              else
-                log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
-                @project.revert(working_dir)
-                exit(1)
-
-              end
-            elsif docker_image_local.size == 1
-              log_message("found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE)
-              @image = Images.new(working_dir, docker_image_local[0])
-            elsif docker_image_local.size > 1
-              log_message("found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE)
-              image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
-              image_name = image_name.strip
-              @image = Images.new(working_dir, image_name)
-            end
-            @image.update_image_activity(nil, nil)
-
-          end
         else
           log_message("Error creating project, please contact support.", Thor::Shell::Color::RED)
           @project.revert(working_dir)
@@ -714,35 +683,6 @@ module Cnvrg
           path = Dir.pwd
           @project = Project.new(path)
           @project.generate_idx()
-          if docker
-            local_images = Docker::Image.all
-            docker_image_local = local_images.map {|x| x.info["RepoTags"]}.flatten.select {|y| y.include? docker_image}.flatten
-            if docker_image_local.size == 0
-              if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
-                image = pull(docker_image)
-                if image
-                  log_message("downloaded image: #{docker_image}", Thor::Shell::Color::BLUE)
-                  @image = Images.new(working_dir, docker_image)
-                else
-                  log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
-                  @project.revert(working_dir)
-                  exit(1)
-                end
-              else
-                log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
-                @project.revert(working_dir)
-                exit(1)
-
-              end
-            elsif docker_image_local.size == 1
-              log_message("found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE)
-              @image = Images.new(working_dir, docker_image_local[0])
-            elsif docker_image_local.size > 1
-              log_message("found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE)
-              image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
-              @image = Images.new(working_dir, image_name)
-            end
-          end
           if sync
             log_message("Syncing project", Thor::Shell::Color::BLUE)
             upload(true)
@@ -2102,7 +2042,7 @@ module Cnvrg
       dataset_dir = is_cnvrg_dir(Dir.pwd)
       @dataset = Dataset.new(dataset_dir)
       @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
-      new_commit ||= @dataset.get_next_commit()
+      new_commit ||= @dataset.get_next_commit
       partial_commit = @dataset.get_partial_commit
       if new_commit.blank?
         log_message("You must specify commit, run start_commit to create new commit", Thor::Shell::Color::RED)
@@ -2427,14 +2367,21 @@ module Cnvrg
         end
         Cnvrg::Logger.log_info("Current commit: #{@dataset.get_current_commit}, destination commit: #{commit}")
         Cnvrg::Logger.log_info("Compare idx res: #{result}")
+        progressbar = ProgressBar.create(:title => "Download Progress",
+                                           :progress_mark => '=',
+                                           :format => "%b>>%i| %p%% %t",
+                                           :starting_at => 0,
+                                           :total => update_total,
+                                           :autofinish => true)
         conflicts = @files.mark_conflicts(result)
         
         log_message("Found some conflicts, check .conflict files.", Thor::Shell::Color::BLUE) if conflicts > 0
-        update_res = @files.download_files_in_chunks(result["updated_on_server"]) if result["updated_on_server"].present?
-        added_res = @files.download_files_in_chunks(result["added"]) if result["added"].present?
+        update_res = @files.download_files_in_chunks(result["updated_on_server"], progress: progressbar) if result["updated_on_server"].present?
+        added_res = @files.download_files_in_chunks(result["added"], progress: progressbar) if result["added"].present?
         # conflict_res = @files.download_files_in_chunks(result["conflicts"], conflict: true) if result["conflicts"].present?
         deleted = result["deleted"].to_a
         delete_res = @files.delete_commit_files_local(deleted)
+        progressbar.progress += deleted.size if progressbar.present?
         if !delete_res
           log_message("Couldn't delete #{deleted.join(" ")}", Thor::Shell::Color::RED)
           log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
@@ -2487,8 +2434,15 @@ module Cnvrg
             log_message("No New files to download", Thor::Shell::Color::GREEN)
             return
           end
-          download_result = @files.download_multpile_files_s3(files, project_home)
+          progressbar = ProgressBar.create(:title => "Download Progress",
+                                         :progress_mark => '=',
+                                         :format => "%b>>%i| %p%% %t",
+                                         :starting_at => 0,
+                                         :total => files['keys'].size,
+                                         :autofinish => true)
+          download_result = @files.download_multpile_files_s3(files, project_home, progress: progressbar)
           if download_result
+            progressbar.finish
             log_message("Done.\nDownloaded  finished successfully", Thor::Shell::Color::GREEN)
           else
             log_message("Error while trying to download\n", Thor::Shell::Color::RED)
@@ -2760,7 +2714,7 @@ module Cnvrg
       end
     end
 
-    desc 'sync', 'Sync with remote server'
+    desc 'sync', 'Sync with cnvrg'
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :default => ""
@@ -3564,21 +3518,6 @@ module Cnvrg
         if !choose_image.nil? and !choose_image.empty?
           invoke :set_image, [choose_image]
         end
-        @image = is_project_with_docker(working_dir)
-        if !@image or !@image.is_docker
-          # say "Couldn't find image related to project", Thor::Shell::Color::RED
-          default_image_name = "cnvrg"
-          if instance_type.eql? "gpu" or instance_type.eql? "gpuxl"
-            default_image_name = "cnvrg-gpu"
-          end
-          # default = yes? "use #{default_image_name} default image?", Thor::Shell::Color::YELLOW
-          # if default
-          @image = Images.new(working_dir, default_image_name)
-          # else
-          #   exit(0)
-          # end
-        end
-
         invoke :sync, [false], []
 
 
@@ -4867,8 +4806,11 @@ module Cnvrg
         user = options[:user]
         is_gpu = options[:gpu]
         res = @image.upload_docker_image(image_path, image_name, workdir, user, description, is_gpu)
-        if res
-          log_message("Successfully uploaded image: #{image_name}", Thor::Shell::Color::GREEN, true)
+        if res["status"] == 200
+          image_slug = res["id"]
+          owner = CLI.get_owner
+          image_url = "#{Cnvrg::Helpers.remote_url}/#{owner}/settings/images/#{image_slug}"
+          log_message("Successfully uploaded image: #{image_url}", Thor::Shell::Color::GREEN, true)
 
 
         else

data/lib/cnvrg/cli/flow.rb

@@ -41,6 +41,7 @@ module Cnvrg
           return false
         end
         begin
+          path = "#{path}.flow.yaml" unless path.end_with? '.flow.yaml'
           @flow = Cnvrg::Flows.new(@curr_dir, path)
           url = @flow.run
         rescue => e

data/lib/cnvrg/cli/subcommand.rb

@@ -6,6 +6,8 @@ class SubCommand < Thor
     super
     @cli = Cnvrg::CLI.new
     @curr_dir = @cli.is_cnvrg_dir
+    @config = YAML.load_file(@curr_dir + "/.cnvrg/config.yml") if @curr_dir
+    @helpers = Cnvrg::OrgHelpers.new();
   end
 
   def self.banner(command, namespace = nil, subcommand = false)

data/lib/cnvrg/cli/task.rb

@@ -1,3 +1,4 @@
+
 module Cnvrg
   module Commands
     class Task < SubCommand
@@ -34,7 +35,6 @@ module Cnvrg
 
       desc "flow task create", "Creates new task"
       def create
-
         @project = Cnvrg::Project.new(@curr_dir)
 
         task = {}
@@ -65,6 +65,7 @@ module Cnvrg
       desc "flow task run", "Running specific task"
       def run(path)
         begin
+          path = "#{path}.task.yaml" unless path.end_with? '.task.yaml'
           @task = Cnvrg::Task.new(@curr_dir, path: path)
           url = @task.run
           @cli.log_message("Task: #{@task.title} is running, you can track its performance on #{url}", Thor::Color::GREEN)

data/lib/cnvrg/datafiles.rb

@@ -878,11 +878,11 @@ module Cnvrg
       response['result']['files']
     end
 
-    def download_files_in_chunks(files, chunk_size: 1000, conflict: false, commit: 'latest')
+    def download_files_in_chunks(files, chunk_size: 1000, conflict: false, commit: 'latest', progress: nil)
       begin
         ttl_files = 0
         files.each_slice(chunk_size).each do |files|
-          ttl_files += download_files_chunk(files, conflict: conflict)
+          ttl_files += download_files_chunk(files, conflict: conflict, progress: progress)
         end
         return Cnvrg::Result.new(true, "Download Completed")
       rescue Exception => e
@@ -890,19 +890,19 @@ module Cnvrg
       end
     end
 
-    def download_files_chunk(files, conflict: false)
+    def download_files_chunk(files, conflict: false, progress: nil)
       (1..5).each do |i|
         response = Cnvrg::API.request("users/#{@owner}/datasets/#{@dataset_slug}/download_multi", 'POST', {files: files})
         next unless Cnvrg::CLI.is_response_success(response, false) #trying to api request 5 times.
         files_to_download = response['files']
         data_home = "#{Dir.pwd}/#{response['name']}"
-        res = download_multiple_files_s3(files_to_download, data_home, conflict: conflict, read_only: false)
+        res = download_multiple_files_s3(files_to_download, data_home, conflict: conflict, read_only: false, progressbar: progress)
         next unless res.is_success? #try again..
         return files_to_download['keys'].length
       end
     end
 
-    def download_multiple_chunks(commit, chunk_size=1000)
+    def download_multiple_chunks(commit, chunk_size=1000, progress: nil)
       begin
       last_chunk_size = chunk_size
       q = { commit: commit, chunk_size: chunk_size}
@@ -914,7 +914,7 @@ module Cnvrg
           data_home = "#{Dir.pwd}/#{response['name']}"
           last_chunk_size = files['keys'].length
           break if last_chunk_size == 0
-          res = download_multiple_files_s3(files, data_home, read_only: false)
+          res = download_multiple_files_s3(files, data_home, read_only: false, progressbar: progress)
           overall += last_chunk_size
           q[:latest] = files['latest']
         else

data/lib/cnvrg/dataset.rb

@@ -367,9 +367,12 @@ module Cnvrg
 
     def list_all_files(with_ignore=false)
       list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~/^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~/^#{self.local_path}\/\.cnvrgignore/) }
+
+      #we want that big files will
+      list = list.sort_by{|fn| File.size(fn)}
       return list if with_ignore
       list_ignore = self.get_ignore_list.map{|ignore_file| "#{self.local_path}/#{ignore_file}"}
-      list.select{|file| !list_ignore.include? file}
+      (list - list_ignore)
     end
 
     def write_idx(tree=nil, commit=nil)

data/lib/cnvrg/files.rb

@@ -57,7 +57,7 @@ module Cnvrg
       files_list = files_list.map{|x| [x,self.parse_file(x)]}.to_h
       resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST', {files: files_list, commit: commit_sha1})
       unless Cnvrg::CLI.is_response_success(resp, false)
-        raise Exception.new("Cant upload files to the server.")
+        raise SignalException.new("Cant upload files to the server.")
       end
       # resolve bucket
       res = resp['result']
@@ -72,7 +72,7 @@ module Cnvrg
       # files.keys.map do |file|
       Parallel.map(files.keys, self.get_upload_options) do |file|
         resp = Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]), s3_bucket, options: upload_options)}
-        raise Exception.new("Cant upload #{file}") unless resp
+        raise SignalException.new("Cant upload #{file}") unless resp
         progress.progress += 1 if progress.present?
       end
 
@@ -80,7 +80,7 @@ module Cnvrg
       blob_ids = files.values.map {|f| f['bv_id']}
       resp = Cnvrg::API.request(@base_resource + "upload_files_save", 'POST', {blob_ids: blob_ids, commit: commit_sha1})
       unless Cnvrg::CLI.is_response_success(resp, false)
-        raise Exception.new("Cant save uploaded files to the server.")
+        raise SignalException.new("Cant save uploaded files to the server.")
       end
     end
 
@@ -108,7 +108,7 @@ module Cnvrg
       return if files.blank?
       resp = Cnvrg::API.request(@base_resource + "delete_files", 'DELETE', {files: files, commit: commit_sha1})
       unless Cnvrg::CLI.is_response_success(resp, false)
-        raise Exception.new("Cant delete the following files from the server.")
+        raise SignalException.new("Cant delete the following files from the server.")
       end
     end
 
@@ -119,7 +119,7 @@ module Cnvrg
       resp = bucket.object(path).
           upload_file(absolute_path, options)
       unless resp
-        raise Exception.new("Cant upload #{absolute_path}")
+        raise SignalException.new("Cant upload #{absolute_path}")
       end
       resp
     end
@@ -665,7 +665,7 @@ module Cnvrg
       end
       res = Cnvrg::API.request(@base_resource + "download_files", 'POST', {files: files, commit: commit})
       unless Cnvrg::CLI.is_response_success(res, false)
-        raise Exception.new("Cant download files from the server.")
+        raise SignalException.new("Cant download files from the server.")
       end
       self.download_multpile_files_s3(res['result'], @project_home, postfix: postfix, progress: progress)
     end
@@ -947,7 +947,7 @@ module Cnvrg
         end
       rescue => e
         Cnvrg::Logger.log_error(e)
-        raise Exception.new("Cant upload files")
+        raise SignalException.new("Cant upload files")
       end
     end
       def rollback_commit(commit_sha1)

data/lib/cnvrg/helpers.rb

@@ -334,7 +334,7 @@ parameters:
       if !server_side_encryption
         upload_options = {:use_accelerate_endpoint => use_accelerate_endpoint}
       else
-        upload_options = {:use_accelerate_endpoint => use_accelerate_endpoint, :server_side_encryption => server_side_encryption}
+        upload_options = {:use_accelerate_endpoint => use_accelerate_endpoint, :server_side_encryption => "AES256"}
       end
       return {client: client, key: key, iv: iv, bucket: bucket, upload_options: upload_options}
     end

data/lib/cnvrg/org_helpers.rb

@@ -0,0 +1,5 @@
+module Cnvrg
+    class OrgHelpers
+    
+    end
+end

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '0.9.3'
+  VERSION = '0.9.4'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.9.3
+  version: 0.9.4
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-29 00:00:00.000000000 Z
+date: 2018-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -414,6 +414,7 @@ files:
 - lib/cnvrg/hyper.rb
 - lib/cnvrg/job.rb
 - lib/cnvrg/logger.rb
+- lib/cnvrg/org_helpers.rb
 - lib/cnvrg/project.rb
 - lib/cnvrg/result.rb
 - lib/cnvrg/runner.rb