cnvrg 1.2.7 → 1.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f7319668db1dc7e8257aa9f6067fc43b91dd559f4e73c1c5d84c6e0a6c4ec30b
-  data.tar.gz: 3958b12ea50a32fa38ebc0c41a3e0b019199fe3f6a4f0f3ebcbe43b839fa25c1
+  metadata.gz: c562ccc1d7fd69f3f2b0d28db538d27d57e4a55df5f1fef3eee6cab511bd30ca
+  data.tar.gz: 624e78308a885897c9fc31402aff21198a3f2db6551b676f14d82d9126624255
 SHA512:
-  metadata.gz: cb413f91c9968b93ec5e1b05be0a2deff6f5aaa5e097b9eda6e02311a36e6b14d8dba34a8385c0e1abc5193810fbaa1a0902f49537ffb79e2a7745117448233c
-  data.tar.gz: 5a4d02fc1920d0fda7e0d5b7dfab5793ee31e6adc8b9fc2e60adc5e55213284dcc3bb3cb1c1974b1da0541da9fd8a86eb94d4c3f1c3f0a82543ec43fd4dcd080
+  metadata.gz: 928e1bb402e0dac773bb9d8150c65815b2af8878c4969ca555eeb79ac7ad51a8f9a7f1d8c69cd58651788100c0cf6b2cd9a2e2b427d7efd667d0d4abf6f41629
+  data.tar.gz: 123068cd21d3900cdf3e9ae982feb2e9e52ee9303236877f41a6653cb8e4e6294f7afa8cb8c7c3a1e665fa0b331d8ee1e337b3e6a1b9e34a4f8293f25381d027

data/cnvrg.gemspec

@@ -6,7 +6,7 @@ require 'cnvrg/version'
 Gem::Specification.new do |spec|
     spec.name          = 'cnvrg'
     spec.version       = Cnvrg::VERSION
-    spec.authors       = ['Yochay Ettun', 'Leah Kolben']
+    spec.authors       = ['Yochay Ettun', 'Leah Kolben', 'Omer Shacham']
     spec.email         = ['info@cnvrg.io']
     spec.summary       = %q{A CLI tool for interacting with cnvrg.io.}
     spec.description   = %q{A CLI tool for interacting with cnvrg.io.}
@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
     spec.executables   = ['cnvrg']
     spec.require_paths = ['lib']
 
-    spec.add_development_dependency 'bundler', '~> 1.11'
+    spec.add_development_dependency 'bundler'
     spec.add_development_dependency 'rake', '~> 10.0'
     spec.add_development_dependency 'rspec', '~> 3.0'
     spec.add_development_dependency 'vcr', '~> 3.0'
@@ -32,6 +32,7 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'highline', '~> 1.7', '>= 1.7.8'
     spec.add_runtime_dependency 'thor', '~> 0.19.0','>=0.19.1'
     spec.add_runtime_dependency 'aws-sdk', '~> 2'
+    spec.add_runtime_dependency 'google-cloud-storage', '~> 1.16'
     spec.add_runtime_dependency 'sucker_punch', '~> 2.0'
     spec.add_runtime_dependency 'urlcrypt', '~> 0.1.1'
     spec.add_runtime_dependency 'parallel', '~> 1.12.0'

data/lib/cnvrg/api.rb

@@ -53,7 +53,7 @@ module Cnvrg
                 end
             conn.headers['Auth-Token'] = @pass
             conn.headers['User-Agent'] = "#{Cnvrg::API::USER_AGENT}"
-                conn.options.timeout = 420
+            conn.options.timeout = 420
                 conn.options.open_timeout=180
             case method
               when 'GET'
@@ -201,7 +201,7 @@ module Cnvrg
             else
             end
             rescue => e
-                puts e
+              Cnvrg::Logger.log_error(e)
                return nil
             rescue SignalException
                 return false

data/lib/cnvrg/cli.rb

@@ -11,6 +11,18 @@ require 'yaml'
 require 'digest' # sha1up
 require "highline/import"
 require 'socket'
+require 'thor'
+require 'docker'
+require 'launchy'
+require 'socket'
+require 'timeout'
+require 'fileutils'
+require 'active_support/all'
+require 'pathname'
+require 'enumerator'
+require 'ruby-progressbar'
+require 'open3'
+require 'logstash-logger'
 require 'cnvrg/helpers'
 require 'cnvrg/api'
 require 'cnvrg/auth'
@@ -21,25 +33,15 @@ require 'cnvrg/Images'
 require 'cnvrg/dataset'
 require 'cnvrg/datafiles'
 require 'cnvrg/data'
+require 'cnvrg/storage'
 require 'cnvrg/ssh'
 require 'cnvrg/result'
 require 'cnvrg/logger'
 require 'cnvrg/org_helpers'
-require 'cnvrg/cli/flow'
-require 'cnvrg/cli/task'
-require 'logstash-logger'
-require 'cnvrg/job'
-require 'docker'
-require 'launchy'
-require 'socket'
-require 'timeout'
-require 'fileutils'
-require 'active_support/all'
-require 'thor'
-require 'pathname'
-require 'enumerator'
-require 'ruby-progressbar'
-require 'open3'
+require 'cnvrg/job_cli'
+Gem.find_files("cnvrg/downloader/**/*.rb").reverse.each{|r| require r}
+Gem.find_files("cnvrg/helpers/**/*.rb").reverse.each{|r| require r}
+Gem.find_files("cnvrg/cli/**/*.rb").each{|r| require r}
 
 
 class Thor
@@ -164,6 +166,9 @@ module Cnvrg
     desc "data", "upload and manage datasets", :hide => false
     subcommand "data", Data
 
+    desc "job", "manage running jobs", :hide => false
+    subcommand "job", JobCli
+
      desc "flow", "mange project flows", :hide => true
     subcommand "flow", Cnvrg::Commands::Flow
 
@@ -605,12 +610,14 @@ module Cnvrg
     desc 'new', 'Create a new cnvrg project'
     method_option :clean, :type => :boolean, :aliases => ["-c"], :default => false
     method_option :docker_image, :type => :string, :aliases => ["-d"], :default => ""
+    method_option :bucket, :type => :string, :aliases => ["-b", "--bucket"], :default => ""
 
     def new(project_name)
       begin
        verify_logged_in(false)
         log_start(__method__, args, options)
         clean = options["clean"]
+        bucket = options["bucket"]
         docker_image = options["docker_image"]
         working_dir = Dir.pwd + "/" + project_name
         docker = false
@@ -662,6 +669,7 @@ module Cnvrg
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
     method_option :docker_image, :type => :string, :aliases => ["-d"], :default => ""
     method_option :git, :type => :boolean, :aliases => ["-g","--git"], :default => false
+    method_option :bucket, :type => :string, :aliases => ["-b","--bucket"], :default => ''
     method_option :title, :type => :string, :aliases => ["-t","--title"], :default => nil
 
     def link
@@ -669,7 +677,8 @@ module Cnvrg
         verify_logged_in(false)
         log_start(__method__, args, options)
         docker_image = options["docker_image"]
-        if !docker_image.nil? and !docker_image.empty?
+        bucket = options["bucket"]
+        if docker_image.present?
           docker = true
         else
           docker = false
@@ -687,7 +696,7 @@ module Cnvrg
         end
         working_dir = Dir.getwd
         owner = CLI.get_owner
-        if Project.link(owner, project_name, docker,git)
+        if Project.link(owner, project_name, docker,git, bucket: bucket)
           path = Dir.pwd
           @project = Project.new(path)
           if sync
@@ -718,7 +727,7 @@ module Cnvrg
     desc 'data init', 'Init dataset directory'
     method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
 
-    def init_data(public)
+    def init_data(public, bucket: nil)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
@@ -733,7 +742,7 @@ module Cnvrg
 
         working_dir = Dir.getwd
         owner = CLI.get_owner
-        if Dataset.init(owner, dataset_name, options["public"])
+        if Dataset.init(owner, dataset_name, options["public"], bucket: bucket)
           path = Dir.pwd
           @dataset = Dataset.new(path)
 
@@ -837,7 +846,7 @@ module Cnvrg
         check = Helpers.checkmark
         if Dataset.clone(owner, dataset_name, slug, remote)
           log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
-          @files = Cnvrg::Datafiles.new(owner, slug)
+          @files = Cnvrg::Datafiles.new(owner, slug, dataset: Cnvrg::Dataset.new(dataset_home, dataset_url: dataset_url))
           log_message("Downloading files", Thor::Shell::Color::BLUE)
           if only_tree
             success = Dataset.clone_tree(commit: commit, dataset_home: dataset_home)
@@ -853,7 +862,6 @@ module Cnvrg
                                            :starting_at => 0,
                                            :total => files_count,
                                            :autofinish => true)
-
           while files['keys'].length > 0
             Cnvrg::Logger.log_info("download multiple files, #{downloaded_files.size} files downloaded")
             @files.download_multiple_files_s3(files, dataset_home, progressbar: progressbar, read_only: read)
@@ -1781,7 +1789,7 @@ module Cnvrg
 
         if clone_resp
           @project = Project.new(project_home)
-          @files = Cnvrg::Files.new(@project.owner, slug, project_home: project_home)
+          @files = Cnvrg::Files.new(@project.owner, slug, project_home: project_home, project: @project)
           response = @project.clone(remote, commit_to_clone)
           Cnvrg::CLI.is_response_success response
           commit_sha1 = response["result"]["commit"]
@@ -1974,7 +1982,7 @@ module Cnvrg
       commit_sha1 = nil
       @dataset = Dataset.new(dataset_dir)
       @dataset.backup_idx
-      @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
+      @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
       next_commit = @dataset.get_next_commit #if there was a partial commit..
       chunks = (@dataset.list_all_files.length.to_f / chunk_size).ceil
       resp = @files.start_commit(new_branch, force, chunks: chunks, dataset: @dataset)
@@ -2001,7 +2009,7 @@ module Cnvrg
         log_start(__method__, args, options)
         dataset_dir = is_cnvrg_dir(Dir.pwd)
         @dataset = Dataset.new(dataset_dir)
-        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
+        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
         force = options["force"] || false
         resp = @files.end_commit(commit, force, success: success, uploaded_files: uploaded_files)
         if (resp.present? and resp["result"])
@@ -2069,7 +2077,7 @@ module Cnvrg
       log_start(__method__, args, options)
       dataset_dir = is_cnvrg_dir(Dir.pwd)
       @dataset = Dataset.new(dataset_dir)
-      @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
+      @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
       new_commit ||= @dataset.get_next_commit
       partial_commit = @dataset.get_partial_commit
       if new_commit.blank?
@@ -2374,7 +2382,7 @@ module Cnvrg
         log_start(__method__, args, options)
         dataset_dir = is_cnvrg_dir(Dir.pwd)
         @dataset = Dataset.new(dataset_dir)
-        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
+        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
         all_files = all_files
         res = @dataset.compare_idx_download(all_files: all_files, desired_commit: commit)
         unless CLI.is_response_success(res, false)
@@ -2476,7 +2484,7 @@ module Cnvrg
         log_start(__method__, args, options)
         project_home = get_project_home
         @project = Project.new(project_home)
-        @files = Cnvrg::Files.new(@project.owner, @project.slug, project_home: project_home)
+        @files = Cnvrg::Files.new(@project.owner, @project.slug, project_home: project_home, project: @project)
         git = options["git"]
         commit = options["commit"]
         if git or @project.is_git
@@ -2607,7 +2615,7 @@ module Cnvrg
           exit(0)
         end
         log_message("Jumping to commit #{commit_sha1}")
-        @files = Cnvrg::Files.new(@project.owner, @project.slug, project_home: project_home)
+        @files = Cnvrg::Files.new(@project.owner, @project.slug, project_home: project_home, project: @project)
         resp = @project.jump_idx(destination: commit_sha1)
         if resp.blank?
           log_message("Cant find the given commit", Thor::Shell::Color::RED)
@@ -2651,7 +2659,7 @@ module Cnvrg
 
 
 
-      @files = Cnvrg::Files.new(@project.owner, @project.slug)
+      @files = Cnvrg::Files.new(@project.owner, @project.slug, project: @project)
       begin
 
         file = @files.show_file_s3(path, commit)
@@ -2788,7 +2796,6 @@ module Cnvrg
     method_option :local_folders, :type => :string, :aliases => ["--local_folders"], :default => nil
 
     def run(*cmd)
-
       verify_logged_in(true)
       log_start(__method__, args, options)
       datasets = options["datasets"]
@@ -4816,7 +4823,6 @@ module Cnvrg
         user = options[:user]
         is_gpu = options[:gpu]
         res = @image.upload_docker_image(image_path, image_name, workdir, user, description, is_gpu)
-
         if res["status"] == 200
           image_slug = res["id"]
           owner = CLI.get_owner
@@ -5326,19 +5332,24 @@ module Cnvrg
       end
 
       def self.log_message(message, type = Thor::Shell::Color::BLUE, to_print: true)
-        say "#{type}#{message}" if to_print
         case type
-        when Thor::Shell::Color::BLUE
+        when Thor::Shell::Color::BLUE, 'blue', 'progress'
+          color = Thor::Shell::Color::BLUE
           $LOG.info message: message, type: "info"
-        when Thor::Shell::Color::RED
+        when Thor::Shell::Color::RED, 'red', 'error'
+          color = Thor::Shell::Color::RED
           $LOG.error message: message, type: "error"
-        when Thor::Shell::Color::YELLOW
+        when Thor::Shell::Color::YELLOW, 'yellow', 'warning'
+          color = Thor::Shell::Color::YELLOW
           $LOG.warn message: message, type: "warning"
-        when Thor::Shell::Color::GREEN
+        when Thor::Shell::Color::GREEN, 'green', 'info'
+          color = Thor::Shell::Color::GREEN
           $LOG.info message: message, type: "success"
         else
+          color = nil
           $LOG.info message: message, type: "unknown"
         end
+        say "#{color}#{message}" if to_print
       end
 
       def log_message(message, type=Thor::Shell::Color::GREEN, to_print = true)
@@ -5450,6 +5461,10 @@ module Cnvrg
         return Cnvrg::CLI.get_project_home
       end
 
+      def get_job
+
+      end
+
       def self.get_project_home
         absolute_path = Dir.pwd
         dirs = absolute_path.split("/")

data/lib/cnvrg/data.rb

@@ -1,5 +1,3 @@
-require 'cnvrg/cli'
-require 'thor'
 class SubCommandBase < Thor
  def self.banner(command, namespace = nil, subcommand = false)
   "#{basename}  #{command.usage}"
@@ -15,10 +13,12 @@ module Cnvrg
     class_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
     desc "data init", "init data folder"
     method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
+    method_option :bucket, :type => :string, :aliases => ["-b", "--bucket"], :default => ""
     def init
       cli = Cnvrg::CLI.new()
       public = options["public"]
-      cli.init_data(public)
+      bucket = options["bucket"]
+      cli.init_data(public, bucket: bucket)
     end
     desc "data upload", "upload data folder"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
@@ -187,5 +187,16 @@ module Cnvrg
       cli.download_tags_yaml()
     end
 
+
+    desc 'data testclient', 'test client'
+    def test
+      cli = Cnvrg::CLI.new
+      cli.verify_logged_in(true)
+      cli.log_start(__method__, args, options)
+      dataset_dir = cli.is_cnvrg_dir(Dir.pwd)
+      @dataset = Project.new(dataset_dir)
+      resp = @dataset.get_storage_client
+    end
+
   end
 end

data/lib/cnvrg/datafiles.rb

@@ -19,6 +19,7 @@ module Cnvrg
       @owner = owner
       @dataset = dataset
       @base_resource = "users/#{owner}/datasets/#{dataset_slug}/"
+      @downloader = @dataset.get_storage_client
     end
 
     def check_file_sha1(filename, org_sha1, tag: 'conflict')
@@ -96,16 +97,11 @@ module Cnvrg
           progressbar.progress += tree.keys.length if progressbar.present?
           return 0
         end
-        props = Cnvrg::Helpers.get_s3_props(results)
-        client = props[:client]
-        bucket = props[:bucket]
-        upload_options = props[:upload_options]
-        s3_bucket = Aws::S3::Resource.new(client: client).bucket(bucket)
         files = results['files']
         progressbar.progress += tree.keys.length - files.length if progressbar.present?
         Parallel.map((files.keys), {in_threads: threads}) do |k|
           o = tree[k].merge(files[k])
-          upload_single_file(o, s3_bucket, upload_options)
+          upload_single_file(o)
           progressbar.progress += 1 if progressbar.present?
         end
         blob_ids = files.values.map {|f| f['bv_id']}
@@ -126,16 +122,14 @@ module Cnvrg
 
     end
 
-    def upload_single_file(file, s3_bucket, upload_options={})
+    def upload_single_file(file)
       begin
-      file = file.with_indifferent_access
-      Cnvrg::Logger.log_info("Uploading #{file[:absolute_path]}")
-      obj = s3_bucket.
-        object(file[:path])
-      obj.upload_file(file[:absolute_path], upload_options)
-      Cnvrg::Logger.log_info("#{file[:absolute_path]} uploaded.")
+      file = file.as_json
+      Cnvrg::Logger.log_info("Uploading #{file["absolute_path"]}")
+      @downloader.upload(file["path"], file["absolute_path"])
+      Cnvrg::Logger.log_info("#{file["absolute_path"]} uploaded.")
       rescue => e
-        Cnvrg::Logger.log_error_message("Error while upload single file #{file[:path]}")
+        Cnvrg::Logger.log_error_message("Error while upload single file #{file["path"]}")
         Cnvrg::Logger.log_error(e)
       end
     end
@@ -889,9 +883,8 @@ module Cnvrg
 
     def download_files_in_chunks(files, chunk_size: 1000, conflict: false, commit: 'latest', progress: nil)
       begin
-        ttl_files = 0
         files.each_slice(chunk_size).each do |files|
-          ttl_files += download_files_chunk(files, conflict: conflict, progress: progress)
+          download_files_chunk(files, conflict: conflict, progress: progress)
         end
         return Cnvrg::Result.new(true, "Download Completed")
       rescue Exception => e
@@ -942,61 +935,26 @@ module Cnvrg
 
     def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false)
       begin
-        props = Cnvrg::Helpers.get_s3_props(files)
-        client = props[:client]
-        iv = props[:iv]
-        key = props[:key]
-        bucket = props[:bucket]
-        download_succ_count = 0
-        if read_only
-          in_threads = 1000
-        else
-          in_threads = ParallelThreads
-        end
         parallel_options = {
-            in_threads: in_threads,
+            in_threads: ParallelThreads,
             isolation: true
         }
         Parallel.map(files["keys"], parallel_options) do |f|
-
-          file_path = f["name"]
+          local_path = @dataset.working_dir + '/' + f['name']
+          Cnvrg::Logger.log_info("Downloading #{local_path}")
           progressbar.progress += 1 if progressbar.present?
-          if file_path.end_with? "/"
-            # dir
-            if download_dir(project_home, file_path)
-              download_succ_count += 1
-            else
-              return Cnvrg::Result.new(false,"Could not create directory: #{file_path}")
-              raise Parallel::Kill
-            end
-          else
+          if local_path.end_with? "/"
+            @downloader.mkdir(local_path, recursive: true)
+            next
+          end
             # blob
             begin
-            file_key =  Cnvrg::Helpers.decrypt(key,iv, f["path"])
-            resp = false
-            file_path = "#{file_path}.conflict" if conflict
-            dirname = File.dirname file_path
-            download_dir(project_home, dirname) unless dirname.eql? "."
-            Cnvrg::Logger.info("Download #{file_path}")
-            File.open(project_home+"/"+file_path, 'w+') do |file|
-              resp = client.get_object({bucket:bucket,
-                                        key:file_key}, target: file)
-            end
-            Cnvrg::Logger.log_info("Download #{file_path} success")
+            local_path = "#{local_path}.conflict" if conflict
+            storage_path = f["path"]
 
-            if resp
-              download_succ_count +=1
-            else
-              return Cnvrg::Result.new(false,"Could not create file: #{file_path}")
+            @downloader.download(storage_path, local_path)
+            Cnvrg::Logger.log_info("Download #{local_path} success")
             end
-            rescue => e
-              return Cnvrg::Result.new(false,"Could not create file: #{file_path}", e.message, e.backtrace)
-              raise Parallel::Kill
-            end
-          end
-        end
-        if download_succ_count == files["keys"].size
-          return Cnvrg::Result.new(true,"Done.\nDownloaded #{download_succ_count} files")
         end
         rescue => e
           return Cnvrg::Result.new(false,"Could not download some files", e.message, e.backtrace)