cnvrg 1.2.7 → 1.3.2

data/lib/cnvrg/job_cli.rb

@@ -0,0 +1,33 @@
+module Cnvrg
+    class JobCli < SubCommandBase
+
+        desc 'log', description: '', hide: true
+        method_option :level, :type => :string, :aliases => ["-l", "--level"], :default => 'info'
+        method_option :step, :type => :string, :aliases => ["-s", "--step"], :default => nil
+        def log(*logs)
+            Cnvrg::CLI.new.log_start(__method__, args, options)
+            @project = Project.new(owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
+            @project.job_log(logs, level: options['level'], step: options['step'])
+        end
+
+
+        desc 'start', description: '', hide: true
+        def start(*logs)
+            cli = Cnvrg::CLI.new
+            cli.log_start(__method__, args, options)
+            @project = Project.new(nil, owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
+            @executer = Helpers::Executer.new(project: @project, job_type: ENV['CNVRG_JOB_TYPE'], job_id: ENV['CNVRG_JOB_ID'])
+            commands = @executer.fetch_commands
+            @executer.execute_cmds(commands)
+        end
+
+        desc 'requirements', description: '', hide: true
+        def requirements
+          cli = Cnvrg::CLI.new
+          cli.log_start(__method__, args, options)
+          project_dir = cli.get_project_home
+          @project = Project.new(project_dir, owner: ENV['CNVRG_OWNER'], slug: ENV['CNVRG_PROJECT'])
+        end
+
+    end
+end

data/lib/cnvrg/project.rb

@@ -7,11 +7,11 @@ module Cnvrg
     IDXParallelThreads ||= Cnvrg::Helpers.parallel_threads
 
 
-    def initialize(project_home)
+    def initialize(project_home=nil, slug: nil, owner: nil)
       begin
         @local_path = project_home
         @working_dir = project_home
-        config = YAML.load_file(project_home + "/.cnvrg/config.yml")
+        config = YAML.load_file(project_home + "/.cnvrg/config.yml") rescue {owner: owner, project_slug: slug}
         @title = config[:project_name]
         @slug = config[:project_slug]
         @owner = config[:owner]
@@ -126,7 +126,7 @@ module Cnvrg
 
     # Create project
 
-    def self.create(project_name, clean, with_docker = false)
+    def self.create(project_name, clean, with_docker = false, bucket: nil)
       if clean
         list_dirs = [project_name, project_name + "/.cnvrg"]
       else
@@ -154,7 +154,7 @@ module Cnvrg
       begin
 
         owner = Cnvrg::CLI.get_owner()
-        response = Cnvrg::API.request("cli/create_project", 'POST', {title: project_name, owner: owner, is_docker: with_docker})
+        response = Cnvrg::API.request("cli/create_project", 'POST', {title: project_name, owner: owner, is_docker: with_docker, bucket: bucket})
         Cnvrg::CLI.is_response_success(response)
         response = JSON.parse response["result"]
         project_slug = response["slug"]
@@ -177,7 +177,7 @@ module Cnvrg
       return true
     end
 
-    def self.link(owner, project_name, docker = false, git = false)
+    def self.link(owner, project_name, docker = false, git = false, bucket: nil)
       ignore_exits = File.exist? ".cnvrgignore"
       list_dirs = [".cnvrg"
       ]
@@ -192,7 +192,7 @@ module Cnvrg
       cnvrgreadme = Helpers.readme_content
       cnvrgignore = Helpers.cnvrgignore_content
       begin
-        response = Cnvrg::API.request("cli/create_project", 'POST', {title: project_name, owner: owner, is_docker: docker})
+        response = Cnvrg::API.request("cli/create_project", 'POST', {title: project_name, owner: owner, is_docker: docker, bucket: bucket})
         Cnvrg::CLI.is_response_success(response)
         response = JSON.parse response["result"]
         project_slug = response["slug"]
@@ -324,6 +324,30 @@ module Cnvrg
       YAML.load_file(@working_dir + "/.cnvrg/config.yml") rescue {}
     end
 
+    def get_storage_client
+      response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/client", 'GET')
+      if Cnvrg::CLI.is_response_success(response, false)
+        client_params = response['client']
+      else
+        client_params = get_storage_client_fallback
+      end
+      Cnvrg::Downloader::Client.factory(client_params)
+    end
+
+    def get_storage_client_fallback
+      response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/download_files", "POST", {files: [], commit: get_latest_commit})
+      raise StandardError.new("Can't find project credentials") unless Cnvrg::CLI.is_response_success(response, false)
+      files = response['result']
+      storage = files['is_s3'] ? 's3' : 'minio'
+      files['storage'] = storage
+      files
+    end
+
+    def get_latest_commit
+      resp = clone(0, '')
+      resp['result']['commit']
+    end
+
     def set_config(config)
       slug = config[:project_slug] rescue nil
       owner = config[:owner] rescue nil
@@ -631,6 +655,21 @@ module Cnvrg
       self.set_config(config)
     end
 
+    def job_log(logs, level: 'info', step: nil, job_type: nil, job_id: nil)
+      job_type ||= ENV['CNVRG_JOB_TYPE']
+      job_id ||= ENV['CNVRG_JOB_ID']
+      logs = [logs].flatten
+      if job_type.blank? or job_id.blank?
+        raise StandardError.new("Cant find job env variables")
+      end
+      Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/jobs/#{job_type.underscore}/#{job_id}/log", "POST", {job_type: job_type, job_id: job_id, logs: logs, log_level: level, step: step, timestamp: Time.now})
+    end
+
+
+    def job_commands
+      job_type, job_id = ENV['CNVRG_JOB_TYPE'], ENV['CNVRG_JOB_ID']
+      resp = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/jobs/#{job_type.underscore}/#{job_id}/commands", "GET")
+    end
 
     def spot_will_terminate
       restart = false

data/lib/cnvrg/storage.rb

@@ -0,0 +1,128 @@
+module Cnvrg
+  class Storage
+    def initialize(dataset: nil, project: nil, root_path: nil)
+      @element = dataset || project
+      @root_path = root_path
+      @client = @element.get_storage_client
+    end
+
+    def log_error(action: nil, error: '')
+      "[#{Time.now}] (#{action || 'default'}) #{error}"
+    end
+
+
+    def get_chunks_size
+      (ENV['CNVRG_STORAGE_CHUNK_SIZE'] || 10).to_i
+    end
+
+    def init_progress_bar(size: nil, title: "Download Progress")
+      @progressbar = ProgressBar.create(:title => title,
+                                       :progress_mark => '=',
+                                       :format => "%b>>%i| %p%% %t",
+                                       :starting_at => 0,
+                                       :total => size,
+                                       :autofinish => true)
+    end
+
+    def make_progress(size: 1)
+      @progressbar.progress += size
+    end
+
+
+    def clone(commit: nil)
+      files_generator = Proc.new do |params|
+        @element.get_clone_chunk(commit: commit, chunk_size: params[:limit], offset: params[:offset])
+      end
+      action = Proc.new do |storage, local|
+        @client.download(storage, local)
+      end
+
+      @stats = @element.get_stats
+      progress = {size: @stats['commit_size'], title: "Clone Progress"}
+
+      storage_action(files_generator: files_generator, action: action, progress: progress)
+    end
+
+    def upload_files(commit: nil)
+
+    end
+
+    def upload_files(files_generator, progress: {size: 0, title: ''})
+      init_progress_bar(progress)
+      @storage_errors = []
+      @finished = false
+      @files = Queue.new
+      t = Thread.new{file_gen_thread(files_generator)}
+      do_parallel{|file| self.upload_files_thread(file); self.make_progress(size: file['size'])}
+      t.join
+      handle_errors
+    end
+
+    def file_gen_upload_thread(files_generator)
+      while true
+        files = files_generator
+        files.each{|f| @files.push(f)}
+        break if files.blank?
+      end
+      @finished = true
+    end
+
+    def storage_action(files_generator: nil, action: nil, progress: {size: 0, title: ''})
+      ### the generator files should have {path (encrypted), name, size}
+      init_progress_bar(progress)
+      @storage_errors = []
+      @finished = false
+      @files = Queue.new
+      t = Thread.new{file_gen_thread(files_generator)}
+      do_parallel do |file|
+        self.download_file_thread(file) do |local, storage|
+          action.call(local, storage)
+        end
+        self.make_progress(size: file['size'])
+      end
+      t.join
+      handle_errors
+    end
+
+    def file_gen_thread(file_gen)
+      offset = 0
+      chunk_size = get_chunks_size
+      while true
+        files = file_gen.call(limit: chunk_size, offset: offset)
+        break if files.blank?
+        files.each{|f| @files.push(f)}
+        offset += files.size
+      end
+      @finished = true
+    end
+
+    def handle_errors
+      if @storage_errors.present?
+        File.open(@element.working_dir + "/.cnvrg/errors.yml", "w+"){|f| f.write @storage_errors.to_yaml}
+      end
+    end
+
+    def do_parallel
+      Parallel.each( -> { @files.empty? ? (@finished ? Parallel::Stop : sleep(1)) : @files.pop }, in_threads: get_chunks_size) do |file|
+        if file == 1
+          next
+        end
+        yield(file)
+      end
+    end
+
+    def download_file_thread(file)
+      return if file.blank?
+      local_path = file['name']
+      storage_path = file['path']
+      (0..5).each do
+        begin
+        # @client.download(storage_path, "#{@root_path}/#{local_path}")
+        break
+        rescue => e
+          log_error(action: "download #{local_path}", error: e.message)
+        end
+      end
+    end
+  end
+end

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '1.2.7'
+  VERSION = '1.3.2'
 end
 

metadata

@@ -1,30 +1,31 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.2.7
+  version: 1.3.2
 platform: ruby
 authors:
 - Yochay Ettun
 - Leah Kolben
+- Omer Shacham
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-26 00:00:00.000000000 Z
+date: 2019-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -217,6 +218,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
+- !ruby/object:Gem::Dependency
+  name: google-cloud-storage
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
 - !ruby/object:Gem::Dependency
   name: sucker_punch
   requirement: !ruby/object:Gem::Requirement
@@ -379,18 +394,24 @@ files:
 - lib/cnvrg/data.rb
 - lib/cnvrg/datafiles.rb
 - lib/cnvrg/dataset.rb
+- lib/cnvrg/downloader/client.rb
+- lib/cnvrg/downloader/clients/azure_client.rb
+- lib/cnvrg/downloader/clients/gcp_client.rb
+- lib/cnvrg/downloader/clients/s3_client.rb
 - lib/cnvrg/experiment.rb
 - lib/cnvrg/files.rb
 - lib/cnvrg/flow.rb
 - lib/cnvrg/helpers.rb
+- lib/cnvrg/helpers/executer.rb
 - lib/cnvrg/hyper.rb
-- lib/cnvrg/job.rb
+- lib/cnvrg/job_cli.rb
 - lib/cnvrg/logger.rb
 - lib/cnvrg/org_helpers.rb
 - lib/cnvrg/project.rb
 - lib/cnvrg/result.rb
 - lib/cnvrg/runner.rb
 - lib/cnvrg/ssh.rb
+- lib/cnvrg/storage.rb
 - lib/cnvrg/task.rb
 - lib/cnvrg/version.rb
 homepage: https://cnvrg.io

data/lib/cnvrg/job.rb

@@ -1,40 +0,0 @@
-require 'mimemagic'
-require 'aws-sdk'
-require 'URLcrypt'
-
-require 'sucker_punch'
-module Cnvrg
-
-class LogJob
-    include SuckerPunch::Job
-
-    def perform(upload_resp, file_path)
-        begin
-            sts_path = upload_resp["result"]["path_sts"]
-            uri = URI.parse(sts_path)
-            http_object = Net::HTTP.new(uri.host, uri.port)
-            http_object.use_ssl = true if uri.scheme == 'https'
-            request = Net::HTTP::Get.new(sts_path)
-            body = ""
-            http_object.start do |http|
-                response = http.request request
-                body = response.read_body
-            end
-            URLcrypt::key = [body].pack('H*')
-            s3 = Aws::S3::Resource.new(
-                :access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
-                :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
-                :session_token => URLcrypt.decrypt(upload_resp["result"]["sts_st"]),
-                :region => URLcrypt.decrypt(upload_resp["result"]["region"]))
-            resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
-                object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
-                upload_file(file_path)
-            return resp
-        rescue =>e
-            return false
-
-        end
-        return true
-    end
-end
-end