cnvrg 1.9.9.9.7

data/lib/cnvrg/result.rb

@@ -0,0 +1,29 @@
+require 'fileutils'
+module Cnvrg
+  class Result
+    attr_reader :msg, :msg_color, :e_msg, :e_msg_backtrace
+
+    def initialize(success, msg, e_msg = "", e_backtrace = "")
+      begin
+       @success = success
+        @msg = msg
+       @e_msg = e_msg
+       @e_msg_backtrace = e_backtrace
+        if !@success
+          @msg_color = Thor::Shell::Color::RED
+        else
+          @msg_color = Thor::Shell::Color::GREEN
+
+        end
+      rescue => e
+      end
+
+    end
+    def is_success?
+      return @success
+    end
+
+  end
+
+
+  end

data/lib/cnvrg/runner.rb

@@ -0,0 +1,49 @@
+require 'cnvrg'
+
+module Cnvrg
+  class Runner
+    # Allow everything fun to be injected from the outside while defaulting to normal implementations.
+    def initialize(argv, stdin = STDIN, stdout = STDOUT, stderr = STDERR, kernel = Kernel)
+      @argv, @stdin, @stdout, @stderr, @kernel = argv, stdin, stdout, stderr, kernel
+    end
+
+    def execute!
+      exit_code = begin
+                    # Thor accesses these streams directly rather than letting them be injected, so we replace them...
+        $stderr = @stderr
+        $stdin = @stdin
+        $stdout = @stdout
+
+        # Run our normal Thor app the way we know and love.
+        Cnvrg::CLI.start(@argv)
+
+        # Thor::Base#start does not have a return value, assume success if no exception is raised.
+        0
+      rescue StandardError => e
+        # The ruby interpreter would pipe this to STDERR and exit 1 in the case of an unhandled exception
+        b = e.backtrace
+        @stderr.puts("#{b.shift}: #{e.message} (#{e.class})")
+        @stderr.puts(b.map{|s| "\tfrom #{s}"}.join("\n"))
+        1
+      rescue SystemExit => e
+        e.status
+      ensure
+        # TODO: reset your app here, free up resources, etc.
+        # Examples:
+        # MyApp.logger.flush
+        # MyApp.logger.close
+        # MyApp.logger = nil
+        #
+        # MyApp.reset_singleton_instance_variables
+
+        # ...then we put the streams back.
+        $stderr = STDERR
+        $stdin = STDIN
+        $stdout = STDOUT
+      end
+
+      # Proxy our exit code back to the injected kernel.
+      @kernel.exit(exit_code)
+    end
+  end
+end

data/lib/cnvrg/ssh.rb

@@ -0,0 +1,94 @@
+require 'fileutils'
+require 'cnvrg/files'
+require 'net/ssh'
+
+
+module Cnvrg
+  class Ssh
+    attr_reader :is_ssh
+
+
+    def initialize(resp)
+      begin
+        @is_ssh = false
+        sts_path = resp["result"]["sts_path"]
+
+        uri = URI.parse(sts_path)
+
+        http_object = Net::HTTP.new(uri.host, uri.port)
+        http_object.use_ssl = true if uri.scheme == 'https'
+        request = Net::HTTP::Get.new(sts_path)
+
+        body = ""
+        http_object.start do |http|
+          response = http.request request
+          body = response.read_body
+        end
+
+        URLcrypt::key = [body].pack('H*')
+        ip = URLcrypt.decrypt(resp["result"]["machine_i"])
+
+        @user = URLcrypt.decrypt(resp["result"]["machine_u"])
+        key = URLcrypt.decrypt(resp["result"]["machine_k"])
+        @container = URLcrypt.decrypt(resp["result"]["machine_c"])
+
+        tempssh = Tempfile.new "sshkey"
+        tempssh.write open(key).read
+        tempssh.rewind
+        key_path = tempssh.path
+        count = 0
+        while count < 5
+
+          begin
+            @ssh = Net::SSH.start(ip, user=@user, :keys => key_path, :timeout => 10)
+            if !@ssh.nil?
+              @is_ssh = true
+              return
+            else
+              count+=1
+              sleep(2)
+
+            end
+          rescue
+            count+=1
+            sleep(2)
+
+
+          end
+        end
+        if tempssh
+          tempssh.close
+          tempssh.unlink
+        end
+        return false
+      rescue => e
+
+        puts e
+      end
+    end
+
+
+
+    def exec_command(command)
+      exec_command = "sudo -i -u #{@user} cnvrg exec_container #{@container} \"#{command}\" "
+      return @ssh.exec!(exec_command)
+    end
+
+
+    def close_ssh()
+
+
+      begin
+
+        @ssh.close
+      rescue => e
+
+      end
+
+
+    end
+  end
+
+
+
+end

data/lib/cnvrg/storage.rb

@@ -0,0 +1,128 @@
+module Cnvrg
+  class Storage
+    def initialize(dataset: nil, project: nil, root_path: nil)
+      @element = dataset || project
+      @root_path = root_path
+      @client = @element.get_storage_client
+    end
+
+    def log_error(action: nil, error: '')
+      "[#{Time.now}] (#{action || 'default'}) #{error}"
+    end
+
+
+    def get_chunks_size
+      (ENV['CNVRG_STORAGE_CHUNK_SIZE'] || 10).to_i
+    end
+
+    def init_progress_bar(size: nil, title: "Download Progress")
+      @progressbar = ProgressBar.create(:title => title,
+                                       :progress_mark => '=',
+                                       :format => "%b>>%i| %p%% %t",
+                                       :starting_at => 0,
+                                       :total => size,
+                                       :autofinish => true)
+    end
+
+    def make_progress(size: 1)
+      @progressbar.progress += size
+    end
+
+
+    def clone(commit: nil)
+      files_generator = Proc.new do |params|
+        @element.get_clone_chunk(commit: commit, chunk_size: params[:limit], offset: params[:offset])
+      end
+      action = Proc.new do |storage, local|
+        @client.download(storage, local)
+      end
+
+      @stats = @element.get_stats
+      progress = {size: @stats['commit_size'], title: "Clone Progress"}
+
+      storage_action(files_generator: files_generator, action: action, progress: progress)
+    end
+
+    def upload_files(commit: nil)
+
+    end
+
+    def upload_files(files_generator, progress: {size: 0, title: ''})
+      init_progress_bar(progress)
+      @storage_errors = []
+      @finished = false
+      @files = Queue.new
+      t = Thread.new{file_gen_thread(files_generator)}
+      do_parallel{|file| self.upload_files_thread(file); self.make_progress(size: file['size'])}
+      t.join
+      handle_errors
+    end
+
+    def file_gen_upload_thread(files_generator)
+      while true
+        files = files_generator
+        files.each{|f| @files.push(f)}
+        break if files.blank?
+      end
+      @finished = true
+    end
+
+    def storage_action(files_generator: nil, action: nil, progress: {size: 0, title: ''})
+      ### the generator files should have {path (encrypted), name, size}
+      init_progress_bar(progress)
+      @storage_errors = []
+      @finished = false
+      @files = Queue.new
+      t = Thread.new{file_gen_thread(files_generator)}
+      do_parallel do |file|
+        self.download_file_thread(file) do |local, storage|
+          action.call(local, storage)
+        end
+        self.make_progress(size: file['size'])
+      end
+      t.join
+      handle_errors
+    end
+
+    def file_gen_thread(file_gen)
+      offset = 0
+      chunk_size = get_chunks_size
+      while true
+        files = file_gen.call(limit: chunk_size, offset: offset)
+        break if files.blank?
+        files.each{|f| @files.push(f)}
+        offset += files.size
+      end
+      @finished = true
+    end
+
+    def handle_errors
+      if @storage_errors.present?
+        File.open(@element.working_dir + "/.cnvrg/errors.yml", "w+"){|f| f.write @storage_errors.to_yaml}
+      end
+    end
+
+    def do_parallel
+      Parallel.each( -> { @files.empty? ? (@finished ? Parallel::Stop : sleep(1)) : @files.pop }, in_threads: get_chunks_size) do |file|
+        if file == 1
+          next
+        end
+        yield(file)
+      end
+    end
+
+    def download_file_thread(file)
+      return if file.blank?
+      local_path = file['name']
+      storage_path = file['path']
+      (0..5).each do
+        begin
+        # @client.download(storage_path, "#{@root_path}/#{local_path}")
+        break
+        rescue => e
+          log_error(action: "download #{local_path}", error: e.message)
+        end
+      end
+    end
+  end
+end

data/lib/cnvrg/task.rb

@@ -0,0 +1,165 @@
+module Cnvrg
+  class Task
+    attr_accessor :title, :path, :source
+    def initialize(project_path, path: nil, content: {})
+      @path = path
+      @content = content
+      @source = 'flow' if @content.present?
+      @source = 'file' if @path.present?
+      @project = Cnvrg::Project.new(project_path)
+      @project_path = project_path
+      #generic
+      @type = nil #type can be exec, data, library, deploy
+      @compute = 'medium'
+      @title = nil
+      @uid = nil
+      @params = {}
+
+      #exec
+      @cmd = nil
+      @params_path = nil
+
+      #library
+      @library = nil
+
+      #dataset
+      @dataset = nil
+      @query = nil
+
+      #deploy
+      @function = nil
+
+
+      @base_resource = @project.base_resource + "/tasks"
+
+      self.reload_task
+    end
+
+    def save
+      path = @path || gen_path
+      File.open(path, 'w'){|f| f.write(get_content.to_yaml)}
+    end
+
+    def reload_task
+      task_raw = get_content
+      @title = task_raw[:title]
+      @type = task_raw[:type]
+      @uid = task_raw[:uid]
+      @title = task_raw[:title] || @uid
+      @compute = task_raw[:compute] || @compute
+
+      case @type
+      when 'exec'
+        @cmd = task_raw[:cmd]
+        init_params(task_raw)
+      when 'library'
+        @library = task_raw[:library]
+        init_params(task_raw)
+      when 'data'
+        @dataset = task_raw[:dataset]
+        @query = task_raw[:query]
+      when 'deploy'
+        @cmd = task_raw[:cmd]
+        @function = task_raw[:function]
+      else
+        error("Cant parse task of type #{@type}")
+      end
+    end
+
+    def verify_task
+      case @type
+      when 'exec'
+        verify_exec
+      when 'data'
+        verify_data
+      when 'deploy'
+        verify_deploy
+      when 'library'
+        verify_library
+      else
+        error("Cant parse task of type #{@type}")
+      end
+    end
+
+
+    def to_api
+      get_content.merge(params: @params)
+    end
+
+    def run
+      verify_task
+      if @type == 'data'
+        raise StandardError.new("Data Tasks are not runnable")
+      end
+      resp = Cnvrg::API.request(@base_resource, "POST", {task: to_api})
+      Cnvrg::CLI.is_response_success(resp, true)
+      Cnvrg::Helpers.remote_url + resp['result']['url']
+    end
+
+    private
+    def verify_compute
+      unless @project.check_machine(@compute)
+        raise StandardError.new("Cant find #{@compute} machine in project.")
+      end
+    end
+
+    def init_params(task_raw)
+      @params_path = task_raw[:params_path].presence
+      @params = task_raw[:params] || @params
+      if @params_path.present?
+        @hyper = Cnvrg::Hyper.new(@project_path, @params_path)
+        @params = @hyper.resolve_params
+      end
+    end
+
+    def verify_exec
+      if @cmd.blank?
+        error("Cant find command")
+      end
+      verify_compute
+    end
+
+    def verify_data
+      if @dataset.blank?
+        error("Cant find dataset slug")
+      end
+    end
+
+    def verify_deploy
+      error("Cant find command") if @cmd.blank?
+      error("Cant find function") if @function.blank?
+      verify_compute
+    end
+
+    def verify_library
+      error("Cant find library") if @library.blank?
+    end
+
+    def get_content
+      return @content if @source == 'flow'
+      unless File.exists? @path
+        raise StandardError.new("Cant find task in #{@path}")
+      end
+      YAML.load_file(@path)
+    end
+
+    def gen_path
+      @title ||= "#{@type.capitalize}Task"
+      unless File.exists? @title
+        @path = "#{@title}.task.yaml"
+        return @path
+      end
+      i = 0
+      while File.exists? "#{@title}_#{i}.task.yaml"
+        i += 1
+      end
+      @path = "#{@title}.task.yaml"
+      return @path
+    end
+
+    def error(msg)
+      raise StandardError.new("task: #{@uid} - #{msg}")
+    end
+
+  end
+end