cnvrg 1.9.9.9.7

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: cb866c9448de7ea2062759a1f9cd9bf0f9db23b3fae06c8b61d178dbd1a227c9
+  data.tar.gz: dfc63cef384e0a8df9a367e03f544462638f5d0b43913dfe40b5dd59429ea2f8
+SHA512:
+  metadata.gz: 760d90fb1e7516a3e2f9ae72a7662ff02ae0e6324a883a17471ad5edacccc252f3b127a5bddf7ec0afcff6d42fe3b3f5596a6697fa058f5740ff0799894934b0
+  data.tar.gz: ae6196d1a37f4bc68534aa728199d93500ee8299c99a896452158d22cf1e73578cf277ba0742f3e302ef9e677c466a335e9dac214c608e15c8ed088d870ddb60

data/bin/cnvrg

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+# -*- mode: ruby -*-
+
+# require 'cnvrg'
+require 'cnvrg/runner'
+
+Cnvrg::Runner.new(ARGV.dup).execute!
+
+# Cnvrg::CLI.start(ARGV)

data/cnvrg.gemspec

@@ -0,0 +1,47 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'cnvrg/version'
+
+Gem::Specification.new do |spec|
+    spec.name          = 'cnvrg'
+    spec.version       = Cnvrg::VERSION
+    spec.authors       = ['Yochay Ettun', 'Leah Kolben', 'Omer Shacham']
+    spec.email         = ['info@cnvrg.io']
+    spec.summary       = %q{A CLI tool for interacting with cnvrg.io.}
+    spec.description   = %q{A CLI tool for interacting with cnvrg.io.}
+    spec.homepage      = 'https://cnvrg.io'
+
+    #spec.files         = `git ls-files`.split($/)
+    spec.files = %w[cnvrg.gemspec] + Dir['*.md', 'bin/*', 'lib/**/*.rb']
+    spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+    spec.executables   = ['cnvrg']
+    spec.require_paths = ['lib']
+
+    spec.add_development_dependency 'bundler'
+    spec.add_development_dependency 'rake', '~> 10.0'
+    spec.add_development_dependency 'rspec', '~> 3.0'
+    spec.add_development_dependency 'vcr', '~> 3.0'
+    spec.add_development_dependency 'aruba'
+    spec.add_development_dependency 'pry'	
+    
+    spec.add_runtime_dependency 'mimemagic', '~> 0.3.1','>=0.3.2'
+    spec.add_runtime_dependency 'faraday', '~> 0.15.2'
+    spec.add_runtime_dependency 'netrc', '~> 0.11.0'
+    spec.add_runtime_dependency 'open4', '~> 1.3', '>= 1.3.4'
+    spec.add_runtime_dependency 'highline', '~> 1.7', '>= 1.7.8'
+    spec.add_runtime_dependency 'thor', '~> 0.19.0','>=0.19.1'
+    spec.add_runtime_dependency 'aws-sdk-s3', '~> 1'
+    spec.add_runtime_dependency 'signet', '~> 0.11.0'
+    spec.add_runtime_dependency 'google-cloud-env', '~> 1.2.1'
+    spec.add_runtime_dependency 'google-cloud-core', '~> 1.3.2'
+    spec.add_runtime_dependency 'google-cloud-storage', '~> 1.21.1'
+    spec.add_runtime_dependency 'sucker_punch', '~> 2.0'
+    spec.add_runtime_dependency 'urlcrypt', '~> 0.1.1'
+    spec.add_runtime_dependency 'parallel', '~> 1.12.0'
+    spec.add_runtime_dependency 'azure-storage-blob', '~> 1.1.0'
+    spec.add_runtime_dependency 'logstash-logger', '~> 0.22.1'
+    spec.add_runtime_dependency 'activesupport', '~> 5.2.0'
+    spec.add_runtime_dependency 'ruby-progressbar'
+    spec.add_runtime_dependency 'down'
+end

data/lib/cnvrg.rb

@@ -0,0 +1,7 @@
+require 'rubygems'
+require 'cnvrg/version'
+require 'cnvrg/cli'
+require 'thor'
+
+module Cnvrg
+end

data/lib/cnvrg/Images.rb

@@ -0,0 +1,351 @@
+require 'fileutils'
+require 'cnvrg/files'
+require 'mimemagic'
+
+
+module Cnvrg
+  class Images
+    # attr_reader :image_name, :image_tag, :is_docker, :project_slug, :commit_id, :owner, :port, :image_slug
+
+
+    def initialize()
+      begin
+        home_dir = File.expand_path('~')
+        config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
+        @owner = config.to_h[:owner]
+        @username =  config.to_h[:username]
+      rescue => e
+        @owner = ""
+        @username =  ""
+        Cnvrg::Logger.log_info("cnvrg is not configured")
+      end
+
+    end
+
+    def upload_docker_image(image_path, image_name,workdir, user, description, is_gpu )
+      begin
+      if image_name.present? and image_name.include? ":"
+        image_split = image_name.split(":")
+        image_name = image_split[0]
+        image_tag = image_split[1]
+        if image_tag.blank?
+          image_tag = "latest"
+        end
+        if image_name.blank?
+          Cnvrg::Logger.log_info("image_name: #{image_name} is not valid")
+          return false
+        end
+      end
+      if !File.exist? image_path
+        Cnvrg::Logger.log_info("image_path: #{image_path} was not found")
+        return false
+      end
+      is_dockerfile = (!image_path.end_with? ".tar") ? true : false
+      file_name = File.basename image_path
+      file_size = File.size(image_path).to_f
+      mime_type = MimeMagic.by_path(image_path)
+      content_type = mime_type.present? ? mime_type.type : "application/x-tar"
+
+      image_res = Cnvrg::API.request("users/#{@owner}/images/upload" , 'POST_FILE', {image_name: image_name, image_tag: image_tag,
+                                                                                workdir: workdir, user: user, description:description, is_gpu:is_gpu,
+                                                                                file_name: file_name,relative_path:image_path,
+                                                                                file_size: file_size, file_content_type: content_type, is_dockerfile: is_dockerfile
+                                                                                 })
+
+      Cnvrg::CLI.is_response_success(image_res, true)
+
+      path = image_res["result"]["path"]
+      image_id = image_res["result"]["image_id"]
+
+      props = Cnvrg::Helpers.get_s3_props(image_res["result"])
+      if props.is_a? Cnvrg::Result
+        return false
+      end
+
+      client = props[:client]
+      upload_options = props[:upload_options]
+      bucket = Aws::S3::Resource.new(client: client).bucket(props[:bucket])
+      resp = bucket.object(path).
+          upload_file(image_path, upload_options)
+      unless resp
+        raise Exception.new("Cant upload #{image_path}")
+        return false
+      end
+      return save_docker_image(image_id)
+      rescue => e
+        Cnvrg::Logger.log_error(e)
+
+        return false
+      end
+
+
+
+    end
+
+    def save_docker_image(image_id)
+      image_res = Cnvrg::API.request("users/#{@owner}/images/#{image_id}/save" , 'POST', {})
+       Cnvrg::CLI.is_response_success(image_res, true)
+      return image_res
+    end
+
+    def is_container_exist()
+      config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
+      if config[:container].nil?
+        return false
+      end
+      return config[:container]
+    end
+
+    def container_port()
+      config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
+      if config[:container].nil?
+        return false
+      else
+        return config[:port]
+      end
+    end
+
+    def self.image_exist(owner, image_name)
+
+      image_res = Cnvrg::API.request("users/#{owner}/images/" + "find", 'POST', {image_name: image_name})
+
+      if Cnvrg::CLI.is_response_success(image_res)
+        image= image_res["result"]["image"]
+        return image
+      else
+        return false
+
+      end
+    end
+    def store_image_build_commands(working_dir, cmd)
+      begin
+      custom_image_file = working_dir+"/.cnvrg/custom_image.txt"
+      if !File.exist? custom_image_file
+        FileUtils.touch [custom_image_file]
+      end
+      File.open(custom_image_file, 'a' ) do  |f|
+        f.puts cmd
+      end
+      rescue
+      end
+
+
+    end
+    def self.create_new_custom_image(type,owner,image_name,is_public,is_base,image_extend,python3,tar_path)
+       response = Cnvrg::API.request("users/#{owner}/images/custom", 'POST', {instance_type:type,image_name:image_name,is_public:is_public,
+                                                                                is_base:is_base,image_extend:image_extend,
+                                                                                python3:python3})
+
+      return response
+    end
+    def self.create_new_custom_image_with_docker(type,owner,image_name,is_public,is_base,image_extend,python3,tar_path,files)
+        file_name = File.basename tar_path
+        file_size = File.size(tar_path).to_f
+        mime_type = MimeMagic.by_path(tar_path)
+        content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
+
+        upload_resp = Cnvrg::API.request("/users/#{owner}/images/upload_docker", 'POST_FILE', {absolute_path: tar_path, relative_path: tar_path,
+                                                                                        file_name: file_name, file_size: file_size,
+                                                                                        file_content_type: content_type,
+                                                                                        image_name:image_name,is_public:is_public,
+                                                                                        is_base:is_base,image_extend:image_extend,
+                                                                                        python3:python3 })
+
+        if Cnvrg::CLI.is_response_success(upload_resp, false)
+          path = upload_resp["result"]["path"]
+
+          s3_res = files.upload_small_files_s3(path, tar_path, content_type)
+          if s3_res
+            image_slug = upload_resp["result"]["id"]
+            response = Cnvrg::API.request("users/#{owner}/images/#{image_slug}/build", 'POST', {instance_type:type,image_extend:image_extend})
+          end
+        end
+
+    end
+
+
+
+    def self.revoke_custom_new_image(owner,slug)
+      response = Cnvrg::API.request("users/#{owner}/images/#{slug}/revoke_image", 'GET')
+      return response
+    end
+    def self.commit_custom_image(owner,slug,logs)
+      response = Cnvrg::API.request("users/#{owner}/images/#{slug}/commit_custom_image", 'POST', {image_logs:logs})
+      return response
+    end
+
+
+    def create_custom_image(new_image_name,working_dir,stored_commands)
+
+      python2_arr = get_installed_packages("python2")
+      py2 = python2_arr.join(",") unless python2_arr.nil? or python2_arr.empty?
+      python3_arr = get_installed_packages("python3")
+      py3 = python3_arr.join(",") unless python3_arr.nil? or python3_arr.empty?
+      system_arr = get_installed_packages("system")
+      sys = system_arr.join(",") unless system_arr.nil? or system_arr.empty?
+
+      response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/images/push", 'POST', {image_slug: @image_slug, py2: py2,py3:py3,
+                                                                                             dpkg: sys, new_image_name: new_image_name,
+                                                                                             run_commands:stored_commands})
+      if Cnvrg::CLI.is_response_success(response) and !response["result"]["slug"].nil?
+        container = get_container()
+        name = response["result"]["name"]
+        container = get_container()
+        container.commit({repo:name,tag:"latest"})
+        update_image(name+":latest", container, response["result"]["slug"])
+        File.truncate(working_dir+"/.cnvrg/custom_image.txt", 0)
+
+      end
+
+      return true
+
+    end
+
+    def update_image(image_name, container, image_slug)
+      config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
+      @image_name = image_name
+      if !@image_name.nil? and !@image_name.empty?
+        @image_name = image_name[0, image_name.index(":")]
+        @image_tag = image_name[image_name.index(":")+1, image_name.size]
+      end
+      config = {project_name: config[:project_name],
+                project_slug: config[:project_slug],
+                owner: config[:owner],
+                docker: true, image_base: @image_name, image_tag: @image_tag, container: container.id, image_slug: image_slug}
+
+      File.open(@working_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+    end
+
+    def save_installed_libraries(container)
+      begin
+        command = ['/bin/bash', '-lc', '/opt/ds/bin/pip freeze']
+        pip = container.exec(command, tty: true)[0]
+        command = ["/bin/bash", "-lc", "dpkg -l | grep '^ii' | awk '{print $2\"==\"$3}'"]
+        dpkg = container.exec(command, tty: true)[0]
+        File.open(@working_dir+"/.cnvrg/pip_base.txt", "w+") { |f| f.write pip }
+        File.open(@working_dir+"/.cnvrg/dpkg_base.txt", "w+") { |f| f.write dpkg }
+      rescue => e
+      end
+
+
+    end
+
+    def remote_notebook(notebook_path, instance_type, kernel,data,data_commit)
+      response = Cnvrg::API.request("users/#{@owner}/images/#{@image_slug}/remote_notebook", 'POST', {dir: notebook_path,
+                                                                                                      project_slug: @project_slug,
+                                                                                                      instance_type: instance_type,
+                                                                                                      kernel: kernel,
+                                                                                                      dataset_slug:data,
+                                                                                                      dataset_commit: data_commit})
+      return response
+    end
+
+    def get_installed_packages(repo)
+      container = get_container()
+      case repo
+        when "python2"
+          command = ['/bin/bash', '-lc', '/opt/ds/bin/pip freeze']
+        when "python3"
+          command = ['/bin/bash', '-lc', '/opt/ds3/bin/pip3 freeze']
+        when "system"
+          command = ["/bin/bash", "-lc", "dpkg -l | grep '^ii' | awk '{print $2\"==\"$3}'"]
+      end
+
+      libs = container.exec(command, tty: true)[0]
+      libs_arr = libs.join("").split("\r\n")
+      return libs_arr
+
+    end
+
+    def get_bash_history
+      container = get_container()
+      command = ["/bin/bash", "-lc", "cat /home/ds/.bash_history"]
+      history = container.exec(command, tty: true)[0][0]
+      if history.include? "No such file"
+        history = ""
+      end
+      return history
+    end
+
+
+    def get_image_state
+      python_arr = self.get_installed_packages("python")
+      py = python_arr.join(",") unless python_arr.nil? or python_arr.empty?
+      system_arr = self.get_installed_packages("system")
+      sys = system_arr.join(",") unless system_arr.nil? or system_arr.empty?
+      # bash_history = self.get_bash_history
+      diff = [py, sys]
+
+    end
+
+    def find_image(update=true)
+      config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
+      image_slug = config[:image_slug]
+      if image_slug.nil? or image_slug.empty?
+        image_res = Cnvrg::API.request("users/#{@owner}/images/" + "find", 'POST', {image_name: @image_name})
+
+        if Cnvrg::CLI.is_response_success(image_res)
+          image_slug = image_res["result"]["image"]["slug"]
+          update_slug(image_slug) unless !update
+          return image_slug
+        end
+      else
+        return image_slug
+
+      end
+    end
+
+    def set_note_url(note_slug)
+      config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
+      config[:notebook_slug] = note_slug
+      File.open(@working_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+    end
+    def note_slug
+      config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
+      note_slug = config[:notebook_slug]
+      if note_slug.nil? or note_slug.empty?
+        return false
+      else
+        return note_slug
+      end
+    end
+    def remove_note_slug
+      config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
+      config[:notebook_slug] = ""
+      File.open(@working_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+
+    end
+
+
+    def update_slug(slug)
+      config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
+      config[:image_slug] = slug
+      File.open(@working_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+    end
+
+    def new_machine(instance_type)
+      title = "#{instance_type} auto created by cli"
+      response = Cnvrg::API.request("users/#{@owner}/machines/new", 'POST', {machine_name: title, instance_type: instance_type})
+      return Cnvrg::CLI.is_response_success(response)
+
+    end
+
+    def update_image_activity(commit, exp_slug)
+      response = Cnvrg::API.request("users/#{@owner}/images/#{@image_slug}/update_activity", 'POST', {commit: commit, project_slug: @project_slug, experiment: exp_slug})
+      return Cnvrg::CLI.is_response_success(response)
+    end
+
+    def handle_image_activity
+      home_dir = File.expand_path('~')
+      zip_dir = "#{home_dir}/.cnvrg/tmp/config.zip"
+      compress = `zip -j #{zip_dir} #{home_dir}/.netrc #{home_dir}/.cnvrg/config.yml`
+      @files = Cnvrg::Files.new(@owner, @project_slug)
+      res_id = @files.upload_exec_file(zip_dir, @image_name, @commit_id)
+      FileUtils.remove zip_dir
+      return res_id
+    end
+
+
+  end
+
+end

data/lib/cnvrg/api.rb

@@ -0,0 +1,247 @@
+require 'netrc'
+require 'faraday'
+require 'json'
+require 'fileutils'
+require 'cnvrg/helpers'
+require 'logger'
+
+module Cnvrg
+    class API
+        USER_AGENT       = "CnvrgCLI/#{Cnvrg::VERSION}"
+        ENDPOINT_VERSION = 'v1'
+
+        def self.get_api
+            home_dir = File.expand_path('~')
+            config = ""
+            begin
+                if File.exist? home_dir+"/.cnvrg/config.yml"
+                    config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
+                else
+                    return "https://app.cnvrg.io/api"
+                end
+
+            rescue
+                return "https://app.cnvrg.io/api"
+            end
+            if !config or config.empty? or config.to_h[:api].nil?
+                return "https://app.cnvrg.io/api"
+            else
+              return config.to_h[:api]
+            end
+        end
+        def self.request(resource, method = 'GET', data = {}, parse_request = true)
+            resource = URI::encode resource
+            begin
+                n = Netrc.read
+            rescue => e
+            end
+
+            # Make sure there is an entry for the Acquia API before generating the
+            # requests.
+            if n['cnvrg.io'].nil?
+                puts 'You\'re not logged in'
+                puts 'Please log in via `cnvrg login`'
+                return
+            end
+
+            @user, @pass = n[Cnvrg::Helpers.netrc_domain]
+            begin
+                if !Helpers.is_verify_ssl
+
+                    conn = Faraday.new "#{endpoint_uri}", :ssl => {:verify => false}
+                else
+                    conn = Faraday.new "#{endpoint_uri}"
+                end
+            conn.headers['Auth-Token'] = @pass
+            conn.headers['User-Agent'] = "#{Cnvrg::API::USER_AGENT}"
+            conn.options.timeout = 420
+                conn.options.open_timeout=180
+            case method
+              when 'GET'
+                retries = 0
+                success = false
+                while !success and retries < 20
+                  begin
+                    response = conn.get "#{resource}", data
+                    success = true
+                    Cnvrg::API.parse_version(response)
+                  rescue => e
+                    Cnvrg::Logger.log_error(e)
+                    sleep(5)
+                    retries +=1
+                  end
+                end
+                if !success
+                  return false
+                end
+                if response.to_hash[:status] == 404
+                  return false
+                end
+                if parse_request
+                    JSON.parse(response.body)
+                else
+                    response
+                end
+            when 'POST', 'PUT'
+                conn.options.timeout = 4200
+                conn.options.open_timeout = 180
+                conn.headers['Content-Type'] = "application/json"
+                retries = 0
+                success = false
+                data = data || {}
+                while !success and retries < 20
+                    begin
+                        response = conn.post "#{resource}", data.to_json if method.eql? 'POST'
+                        response = conn.put "#{resource}", data.to_json if method.eql? 'PUT'
+                        success = true
+                        Cnvrg::API.parse_version(response)
+
+                    rescue => e
+                      Cnvrg::Logger.log_error(e)
+                        sleep(5)
+                        retries +=1
+                    end
+                end
+                if !success
+                    return false
+                end
+                if response.to_hash[:status] == 404
+                  return false
+                end
+                if parse_request == true
+                    JSON.parse(response.body)
+                else
+                    response
+                end
+            when 'POST_JSON'
+              conn.options.timeout = 4200
+              conn.options.open_timeout = 4200
+              conn.headers['Content-Type'] = "application/json"
+              new_data = JSON.dump(data)
+
+              retries = 0
+              success = false
+
+              while !success and retries < 20
+                begin
+                  response = conn.post "#{resource}", new_data
+                  success = true
+                rescue => e
+                  Cnvrg::Logger.log_error(e)
+                  sleep(5)
+                  retries +=1
+                end
+              end
+              if !success
+                return false
+              end
+              if response.to_hash[:status] == 404
+                return false
+              end
+              if parse_request == true
+                JSON.parse(response.body)
+              else
+                response
+              end
+            when 'POST_FILE'
+                conn = Faraday.new do |fr|
+                    fr.headers['Auth-Token'] = @pass
+                    fr.headers['User-Agent'] = "#{Cnvrg::API::USER_AGENT}"
+                    fr.headers["Content-Type"] = "multipart/form-data"
+                    if !Helpers.is_verify_ssl
+	                      fr.ssl.verify = false
+                    end
+
+
+                    fr.request :multipart
+                    fr.request :url_encoded
+                    fr.request :retry, max: 2, interval: 0.05,interval_randomness: 0.5, backoff_factor: 2
+                    fr.adapter :net_http
+                end
+                conn.options.timeout = 4200
+                conn.options.open_timeout =4200
+
+
+                # what if windows?
+                # data[:file] = Faraday::UploadIO.new(data[:absolute_path], content_type)
+                if not File.exists? data[:relative_path]
+                  file_base = File.basename(data[:relative_path])
+
+                  begin
+                      temp_path = File.expand_path('~')+"/.cnvrg/tmp_files/#{file_base}"
+                      FileUtils.touch(temp_path)
+                  rescue
+                      temp_path ="/tmp/#{file_base}"
+                      FileUtils.touch(temp_path)
+                  end
+                else
+                  temp_path = data[:relative_path]
+                end
+
+
+                data[:file] = Faraday::UploadIO.new("#{temp_path}", "application/tar+gzip")
+
+                response = conn.post "#{endpoint_uri}/#{resource}", data
+                Cnvrg::API.parse_version(response)
+                FileUtils.rm_rf(temp_path)
+                if response.to_hash[:status] == 404
+                  return false
+                end
+
+
+                if parse_request == true
+                    JSON.parse(response.body)
+                else
+                    response
+                end
+            when 'DELETE'
+                response = conn.delete "#{endpoint_uri}/#{resource}", data
+                Cnvrg::API.parse_version(response)
+                if response.to_hash[:status] == 404
+                  return false
+                end
+                if parse_request == true
+                    JSON.parse(response.body)
+                else
+                    response
+                end
+            else
+            end
+            rescue => e
+              Cnvrg::Logger.log_error(e)
+               return nil
+            rescue SignalException
+                return false
+
+            end
+
+        end
+
+        def self.endpoint_uri
+            api = get_api()
+            return "#{api}/#{Cnvrg::API::ENDPOINT_VERSION}"
+        end
+
+
+        def self.display_error(response)
+            "Oops, an error occurred! Reason: #{response['message']}"
+        end
+
+      def self.parse_version(resp)
+        begin
+          version = resp.headers["cnvrg-version"]
+          Cnvrg::Helpers.update_version(version)
+        rescue => e
+          Cnvrg::Logger.log_error(e)
+        end
+      end
+
+        # Internal: Ensure the response returns a HTTP 200.
+        #
+        # If the response status isn't a HTTP 200, we need to find out why. This
+        # helps identify the issues earlier on and will prevent extra API calls
+        # that won't complete.
+        #
+        # Returns false if the response code isn't a HTTP 200.
+    end
+end