cnvrg 1.9.9.9.7

data/lib/cnvrg/flow.rb

@@ -0,0 +1,137 @@
+module Cnvrg
+  class Flows
+    def initialize(flow_slug, project: nil)
+      @project = project || Cnvrg::Project.new(Cnvrg::CLI.get_project_home)
+      @flow_info= Flows.resolve_flow_title(flow_slug, project)
+      @slug = @flow_info["slug"]
+      @tasks = {}
+      @relations = {}
+      @title = nil
+      @base_resource = @project.base_resource + "flows/#{@slug}"
+      @public_url = "#{@project.url}/flows/#{@slug}"
+      # self.reload_flow
+    end
+
+    def self.resolve_flow_title(title, project)
+      resp = Cnvrg::API.request("#{project.base_resource}/flows", 'GET')
+      if resp.blank?
+        raise StandardError.new("Can't resolve flow")
+      end
+      res = resp["result"].find{|flow| flow["slug"].downcase == title.downcase}
+      res ||= resp["result"].find{|flow| flow["title"].downcase == title.downcase}
+      if res.blank?
+        raise StandardError.new("Can't find flow with title #{title}")
+      end
+      res
+    end
+
+    def edit_href
+      "#{@public_url}/flow_versions/new"
+    end
+
+    def edit_version_href(version)
+      return "#{edit_href}?flow_version_slug=#{version}"
+    end
+
+    def version_href(version=nil)
+      "#{@base_resource}/flow_versions/#{version || 'latest'}"
+    end
+
+    def export(version, file: nil)
+      resp = Cnvrg::API.request(version_href(version), 'GET')
+      if resp["status"] != 200
+        raise StandardError.new("Cant find flow version: #{version} for flow: #{@slug}")
+      end
+      flow_version = resp["flow_version"]
+      api_recipe = flow_version["api_recipe"]
+      file = file.presence  || "flow-#{@slug.downcase.gsub("\s", "_")}.yml"
+      File.open(file, "w"){|f| f.write api_recipe.to_yaml}
+      file
+    end
+
+    def get_version(version)
+
+
+    end
+
+    def self.create_flow(project, recipe, run: false)
+      url = "#{project.base_resource}flows"
+      if run
+        url += "/run"
+      end
+      resp = Cnvrg::API.request(url, 'POST', {flow_version: recipe.to_json}) || {}
+      if resp["status"] == 200
+        return [Flows.new(resp["flow_version"]["flow_id"], project: project), resp["flow_version"]["id"]]
+      elsif resp["status"] == 400
+        raise StandardError.new(resp["message"])
+      end
+      raise StandardError.new("Can't create new flow")
+    end
+
+    def get_flow
+      unless File.exists? @fullpath
+        raise StandardError.new("Cant find flow in #{@fullpath}")
+      end
+      YAML.load_file(@fullpath)
+    end
+
+    def set_flow(new_flow)
+      File.open(@fullpath, "w"){|file| file.write new_flow.to_yaml}
+    end
+
+    def set_flow_slug(slug)
+      flow = self.get_flow
+      flow[:slug] = slug
+      self.set_flow(flow)
+    end
+
+    def reload_flow
+      flow = self.get_flow
+      @title = flow[:title]
+      @slug = flow[:slug]
+      @relations = flow[:relations]
+      local_tasks = flow[:tasks] || {}
+      @relations.each do |relation|
+        relation.values.each do |task|
+          if local_tasks[task].present?
+            @tasks[task] = Cnvrg::Task.new(@project.local_path, content: local_tasks[task])
+          else
+            @tasks[task] = Cnvrg::Task.new(@project.local_path, path: task)
+          end
+        end
+      end
+    end
+
+
+    def run
+      resp = Cnvrg::API.request("#{@base_resource}/#{@slug}/run", 'POST')
+      if Cnvrg::CLI.is_response_success(resp)
+        return resp
+      end
+      Cnvrg::CLI.log_message("Cant run flow #{@slug}")
+    end
+
+    ### in use for yaml file
+    # def run
+    #   resp = Cnvrg::API.request(@base_resource, 'POST', {data: to_api})
+    #   Cnvrg::CLI.is_response_success(resp, true)
+    #   flow_slug = resp['result']['flow']
+    #   self.set_flow_slug(flow_slug)
+    #   url = Cnvrg::Helpers.remote_url + resp['result']['url']
+    #   return url
+    # end
+
+
+    private
+    def to_api
+      {
+          relations: @relations,
+          tasks: @tasks.keys.map{|task| [task, @tasks[task].to_api]}.to_h,
+          title: @title,
+          slug: @slug
+      }
+    end
+
+
+  end
+end

data/lib/cnvrg/helpers.rb

@@ -0,0 +1,422 @@
+module Cnvrg
+  module Helpers
+
+    extend self
+    def parallel_threads()
+      threads = ENV["CNVRG_PARALLEL_THREADS"].to_i
+
+      
+      ### if empty, default will be 15 threads
+      threads = threads > 0 ? threads : 15
+
+      ### set max threads to be 100k
+      [threads, 100000].min
+    end
+
+    def self.parallel_options
+      {
+          in_processes: Cnvrg::CLI::ParallelProcesses,
+          in_thread: Cnvrg::CLI::ParallelThreads,
+          isolation: true
+      }
+    end
+    def checkmark
+      checkmark = "\u2713"
+      return checkmark.encode('utf-8')
+    end
+
+    def internet_connection?
+      begin
+        true if open("http://www.google.com/")
+      rescue
+        false
+      end
+    end
+
+    def try_until_success(tries: 3)
+      exception = nil
+      tries.times do |i|
+        begin
+          yield
+          return true
+        rescue => e
+          Cnvrg::Logger.log_info("Error while trying for the #{i} time")
+          Cnvrg::Logger.log_error(e)
+          sleep(1)
+          exception = e
+        end
+      end
+      raise exception
+    end
+
+    def get_config
+      home_dir = File.expand_path('~')
+      config = {}
+      begin
+        if File.exist? home_dir+"/.cnvrg/config.yml"
+          config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
+        end
+      end
+      return config
+    end
+
+    def set_config(config)
+      home_dir = File.expand_path('~')
+      File.open("#{home_dir}/.cnvrg/config.yml", "w"){|f| f.write config.to_yaml }
+      return config
+    end
+
+
+    def remote_url
+      home_dir = File.expand_path('~')
+      config = ""
+      begin
+        if File.exist? home_dir+"/.cnvrg/config.yml"
+          config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
+        else
+          return "https://app.cnvrg.io"
+        end
+
+      rescue
+        return "https://app.cnvrg.io"
+      end
+      if !config or config.empty? or config.to_h[:api].nil?
+        return "https://app.cnvrg.io"
+      else
+        return config.to_h[:api].gsub("/api", "")
+      end
+    end
+
+    def server_version
+      config = self.get_config
+      config[:version].try(:to_i) || 0
+    end
+
+    def update_version(version)
+      config = self.get_config
+      if config[:version].to_s.eql? version
+        return
+      end
+      config[:version] = version
+      self.set_config(config)
+    end
+
+    def is_verify_ssl
+      home_dir = File.expand_path('~')
+      config = ""
+      begin
+        if File.exist? home_dir+"/.cnvrg/config.yml"
+          config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
+        else
+         return true
+
+        end
+
+      rescue
+        return true
+      end
+      if !config or config.empty? or config.to_h[:verify_ssl].nil?
+        return true
+      else
+        return config.to_h[:verify_ssl]
+      end
+    end
+
+    def os
+
+      if windows?
+        return "windows"
+      elsif mac?
+        return "mac"
+      elsif ubuntu?
+        return "ubuntu"
+      elsif linux?
+
+        return "linux"
+      else
+
+        return "N/A"
+      end
+    end
+
+    def windows?
+      !!(RUBY_PLATFORM =~ /mswin32|mingw32/)
+    end
+
+    def mac?
+      !!(RUBY_PLATFORM =~ /-darwin\d/)
+    end
+
+    def linux?
+      not mac? and not windows?
+    end
+
+    def ubuntu?
+      unix = `if [ -f  /etc/lsb-release ];  then echo "ubuntu"; fi`
+      return unix.include? "ubuntu"
+    end
+
+    def cnvrgignore_content
+      #TODO: cnvrg ignore add .conflict
+      %{
+# cnvrg ignore: Ignore the following directories and files
+# for example:
+# some_dir/
+# some_file.txt
+.git*
+.gitignore
+*.conflict
+*.deleted
+            }.strip
+    end
+
+    def hyper_content
+      %{# Hyperparameter Optimization is the process of choosing a set of parameters for a learning algorithm, usually with the goal of optimizing a measure of the algorithm's performance on an independent data set.
+
+# Below is the list of parameters that will be used in the optimization process. Each parameter has a param_name that should match the argument that is feeded to the experiment s.t kernel => --kernel='rbf'
+
+parameters:
+  # Integer parameter is a range of possible values between a minimum (inclusive)
+  # and maximum (not inclusive) values. Values are floored (0.7 => 0)
+    - param_name: "learning_rate"
+      type: "integer" 
+      min: 0 # inclusive
+      max: 10 # not inclusive
+      scale: "linear"
+      steps: 4 # The number of linear steps to produce.
+
+
+    # Float parameter is a range of possible values between a minimum (inclusive)
+    # and maximum (not inclusive) values.
+    #
+    - param_name: "learning_rate"
+      type: "float" # precision is 9 after period
+      min: 0.00001
+      max: 0.1
+      scale: "log2" # Could be log10 as well
+      steps: 2
+
+    # Discrete parameter is an array of numerical values.
+    #
+    - param_name: "c"
+      type: "discrete"
+      values: [0, 0.1 ,0.001]
+
+    # Categorical parameter is an array of string values
+    #
+    - param_name: "kernel"
+      type: "categorical"
+      values: ["linear", "poly", "rbf"] 
+    
+}
+    end
+
+    def readme_content
+      %{
+            # README
+
+            This README would normally contain some context and description about the project. 
+
+            Things you may want to cover:
+
+            * Data description
+
+            * Benchmark and measurement guidelines
+
+            * Used algorithms
+
+            * Scores
+
+            * Configurations
+
+            * Requirements
+
+            * How to run the experiments
+
+            * ...}.strip
+    end
+
+    def netrc_domain
+      "cnvrg.io"
+    end
+
+    def look_for_in_path(path, name)
+      url_split = path.split("/")
+      url_split.each_with_index do |u, i|
+        if u == name
+          return i
+        end
+      end
+      return -1
+    end
+
+    def extract_owner_slug_from_url(url, breaker)
+      url_parts = url.split("/")
+      project_index = Cnvrg::Helpers.look_for_in_path(url, breaker)
+      slug = url_parts[project_index + 1]
+      owner = url_parts[project_index - 1]
+      return owner, slug
+    end
+
+    # cpu
+
+    def cpu_time
+      Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :microsecond)
+    end
+
+    def wall_time
+      Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond)
+    end
+
+    def decrypt(key,iv,str)
+      begin
+
+      cipher = OpenSSL::Cipher.new("aes-256-cbc").decrypt
+      cipher.key = key
+      cipher.iv = Base64.decode64 iv.encode('utf-8')
+
+      result = Base64.decode64 (str.encode('utf-8'))
+      result = cipher.update(result)
+      result << cipher.final
+      return result.force_encoding('utf-8')
+
+      # return result
+      rescue => e
+        puts e
+
+
+      end
+
+
+    end
+
+    # memory
+    #
+    def get_mem(pid)
+    end
+
+    def get_s3_props(files) #will return client and decryptor
+      sts_path = files["path_sts"]
+      retries = 0
+      success= false
+      while !success and retries < 20
+        begin
+          if !Helpers.is_verify_ssl
+            body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
+          else
+            body = open(sts_path).read
+          end
+          success = true
+        rescue => e
+          retries +=1
+          sleep(5)
+
+        end
+      end
+      if !success
+        return Cnvrg::Result.new(false,"couldn't download some files", "error in sts", "" )
+      end
+      split = body.split("\n")
+      key = split[0]
+      iv = split[1]
+
+      access =  Cnvrg::Helpers.decrypt(key, iv, files["sts_a"])
+
+      secret =  Cnvrg::Helpers.decrypt(key,iv, files["sts_s"])
+
+      session =  Cnvrg::Helpers.decrypt(key,iv, files["sts_st"])
+      region =  Cnvrg::Helpers.decrypt(key,iv, files["region"])
+
+      bucket =  Cnvrg::Helpers.decrypt(key,iv, files["bucket"])
+      is_s3 = files["is_s3"]
+      server_side_encryption =files["server_side_encryption"]
+
+      if is_s3 or is_s3.nil?
+        client = Aws::S3::Client.new(
+            :access_key_id =>access,
+            :secret_access_key => secret,
+            :session_token => session,
+            :region => region,
+            :http_open_timeout => 60, :retry_limit => 20)
+        use_accelerate_endpoint = true
+      else
+
+        endpoint = Cnvrg::Helpers.decrypt(key,iv, files["endpoint"])
+        client = Aws::S3::Client.new(
+            :access_key_id =>access,
+            :secret_access_key => secret,
+            :region => region,
+            :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
+            :http_open_timeout => 60, :retry_limit => 20)
+        use_accelerate_endpoint = false
+      end
+
+      if !server_side_encryption
+        upload_options = {:use_accelerate_endpoint => use_accelerate_endpoint}
+      else
+        upload_options = {:use_accelerate_endpoint => use_accelerate_endpoint, :server_side_encryption => server_side_encryption}
+      end
+      return {client: client, key: key, iv: iv, bucket: bucket, upload_options: upload_options}
+    end
+
+    def get_experiment_events_log_from_server(exp, project, commit: nil)
+      dest_dir = exp["slug"]
+      commit = commit || exp["end_commit"]
+      response = project.clone(0, commit)
+      Cnvrg::CLI.is_response_success(response, should_exit=false)
+      commit_sha1 = response["result"]["commit"]
+      files = response["result"]["tree"].keys
+      files = files.select do |f| f.include?("tfevents") end
+      @files = Cnvrg::Files.new(project.owner, project.slug, project_home: "", project: project)
+      @files.download_files(files, commit_sha1, progress: nil)
+      FileUtils.rm_rf("#{dest_dir}")
+      FileUtils.mkdir_p(dest_dir)
+      files.each do |f|
+        file_dir = "#{dest_dir}/#{File.dirname(f)}"
+        FileUtils.mkdir_p(file_dir)
+        FileUtils.mv(f, "#{dest_dir}/#{f}")
+      end
+    end
+
+    def get_experiment_events_log_via_kubectl(exp, namespace)
+      dest_dir = exp["slug"]
+      result = `kubectl -n #{namespace} get pods | grep #{exp["slug"]}`
+
+      pod_name = result.split(" ")[0]
+      return false if pod_name.blank?
+      FileUtils.mkdir_p(dest_dir)
+      working_dir = `kubectl -n #{namespace} exec #{pod_name} -c agent -- pwd`
+      working_dir.strip!
+      res = `kubectl -n #{namespace} exec #{pod_name} -c agent -- /bin/bash -c "ls -R #{working_dir}"`
+      files_and_folders = res.split("\n\n")
+      all_files = []
+
+      files_and_folders.each do |file_and_folder|
+        files = file_and_folder.split("\n")
+        if files.first.include?(":")
+          folder = files.first.gsub(":", "")
+
+          folder = folder.sub(working_dir + "/", "")
+          files = files.drop(1)
+        end
+        files.each do |file|
+          if file.include?("tfevents")
+            all_files << "#{folder}/#{file}"
+          end
+        end
+      end
+
+      all_files.each do |file|
+        file_dir = "#{dest_dir}/#{File.dirname(file)}"
+        FileUtils.mkdir_p(file_dir)
+        res = `kubectl -n #{namespace} cp #{pod_name}:#{file} -c agent #{dest_dir}/#{file}`
+      end
+
+      return true
+    rescue => e
+      Cnvrg::Logger.log_error(e)
+      return false
+    end
+  end
+
+end