cnvrg 0.0.14 → 0.0.15

data/lib/cnvrg/helpers.rb

@@ -72,7 +72,7 @@ module Cnvrg
 
     def cnvrgignore_content
       %{
-               # cnvrg ignore: Ignore the following directories and files
+# cnvrg ignore: Ignore the following directories and files
 # for example:
 # some_dir/
 # some_file.txt
@@ -81,6 +81,47 @@ module Cnvrg
             }.strip
     end
 
+    def hyper_content
+      %{# Hyperparameter Optimization is the process of choosing a set of parameters for a learning algorithm, usually with the goal of optimizing a measure of the algorithm's performance on an independent data set.
+
+# Below is the list of parameters that will be used in the optimization process. Each parameter has a param_name that should match the argument that is feeded to the experiment s.t kernel => --kernel='rbf'
+
+parameters:
+  # Integer parameter is a range of possible values between a minimum (inclusive)
+  # and maximum (not inclusive) values. Values are floored (0.7 => 0)
+    - param_name: "learning_rate"
+      type: "integer" 
+      min: 0 # inclusive
+      max: 10 # not inclusive
+      scale: "linear"
+      steps: 4 # The number of linear steps to produce.
+
+
+    # Float parameter is a range of possible values between a minimum (inclusive)
+    # and maximum (not inclusive) values.
+    #
+    - param_name: "learning_rate"
+      type: "float" # precision is 9 after period
+      min: 0.00001
+      max: 0.1
+      scale: "log2" # Could be log10 as well
+      steps: 2
+
+    # Discrete parameter is an array of numerical values.
+    #
+    - param_name: "c"
+      type: "discrete"
+      values: [0, 0.1 ,0.001]
+
+    # Categorical parameter is an array of string values
+    #
+    - param_name: "kernel"
+      type: "categorical"
+      values: ["linear", "poly", "rbf"] 
+    
+}
+    end
+
     def readme_content
       %{
             # README

data/lib/cnvrg/job.rb

@@ -31,7 +31,6 @@ class LogJob
                 upload_file(file_path)
             return resp
         rescue =>e
-            puts e
             return false
 
         end

data/lib/cnvrg/project.rb

@@ -1,7 +1,7 @@
 require 'fileutils'
 module Cnvrg
   class Project
-    attr_reader :slug, :owner, :title, :local_path
+    attr_reader :slug, :owner, :title, :local_path, :working_dir
 
     RemoteURL ||= "https://cnvrg.io"
 
@@ -12,6 +12,7 @@ module Cnvrg
         @title = config[:project_name]
         @slug = config[:project_slug]
         @owner = config[:owner]
+        @working_dir = project_home
       rescue => e
       end
 
@@ -23,18 +24,21 @@ module Cnvrg
       return idx[:commit]
     end
 
+
     def url
       url = Cnvrg::Helpers.remote_url
       "#{url}/#{self.owner}/projects/#{self.slug}"
     end
 
     def update_ignore_list(new_ignore)
-      if new_ignore.empty?
+      if new_ignore.nil? or new_ignore.empty?
         return true
       end
+      list = new_ignore.split(",")
       begin
         File.open(self.local_path+"/.cnvrgignore", "a+") do |f|
-          new_ignore.each do |i|
+          f.puts("\n")
+          list.each do |i|
             f.puts("#{i}\n")
           end
         end
@@ -48,13 +52,18 @@ module Cnvrg
       ignore_list = []
       File.open(self.local_path+"/.cnvrgignore", "r").each_line do |line|
         line = line.strip
-        if line.start_with? "#"
+        if line.start_with? "#" or ignore_list.include? line or line.empty?
           next
         end
-        if line.end_with? "/"
-          ignore_list << line.chop
-          sub_dirs = Dir.glob("#{line}/**/*").each { |x| x.gsub!("//", "/") }
-          ignore_list << sub_dirs.flatten
+        if line.end_with? "/" or File.directory?(line)
+          if line.end_with? "/"
+            ignore_list << line.chop
+          else
+            ignore_list << line
+          end
+            all_sub = Dir.glob("#{line}/**/*", File::FNM_DOTMATCH).flatten
+
+          ignore_list << all_sub.flatten
         else
           ignore_list << line
         end
@@ -71,7 +80,6 @@ module Cnvrg
       else
 
         list_dirs = [project_name,
-                     project_name + "/data",
                      project_name + "/models",
                      project_name + "/notebooks",
                      project_name + "/src",
@@ -89,6 +97,7 @@ module Cnvrg
       ]
       cnvrgreadme = Helpers.readme_content
       cnvrgignore = Helpers.cnvrgignore_content
+      cnvrghyper  = Helpers.hyper_content
 
       begin
 
@@ -108,6 +117,8 @@ module Cnvrg
         File.open(project_name + "/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
         File.open(project_name + "/.cnvrgignore", "w+") { |f| f.write cnvrgignore }
         File.open(project_name + "/README.md", "w+") { |f| f.write cnvrgreadme }
+        File.open(project_name + "/src/hyper.yaml", "w+") { |f| f.write cnvrghyper }
+        
       rescue
         return false
       end
@@ -136,14 +147,13 @@ module Cnvrg
           FileUtils.mkdir_p list_dirs
           FileUtils.touch list_files
           File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
-          File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore }
+          File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
           if !File.exist? "README" and !File.exist? "README.md"
             FileUtils.touch [ "README.md" ]
             File.open("README.md", "w+") { |f| f.write cnvrgreadme }
           end
 
         rescue => e
-          puts e
           return false
         end
         return true
@@ -205,6 +215,26 @@ module Cnvrg
         end
         return true
       end
+    def update_is_new_branch(new_branch)
+      config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
+      config[:new_branch] = new_branch
+      File.open(@working_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+    end
+    def get_new_branch
+      begin
+      config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
+      return config[:new_branch]
+      rescue =>e
+        return false
+
+      end
+    end
+
+    def remove_new_branch
+      config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
+      new_config = config.except(:new_branch)
+      File.open(@working_dir+"/.cnvrg/config.yml", "w+") { |f| f.write new_config.to_yaml }
+    end
 
       def generate_idx
         if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
@@ -219,15 +249,18 @@ module Cnvrg
         list_ignore = self.get_ignore_list()
         list.each do |e|
           label = e.gsub(self.local_path + "/", "")
+
           if File.directory? e
             if list_ignore.include? label
               next
             end
+
             tree_idx[label+"/"] = nil
           else
             if list_ignore.include? label
               next
             end
+
             sha1 = Digest::SHA1.file(e).hexdigest
             if old_idx.nil? or old_idx.to_h[:tree].nil?
               tree_idx[label] = {sha1: sha1, commit_time: nil}
@@ -258,7 +291,7 @@ module Cnvrg
 
         local_idx = self.generate_idx
         response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/status", 'POST', {idx: local_idx, new_branch: new_branch, current_commit: commit})
-        CLI.is_response_success(response)
+        CLI.is_response_success(response,false)
         return response
       end
 
@@ -267,7 +300,8 @@ module Cnvrg
           commit = last_local_commit
         end
         response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commit/compare", 'POST', {current_commit: commit})
-        CLI.is_response_success(response)
+        CLI.is_response_success(response,false)
+        update_is_new_branch(response["result"]["new_branch"])
         return response["result"]["new_branch"]
       end
 
@@ -282,16 +316,22 @@ module Cnvrg
         return true
       end
 
-      def deploy(file_to_run, function, input_params, commit_to_run, instance_type, image_slug, scheduling_query, local_timestamp)
+      def deploy(file_to_run, function, input_params, commit_to_run, instance_type, image_slug, scheduling_query, local_timestamp,workers, file_input)
         response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/deploy", 'POST', {file_to_run: file_to_run, function: function,
                                                                                            image_slug: image_slug, input_params: input_params,
                                                                                            commit_sha1: commit_to_run,
                                                                                            instance_type: instance_type,
                                                                                            scheduling_query: scheduling_query,
-                                                                                           local_timestamp: local_timestamp})
+                                                                                           local_timestamp: local_timestamp,
+                                                                                            workers:workers,file_input:file_input})
         return response
       end
+    def list_commits
+      response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commits/list", 'GET')
+      CLI.is_response_success(response)
+      return response
 
+    end
       def update_idx_with_commit!(commit)
         idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
         idx_hash[:commit] = commit

data/lib/cnvrg/ssh.rb

@@ -0,0 +1,95 @@
+require 'fileutils'
+require 'cnvrg/files'
+require 'docker'
+require 'net/ssh'
+
+
+module Cnvrg
+  class Ssh
+    attr_reader :is_ssh
+
+
+    def initialize(resp)
+      begin
+        @is_ssh = false
+        sts_path = resp["result"]["sts_path"]
+
+        uri = URI.parse(sts_path)
+
+        http_object = Net::HTTP.new(uri.host, uri.port)
+        http_object.use_ssl = true if uri.scheme == 'https'
+        request = Net::HTTP::Get.new(sts_path)
+
+        body = ""
+        http_object.start do |http|
+          response = http.request request
+          body = response.read_body
+        end
+
+        URLcrypt::key = [body].pack('H*')
+        ip = URLcrypt.decrypt(resp["result"]["machine_i"])
+
+        @user = URLcrypt.decrypt(resp["result"]["machine_u"])
+        key = URLcrypt.decrypt(resp["result"]["machine_k"])
+        @container = URLcrypt.decrypt(resp["result"]["machine_c"])
+
+        tempssh = Tempfile.new "sshkey"
+        tempssh.write open(key).read
+        tempssh.rewind
+        key_path = tempssh.path
+        count = 0
+        while count < 5
+
+          begin
+            @ssh = Net::SSH.start(ip, user=@user, :keys => key_path, :timeout => 10)
+            if !@ssh.nil?
+              @is_ssh = true
+              return
+            else
+              count+=1
+              sleep(2)
+
+            end
+          rescue
+            count+=1
+            sleep(2)
+
+
+          end
+        end
+        if tempssh
+          tempssh.close
+          tempssh.unlink
+        end
+        return false
+      rescue => e
+
+        puts e
+      end
+    end
+
+
+
+    def exec_command(command)
+      exec_command = "sudo -i -u #{@user} cnvrg exec_container #{@container} \"#{command}\" "
+      return @ssh.exec!(exec_command)
+    end
+
+
+    def close_ssh()
+
+
+      begin
+
+        @ssh.close
+      rescue => e
+
+      end
+
+
+    end
+  end
+
+
+
+end

data/lib/cnvrg/version.rb

@@ -1,4 +1,5 @@
 module Cnvrg
-    VERSION = '0.0.14'
+    VERSION = '0.0.15'
+
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.0.14
+  version: 0.0.15
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-14 00:00:00.000000000 Z
+date: 2017-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -249,46 +249,74 @@ dependencies:
   name: logstash-logger
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.22.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.22.1
 - !ruby/object:Gem::Dependency
   name: launchy
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.4'
 - !ruby/object:Gem::Dependency
   name: docker-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.33'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.33'
 - !ruby/object:Gem::Dependency
   name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -302,7 +330,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  name: net-ssh
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -330,12 +358,16 @@ files:
 - lib/cnvrg/api.rb
 - lib/cnvrg/auth.rb
 - lib/cnvrg/cli.rb
+- lib/cnvrg/data.rb
+- lib/cnvrg/datafiles.rb
+- lib/cnvrg/dataset.rb
 - lib/cnvrg/experiment.rb
 - lib/cnvrg/files.rb
 - lib/cnvrg/helpers.rb
 - lib/cnvrg/job.rb
 - lib/cnvrg/project.rb
 - lib/cnvrg/runner.rb
+- lib/cnvrg/ssh.rb
 - lib/cnvrg/version.rb
 homepage: https://cnvrg.io
 licenses: []
@@ -356,9 +388,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.
 test_files: []
-has_rdoc: