cnvrg 0.2 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f693e07f97fdef6be962e7c57f9fd79ef2699748
-  data.tar.gz: cace232784aef2beec2b93efd2599363c1452796
+  metadata.gz: 89388312156199c10f60b2495c05305ea4342a54
+  data.tar.gz: 072a8e7f4bfcba47ec648eb48516f8fc53111541
 SHA512:
-  metadata.gz: 2910879d667310718d23da4f1ff228e46b6705c61e7c1f975316a568d8c9733d61c5e8d07e3f80836b637fc0fcddf10fe2ba3a4f41826e9e843d6af423c9af21
-  data.tar.gz: cdecfa7b57355410235bd8742c67bc0d53640d863cc4e7cdac1b47bf83e6e22b811b6c1b1509cdfc20f595a0ac41ba413b1dafdb4030b08140ff01361f17c442
+  metadata.gz: ed33b00c74cf61f4bfd259fff6656298f6250acd5f7c8668106e0a67974987b101e8d7923d02bd13c98a51d9fd532279f888ce72931abdcb1be336c394a0b1f5
+  data.tar.gz: aeff07a3e9181dbb3a4610206e4852918d9ea24b338ce307a585dea2739bf6260461b9eb64537ec24c4dfac8f1161b8a8b49d5e0376d802152722046f18d5153

data/cnvrg.gemspec

@@ -35,6 +35,7 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'sucker_punch', '~> 2.0'
     spec.add_runtime_dependency 'urlcrypt', '~> 0.1.1'
     spec.add_runtime_dependency 'parallel', '~> 1.12.0'
+    spec.add_runtime_dependency 'net_http_ssl_fix'
     
     spec.add_runtime_dependency 'logstash-logger', '~> 0.22.1'
     spec.add_runtime_dependency 'launchy', '~> 2.4'

data/lib/cnvrg/api.rb

@@ -85,8 +85,14 @@ module Cnvrg
                 # what if windows?
                 # data[:file] = Faraday::UploadIO.new(data[:absolute_path], content_type)
                   file_base = File.basename(data[:relative_path])
+                begin
                     temp_path = File.expand_path('~')+"/.cnvrg/tmp_files/#{file_base}"
                     FileUtils.touch(temp_path)
+                rescue
+                    temp_path ="/tmp/#{file_base}"
+                    FileUtils.touch(temp_path)
+                end
+
 
 
                 data[:file] = Faraday::UploadIO.new("#{temp_path}", "plain/text")

data/lib/cnvrg/cli.rb

@@ -2,7 +2,6 @@
 require "pty"
 require 'etc'
 require 'parallel'
-require "open4"
 require 'netrc'
 require 'net/http'
 require 'uri'
@@ -12,7 +11,6 @@ require 'yaml'
 require 'digest' # sha1up
 require "highline/import"
 require 'socket'
-include Open4
 require 'cnvrg/helpers'
 require 'cnvrg/api'
 require 'cnvrg/auth'
@@ -37,6 +35,7 @@ require 'thor'
 require 'pathname'
 require 'enumerator'
 require 'ruby-progressbar'
+require 'open3'
 
 class Thor
   module Base
@@ -238,7 +237,33 @@ module Cnvrg
         say "Couldn't set default api, contact help@cnvrg.io", Thor::Shell::Color::RED
       end
     end
+    desc '', '', :hide => true
 
+    def set_remote_login(current_user,owner,url, email, secret)
+      netrc = Netrc.read
+      netrc[Cnvrg::Helpers.netrc_domain] = email, secret
+      netrc.save
+      home_dir = File.expand_path('~')
+      if !url.end_with? "/api"
+        url = url+"/api"
+      end
+      begin
+        if !File.directory? home_dir+"/.cnvrg"
+          FileUtils.mkdir_p([home_dir+"/.cnvrg", home_dir+"/.cnvrg/tmp"])
+        end
+        if !File.exist?(home_dir+"/.cnvrg/config.yml")
+          FileUtils.touch [home_dir+"/.cnvrg/config.yml"]
+        end
+        config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
+
+        compression_path = "#{home_dir}/.cnvrg/tmp"
+        config = {owner: owner, username: current_user, version_last_check: get_start_day(), api: url, compression_path: compression_path}
+        File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+        say "Done"
+      rescue
+        say "ERROR", Thor::Shell::Color::RED
+      end
+    end
     desc '', '', :hide => true
 
     def set_remote_api_url(owner, current_user, url)
@@ -479,6 +504,9 @@ module Cnvrg
         netrc.delete(Cnvrg::Helpers.netrc_domain)
         netrc.save
         log_message("Logged out successfully.\n", Thor::Shell::Color::GREEN)
+      rescue =>  e
+        puts e.message
+        puts e.backtrace
       rescue SignalException
         say "\nAborting"
         exit(1)
@@ -1516,6 +1544,7 @@ module Cnvrg
                 in_thread: ParallelThreads
             }
             begin
+              is_success = true
               clone_result = Parallel.map((response["result"]["tree"]), parallel_options) do |f|
 
                 relative_path = f[0].gsub(/^#{@project.local_path}/, "")
@@ -1525,6 +1554,7 @@ module Cnvrg
                     f
                     successful_changes << relative_path
                   else
+                    is_success =false
                     log_message("Could not create directory: #{f[0]}", Thor::Shell::Color::RED)
                     raise Parallel::Kill
                   end
@@ -1535,6 +1565,7 @@ module Cnvrg
                     f
                     successful_changes << relative_path
                   else
+                    is_success =false
                     log_message("Could not download file: #{f[0]}", Thor::Shell::Color::RED)
                     raise Parallel::Kill
 
@@ -1542,7 +1573,7 @@ module Cnvrg
                 end
               end
             rescue Interrupt
-
+              is_success =false
               log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
 
               @files.revoke_download([], response["result"]["tree"])
@@ -1552,7 +1583,7 @@ module Cnvrg
 
           end
           successful_changes = response["result"]["tree"]
-          if !successful_changes.nil?
+          if !successful_changes.nil? and is_success
             Project.verify_cnvrgignore_exist(project_name)
             log_message("Done.\nDownloaded  #{successful_changes.size}/#{response["result"]["tree"].size} files", Thor::Shell::Color::GREEN)
           else
@@ -1575,7 +1606,7 @@ module Cnvrg
 
     desc 'status', 'Show the working tree status'
     method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
-
+    method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
 
     def status
       begin
@@ -1583,8 +1614,9 @@ module Cnvrg
         log_start(__method__, args, options)
         @project = Project.new(get_project_home)
         new_branch = options["new_branch"] || false
+        force = options["force"] || false
 
-        result = @project.compare_idx(new_branch)["result"]
+        result = @project.compare_idx(new_branch,force:force)["result"]
         commit = result["commit"]
         result = result["tree"]
         log_message("Comparing local changes with remote version:", Thor::Shell::Color::BLUE)
@@ -1661,6 +1693,7 @@ module Cnvrg
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+    method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
 
     def upload(link=false, sync=false, direct=false, ignore_list="")
 
@@ -1672,6 +1705,7 @@ module Cnvrg
 
         @files = Cnvrg::Files.new(@project.owner, @project.slug)
         ignore = options[:ignore] || ""
+        force = options[:force] || false
 
         if ignore.nil? or ignore.empty?
           ignore = ignore_list
@@ -1696,7 +1730,7 @@ module Cnvrg
           end
         end
 
-        result = @project.compare_idx(new_branch)
+        result = @project.compare_idx(new_branch,force:force)
         commit = result["result"]["commit"]
         if !link
           if commit != @project.last_local_commit and !@project.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
@@ -1741,7 +1775,7 @@ module Cnvrg
 
         end
         # Start commit
-        commit_sha1 = @files.start_commit(new_branch)["result"]["commit_sha1"]
+        commit_sha1 = @files.start_commit(new_branch,force:force)["result"]["commit_sha1"]
 
         # upload / update
         begin
@@ -1868,7 +1902,7 @@ module Cnvrg
         end
 
         if update_count == update_total
-          res = @files.end_commit(commit_sha1)
+          res = @files.end_commit(commit_sha1,force:force)
           if (Cnvrg::CLI.is_response_success(res, false))
             # save idx
             begin
@@ -2283,6 +2317,7 @@ module Cnvrg
       method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
       method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
       method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :default => ""
+      method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
 
       def sync(direct=true)
         verify_logged_in(true) if direct
@@ -2293,7 +2328,7 @@ module Cnvrg
 
         invoke :download, [], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true
         invoke :upload, [link=false, sync=true, direct=direct], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
-               :ignore => options[:ignore]
+               :ignore => options[:ignore], :force=> options[:force]
 
 
       end
@@ -2374,7 +2409,6 @@ module Cnvrg
         # end
       end
 
-
       desc '', '', :hide => true
       method_option :sync_before, :type => :boolean, :aliases => ["-sb,--sync_before"], :default => true
       method_option :sync_after, :type => :boolean, :aliases => ["-sa,--sync_after"], :default => true
@@ -2481,6 +2515,7 @@ module Cnvrg
               memory_total = []
               cpu_total = []
               start_loop = Time.now
+              stdout, stderr = '', ''
               begin
                 PTY.spawn(cmd) do |stdout, stdin, pid, stderr|
                   begin
@@ -2563,7 +2598,7 @@ module Cnvrg
 
                     log_message("command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED)
                     log_error(e)
-                  rescue PTY::ChildExited
+                  rescue Open4::ChildExited
                     exp_success = false
                     log_message("The process exited!", Thor::Shell::Color::RED)
                   rescue => e
@@ -2574,6 +2609,8 @@ module Cnvrg
                     exit(0)
                   end
                   ::Process.wait pid
+                end
+
                   cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
                   memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
                   exit_status = $?.exitstatus
@@ -2614,12 +2651,24 @@ module Cnvrg
                   res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
                   check = Helpers.checkmark()
                   log_message("#{check} Done. Experiment's results were updated!", Thor::Shell::Color::GREEN)
-                end
               rescue => e
                 if container
                   container.stop()
                 end
                 log_message("Couldn't run #{cmd}, check your input parameters", Thor::Shell::Color::RED)
+                if @exp
+                  cur_time = Time.now
+                  real_time= Time.now-real
+                  cur_log = {time: cur_time,
+                             message: "Couldn't run #{cmd}, check your input parameters",
+                             type: "stdout",
+                             real: real_time
+
+                  }
+                  log << cur_log
+                  res = @exp.end(log, "-1", end_commit, cpu_average, memory_average)
+
+                end
                 log_error(e)
 
                 exit(1)
@@ -2897,6 +2946,7 @@ module Cnvrg
 
           end
           project = Project.new(working_dir)
+
           choose_image = options["image"]
 
           if !choose_image.nil? and !choose_image.empty?
@@ -3156,7 +3206,7 @@ module Cnvrg
 
       end
 
-      desc 'remote_notebook', 'run notebook server on remote server'
+      desc '', '', :hide=>true
       method_option :notebook_dir, :type => :string, :aliases => ["-n"], :default => "", :desc => "relative path to notebook dir from current directory"
       method_option :machine_type, :type => :string, :default => ""
       method_option :kernel, :type => :string, :aliases => ["--kernel", "-k"], :default => ""
@@ -3165,7 +3215,7 @@ module Cnvrg
       method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
 
 
-      def remote_notebook()
+      def remote_notebook_old()
         verify_logged_in(true)
         log_start(__method__, args, options)
 
@@ -3260,6 +3310,52 @@ module Cnvrg
           exit(1)
         end
       end
+    desc 'remote_notebook', 'run notebook server on remote server'
+    method_option :machine_type, :type => :string, :default => ""
+    method_option :notebook_type, :type => :string, :aliases => ["-n","--notebook_type"], :default => ""
+    method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
+    method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
+    method_option :commit, :type => :string, :aliases => ["--commit"], :default => ""
+
+
+    def remote_notebook()
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+
+      working_dir = is_cnvrg_dir()
+      instance_type = options["machine_type"] || nil
+      data = options["data"]
+      data_commit = options["data_commit"]
+      commit = options["commit"]
+      notebook_type = options["notebook_type"]
+
+
+      begin
+        project = Project.new(working_dir)
+        exp = Experiment.new(project.owner, project.slug)
+
+        if !notebook_type.nil? and !notebook_type.empty?
+          notebook_type = "jupyter"
+        end
+        invoke :sync, [false], []
+        slug = ""
+        res =exp.remote_notebook( instance_type, commit, data, data_commit, notebook_type)
+        if Cnvrg::CLI.is_response_success(res)
+          slug = res["result"]["notebook_url"]
+          log_message("#{Helpers.checkmark} Notebook is ready: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/notebook_sessions/show/#{slug}", Thor::Shell::Color::GREEN)
+
+        end
+      rescue => e
+        log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
+        log_error(e)
+
+      rescue SignalException
+        log_message("Aborting", Thor::Shell::Color::BLUE)
+        notebook_stop(slug) unless slug.nil? or slug.empty?
+
+        exit(1)
+      end
+    end
 
       desc 'search_libraries', 'search if  libraries installed', :hide => true
 
@@ -3560,65 +3656,35 @@ module Cnvrg
       method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false, :desc => "run on remote machine"
       method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
 
-      def notebook_stop
+      def notebook_stop(notebook_slug)
         begin
           verify_logged_in(true)
           log_start(__method__, args, options)
-          remote = options["remote"] || false
           project_dir = is_cnvrg_dir()
 
 
-          log_message('Checking for new updates from remote version', Thor::Shell::Color::BLUE, options["verbose"])
-          invoke :sync, [false], :verbose => options["verbose"]
-
-          log_message("Done Syncing", Thor::Shell::Color::BLUE, options["verbose"])
-
-
           @project = Project.new(project_dir)
 
-          end_commit = @project.last_local_commit
 
           @note = Experiment.new(@project.owner, @project.slug)
+          log_message("Stoping notebook session: #{notebook_slug}", Thor::Shell::Color::BLUE)
 
-          slug = get_note_url(project_dir)
-          @note.end_notebook_session(slug, end_commit)
-
-
-          image = is_project_with_docker(Dir.pwd)
-          if image and image.is_docker and !remote
-            container = image.get_container(true)
-            if !container
-              check = Helpers.checkmark()
-
-              log_message("#{check} Notebook server stopped successfully", Thor::Shell::Color::GREEN)
-              exit(0)
-            end
-
-
-            log_message("Stopping notebook server...", Thor::Shell::Color::BLUE)
-
+          res = @note.end_notebook_session(notebook_slug)
+          if res
             check = Helpers.checkmark()
-            image.remove_note_slug()
-            container.stop()
-
-            log_message("#{check} Notebook server stopped successfully", Thor::Shell::Color::GREEN)
+            log_message("#{check} Notebook session has stopped successfully", Thor::Shell::Color::GREEN)
 
             exit(0)
-          elsif remote
-            log_message("Stopping notebook server...", Thor::Shell::Color::BLUE)
-            check = Helpers.checkmark()
+          else
 
-            log_message("#{check} Notebook server stopped successfully", Thor::Shell::Color::GREEN)
-            exit(0)
+            log_message("Couldn't stop notebook session, try stopping via cnvrg web", Thor::Shell::Color::RED)
+            exit(1)
           end
         rescue => e
           log_message("Error occurd, aborting", Thor::Shell::Color::RED)
           log_error(e)
-          if container
-            container.stop()
-          end
         rescue SignalException
-          say "Aborting"
+          log_message("Aborting", Thor::Shell::Color::BLUE)
           exit(1)
         end
 
@@ -4413,19 +4479,25 @@ module Cnvrg
 
       desc '', '', :hide => true
 
-      def upload_cnvrg_image(image_name)
+      def upload_cnvrg_image(image_path,image_name,secret)
+        begin
         verify_logged_in(false)
-        log_start(__method__, args, options)
-        owner = Cnvrg::CLI.get_owner()
 
-        path = File.expand_path('~')+"/.cnvrg/tmp/#{image_name}.zip"
         @files = Cnvrg::Files.new("", "")
-        # python_arr = Cnvrg::Images.get_installed_packages("python")
-        # py = python_arr.join(",") unless python_arr.nil? or python_arr.empty?
-        # system_arr = Cnvrg::Images.get_installed_packages("system")
-        # sys = system_arr.join(",") unless system_arr.nil? or system_arr.empty?
-        # bash_history = Cnvrg::Images.get_bash_history
-        res = @files.upload_image(path, image_name, owner, true, true, "", "", "", "Image made by cnvrg.io team", "")
+        say "Uploading cnvrg  image file", Thor::Shell::Color::BLUE
+
+        res = @files.upload_cnvrg_image(image_path, image_name, secret)
+        if res
+          say "Successfully uploaded cnvrg image file", Thor::Shell::Color::GREEN
+
+        else
+          say "Couldn't upload cnvrg image file", Thor::Shell::Color::RED
+        end
+        rescue => e
+          puts e
+          puts e.backtrace
+        end
+
 
       end
 
@@ -4515,7 +4587,32 @@ module Cnvrg
           end
         end
       end
+    desc '', '', :hide => true
+    def download_cnvrg_image(image_name, secret)
+      verify_logged_in(false)
+
+      begin
+        @files = Cnvrg::Files.new("", "")
+
+        say "Downloading image file", Thor::Shell::Color::BLUE
+        begin
+          if @files.download_cnvrg_image(image_name, secret)
+
+            say "Successfully downloaded image #{image_name}", Thor::Shell::Color::GREEN
 
+          else
+            say "Couldn't download image #{image_name}", Thor::Shell::Color::RED
+            return false
+          end
+        rescue Interrupt
+          say "The user has exited to process, aborting", Thor::Shell::Color::BLUE
+          exit(1)
+        end
+      rescue SignalException
+        say "\nAborting"
+        exit(1)
+      end
+    end
 
       desc 'list_images', 'lists all custom images you can pull'
 
@@ -4855,7 +4952,7 @@ module Cnvrg
             end
 
           rescue => e
-            puts e
+            puts e.backtrace
           end
         end
 
@@ -4885,6 +4982,7 @@ module Cnvrg
         end
 
         def self.is_response_success(response, should_exit=true)
+          begin
           if response.nil?
             if !Cnvrg::Helpers.internet_connection?
               say("<%= color('Error:You seems to be offline', RED) %>")
@@ -4913,6 +5011,11 @@ module Cnvrg
             end
           end
           return true
+          rescue => e
+            puts e.message
+            puts e.backtrace
+          end
+
         end
 
         def self.get_owner

data/lib/cnvrg/datafiles.rb

@@ -201,44 +201,25 @@ module Cnvrg
 
     end
 
-    def upload_cnvrg_image(absolute_path, image_name, owner, is_public, is_base, dpkg, libraries, bash, message)
-      file_name = File.basename absolute_path
+    def upload_cnvrg_image(absolute_path, image_name,secret)
+      file_name = File.basename relative_path
       file_size = File.size(absolute_path).to_f
-      if is_base
-
-        content_type = "application/zip"
-      else
-        content_type = "application/gzip"
-      end
-      begin
-        upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload_cnvrg", 'POST_FILE', {relative_path: absolute_path,
-                                                                                                  file_name: file_name,
-                                                                                                  image_name: image_name,
-                                                                                                  file_size: file_size,
-                                                                                                  file_content_type: content_type,
-                                                                                                  is_public: is_public,
-                                                                                                  dpkg: dpkg,
-                                                                                                  libraries: libraries,
-                                                                                                  bash_history: bash,
-                                                                                                  commit_message: message,
-                                                                                                  is_base: is_base})
-        # puts upload_resp
-        if Cnvrg::CLI.is_response_success(upload_resp, false)
-          path = upload_resp["result"]["path"]
-          s3_res = upload_small_files_s3(path, absolute_path, content_type)
-          if s3_res
-            commit_resp = Cnvrg::API.request("users/#{owner}/images/#{upload_resp["result"]["id"]}/" + "commit", 'GET')
-            if Cnvrg::CLI.is_response_success(commit_resp, false)
-              return commit_resp["result"]["image"]
-            else
-              return false
-            end
-
-          end
+      mime_type = MimeMagic.by_path(absolute_path)
+      content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
+      upload_resp = Cnvrg::API.request("images/#{image_name}/upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: absolute_path,
+                                                                                     file_name: file_name,
+                                                                                     file_size: file_size, file_content_type: content_type,
+                                                                                     secret: secret})
+      if Cnvrg::CLI.is_response_success(upload_resp, false)
+        path = upload_resp["result"]["path"]
+        s3_res = upload_large_files_s3(upload_resp, absolute_path)
+        if s3_res
+          Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
+                                                                    blob_id: upload_resp["result"]["id"]})
+          return true
         end
-        return false
-      rescue => e
       end
+      return false
 
     end
 
@@ -309,7 +290,7 @@ module Cnvrg
 
         resp = s3.bucket(bucket).
             object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
-            upload_file(file_path, {:use_accelerate_endpoint => true})
+            upload_file(file_path, {:use_accelerate_endpoint => true,:server_side_encryption => 'AES256'})
         # s4cmd_path = upload_resp["result"]["path_s4cmd"]
         #
         # s4cmd_uri = URI.parse(s4cmd_path)
@@ -399,6 +380,7 @@ module Cnvrg
 
     def create_dir(absolute_path, relative_path, commit_sha1)
       response = Cnvrg::API.request(@base_resource + "create_dir", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
+      puts response
       return Cnvrg::CLI.is_response_success(response, false)
     end
 

data/lib/cnvrg/experiment.rb

@@ -34,9 +34,8 @@ module Cnvrg
 
     end
 
-    def end_notebook_session(notebook_slug, end_commit)
-      res = Cnvrg::API.request(@base_resource + "notebook/end_session", 'POST',
-                               {notebook_slug: notebook_slug, end_commit: end_commit})
+    def end_notebook_session(notebook_slug)
+      res = Cnvrg::API.request(@base_resource + "notebook/#{notebook_slug}/stop", 'GET')
       Cnvrg::CLI.is_response_success(res,false)
 
       return res
@@ -89,6 +88,11 @@ module Cnvrg
                                                                                                             dataset_commit: data_commit})
       return response
     end
+    def remote_notebook(instance_type, commit, data, data_commit, notebook_type)
+      response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/notebook/remote", 'POST', {instance_type: instance_type,dataset_slug:data,
+                                                                                                          dataset_commit: data_commit,commit:commit,notebook_type:notebook_type})
+      return response
+    end
 
     def upload_temp_log(temp_log, cpu_average, memory_average)
       response = Cnvrg::API.request(@base_resource + "experiment/upload_temp_log", 'POST', {output: temp_log,

data/lib/cnvrg/files.rb

@@ -3,7 +3,6 @@ require 'aws-sdk'
 require 'URLcrypt'
 require 'tempfile'
 require 'net/http'
-
 module Cnvrg
   class Files
 
@@ -149,44 +148,32 @@ module Cnvrg
 
     end
 
-    def upload_cnvrg_image(absolute_path, image_name, owner, is_public, is_base, dpkg, libraries, bash, message)
+    def upload_cnvrg_image(absolute_path, image_name,secret)
       file_name = File.basename absolute_path
       file_size = File.size(absolute_path).to_f
-      if is_base
-
-        content_type = "application/zip"
-      else
-        content_type = "application/gzip"
-      end
+      content_type = MimeMagic.by_path(absolute_path)
       begin
-        upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload_cnvrg", 'POST_FILE', {relative_path: absolute_path,
+        upload_resp = Cnvrg::API.request("images/#{image_name}/upload", 'POST_FILE', {relative_path: absolute_path,
                                                                                                   file_name: file_name,
-                                                                                                  image_name: image_name,
                                                                                                   file_size: file_size,
                                                                                                   file_content_type: content_type,
-                                                                                                  is_public: is_public,
-                                                                                                  dpkg: dpkg,
-                                                                                                  libraries: libraries,
-                                                                                                  bash_history: bash,
-                                                                                                  commit_message: message,
-                                                                                                  is_base: is_base})
+                                                                                                  secret:secret
+                                                                                                  })
         # puts upload_resp
         if Cnvrg::CLI.is_response_success(upload_resp, false)
-          path = upload_resp["result"]["path"]
-          s3_res = upload_small_files_s3(path, absolute_path, content_type)
+          s3_res = upload_large_files_s3(upload_resp, absolute_path)
           if s3_res
-            commit_resp = Cnvrg::API.request("users/#{owner}/images/#{upload_resp["result"]["id"]}/" + "commit", 'GET')
-            if Cnvrg::CLI.is_response_success(commit_resp, false)
-              return commit_resp["result"]["image"]
-            else
-              return false
-            end
-
+              return true
+          else
+            return false
           end
+
         end
-        return false
       rescue => e
+        return false
       end
+        return false
+
 
     end
 
@@ -213,6 +200,56 @@ module Cnvrg
       end
 
 
+    end
+    def download_cnvrg_image(image_name, secret)
+      res =Cnvrg::API.request("images/#{image_name}/" + "download", 'POST', {secret:secret},true)
+      Cnvrg::CLI.is_response_success(res, true)
+      if res["result"]
+        download_resp = res
+        sts_path = download_resp["result"]["path_sts"]
+        uri = URI.parse(sts_path)
+        http_object = Net::HTTP.new(uri.host, uri.port)
+        http_object.use_ssl = true if uri.scheme == 'https'
+        request = Net::HTTP::Get.new(sts_path)
+        body = ""
+        http_object.start do |http|
+          response = http.request request
+          body = response.read_body
+        end
+        split = body.split("\n")
+        key = split[0]
+        iv = split[1]
+
+        access =  Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
+
+        secret =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_s"])
+
+        session =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_st"])
+        region =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["region"])
+
+        bucket =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["bucket"])
+        key =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
+
+        client = Aws::S3::Client.new(
+            :access_key_id =>access,
+            :secret_access_key => secret,
+            :session_token => session,
+            :region => region,
+            :http_open_timeout => 60, :retry_limit => 20
+        )
+
+        File.open("/tmp/#{image_name}.tar", 'w+') do |file|
+          resp = client.get_object({bucket:bucket,
+                                    key:key}, target: file)
+        end
+        return true
+      end
+
+    rescue => e
+      puts e
+      return false
+
+
     end
 
     def upload_large_files_s3(upload_resp, file_path)
@@ -284,7 +321,7 @@ module Cnvrg
             s3 = Aws::S3::Resource.new(client: client)
             resp = s3.bucket(bucket).
                 object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
-                upload_file(file_path, {:use_accelerate_endpoint => true})
+                upload_file(file_path, {:use_accelerate_endpoint => true, :server_side_encryption => 'AES256'})
 
           return resp
 
@@ -349,6 +386,7 @@ module Cnvrg
         begin
           res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path,
                                                                               commit_sha1: commit_sha1,new_version:true})
+
           Cnvrg::CLI.is_response_success(res, false)
           if res["result"]
             download_resp = res
@@ -387,7 +425,6 @@ module Cnvrg
                 :region => region,
                 :http_open_timeout => 60, :retry_limit => 20
             )
-
             File.open(project_home+"/"+absolute_path, 'w+') do |file|
               resp = client.get_object({bucket:bucket,
                                     key:key}, target: file)
@@ -468,18 +505,18 @@ module Cnvrg
       return true
 
     end
-      def start_commit(new_branch)
+      def start_commit(new_branch,force:false)
 
-        response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug, new_branch: new_branch,
+        response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug, new_branch: new_branch,force:force,
                                                                                 username: @owner})
         Cnvrg::CLI.is_response_success(response,false)
         return response
       end
 
-      def end_commit(commit_sha1)
-        response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1})
-        return response
-      end
+    def end_commit(commit_sha1,force:false)
+      response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1,force:force})
+      return response
+    end
 
       def rollback_commit(commit_sha1)
         response = Cnvrg::API.request("#{base_resource}/commit/rollback", 'POST', {commit_sha1: commit_sha1})

data/lib/cnvrg/project.rb

@@ -351,12 +351,11 @@ module Cnvrg
         return response
       end
 
-      def compare_idx(new_branch, commit=last_local_commit)
-
+      def compare_idx(new_branch, commit:last_local_commit,force:false)
         local_idx = self.generate_idx
         ignore_list = self.send_ignore_list()
         response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/status", 'POST', {idx: local_idx, new_branch: new_branch,
-                                                                                                   current_commit: commit,ignore:ignore_list})
+                                                                                                   current_commit: commit,ignore:ignore_list, force:force})
         CLI.is_response_success(response,false)
         return response
       end

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '0.2'
+  VERSION = '0.2.2'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: '0.2'
+  version: 0.2.2
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-11 00:00:00.000000000 Z
+date: 2017-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -259,6 +259,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.12.0
+- !ruby/object:Gem::Dependency
+  name: net_http_ssl_fix
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: logstash-logger
   requirement: !ruby/object:Gem::Requirement
@@ -402,7 +416,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.