cnvrg 0.0.1414 → 0.0.1418

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a3a999e3ce7616d42f0a81033156a8d5819b68fd
-  data.tar.gz: 5dfc8054f9ee3f38cefaef9a06b113f72e3d12ba
+  metadata.gz: 6e5e43e32a2a764d0177be682b7957bdaa6c184b
+  data.tar.gz: a1bd77092a713f3f6edb16f0f2bbb998d3a795d2
 SHA512:
-  metadata.gz: cd4af46476dcb927770dd1c88c782ab9d2125e94cc496441f8b465b8edbcd22a2dd02d9d5dbb299c2aa1ac9fa183b2ece4c802f2041b243949a1bf385c317877
-  data.tar.gz: 2ec0f4e18e2571ce46dd77fa20bf7b6d5b5e667c4d76816502d5e86799c47417a0faec100c46d97566278fa4d68ce4779ead5e4e320ce767a6da711b3fc2d721
+  metadata.gz: f60c19507e5eed03dd015ec4d8d03694cf518692057ba969b64cc62a2208d50e02007ace23c0d82740ffe263f685271610ba58f87d22a6866c0bd2d4021ab54a
+  data.tar.gz: 8049f800636d14a9de359dd580bece4a28cf6422304e65d17862e5aa85b83604aa9bf798c70b322c2bbcbafc6a40e3bf1659dbb6823d2050f044cd6b58468fef

data/lib/cnvrg/Images.rb

@@ -172,9 +172,10 @@ module Cnvrg
     def create_container(port=7654, is_remote=false)
       begin
         image_settings = {
-            'Image' => "#{@image_name}:#{@image_tag}",
+            'Image' => "#{@image_name}:latest",
             'User' => 'ds',
-            'Cmd' => '/home/ds/run_ipython.sh',
+            'Cmd' => '/usr/local/cnvrg/run_ipython.sh',
+            'WorkingDir' => '/home/ds/notebooks',
             'ExposedPorts' => {
                 '8888/tcp' => {},
             },
@@ -187,9 +188,6 @@ module Cnvrg
                 },
             },
         }
-        # if !is_remote
-        #   image_settings['HostConfig'].merge!({ 'Binds' => ["#{@working_dir}:/home/ds/notebooks"]})
-        # end
         container = Docker::Container.create(image_settings)
         container.start()
         netrc = File.open(File.expand_path('~')+"/.netrc", "rb")
@@ -199,10 +197,6 @@ module Cnvrg
         p = container.exec(command, tty: true)
         command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.netrc"]
         p = container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
-        container.exec(command, tty: true)
         config = File.open(File.expand_path('~')+"/.cnvrg/config.yml", "rb")
         config_content = config.read
         container.store_file("/home/ds/.cnvrg/config.yml", config_content)
@@ -249,11 +243,13 @@ module Cnvrg
 
     end
 
-    def remote_notebook(notebook_path, instance_type, kernel)
+    def remote_notebook(notebook_path, instance_type, kernel,data,data_commit)
       response = Cnvrg::API.request("users/#{@owner}/images/#{@image_slug}/remote_notebook", 'POST', {dir: notebook_path,
                                                                                                       project_slug: @project_slug,
                                                                                                       instance_type: instance_type,
-                                                                                                      kernel: kernel})
+                                                                                                      kernel: kernel,
+                                                                                                      dataset_slug:data,
+                                                                                                      dataset_commit: data_commit})
       return response
     end
 

data/lib/cnvrg/cli.rb

@@ -759,7 +759,7 @@ module Cnvrg
     end
 
     desc 'init_data_container', 'Init dataset directory', :hide => true
-    method_option :login_content, :type => :string, :aliases => ["--l"], :default => ""
+    method_option :login_content, :type => :string, :aliases => ["-l"], :default => ""
 
     def init_data_container(container)
       begin
@@ -2794,7 +2794,10 @@ module Cnvrg
     method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
     method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
     method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
-    method_option :image, :type => :string, :aliases => ["--i"], :default => ""
+    method_option :image, :type => :string, :aliases => ["-i","--image"], :default => ""
+    method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
+    method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
+
 
     desc 'notebook', 'starts a notebook session remotely or locally'
 
@@ -2803,7 +2806,8 @@ module Cnvrg
       notebook_dir = options["notebook_dir"]
       kernel = options["kernel"]
       image = options["image"]
-
+      data = options["data"]
+      data_commit = options["data_commit"]
       if local
         invoke :run_notebook, [], :notebook_dir => notebook_dir, :remote => false, :kernel => kernel, :image => image
         return
@@ -2812,7 +2816,8 @@ module Cnvrg
                      "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
         instance_type = get_instance_type(instances)
 
-        invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type, :image => image
+        invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type, :image => image,
+               :data=> data, :data_commit => data_commit
         return
 
       end
@@ -2825,6 +2830,9 @@ module Cnvrg
     method_option :machine_type, :type => :string, :default => ""
     method_option :kernel, :type => :string, :aliases => ["--kernel", "-k"], :default => ""
     method_option :image, :type => :string, :aliases => ["-i"], :default => ""
+    method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
+    method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
+
 
 
     def remote_notebook()
@@ -2836,6 +2844,8 @@ module Cnvrg
       notebook_dir = options["notebook_dir"]
       instance_type = options["machine_type"] || nil
       kernel = options["kernel"] || nil
+      data = options["data"]
+      data_commit = options["data_commit"]
 
 
       begin
@@ -2862,7 +2872,7 @@ module Cnvrg
         invoke :sync, [false], []
 
 
-        res = @image.remote_notebook(notebook_dir, instance_type, kernel)
+        res = @image.remote_notebook(notebook_dir, instance_type, kernel,data,data_commit)
         if Cnvrg::CLI.is_response_success(res)
           if res["result"]["machine"] == -1
             say "There are no available machines", Thor::Shell::Color::BLUE
@@ -3063,7 +3073,7 @@ module Cnvrg
         notebook_dir = options["notebook_dir"]
         remote = options["remote"] || false
         kernel = options["kernel"] || ""
-
+        notebooks_pid = nil
 
         if notebook_dir.empty?
           notebook_dir = project_dir
@@ -3071,25 +3081,60 @@ module Cnvrg
 
           notebook_dir = project_dir+ notebook_dir
         end
-        say "Linking notebook directory to: #{File.basename notebook_dir}", Thor::Shell::Color::BLUE
         choose_image = options["image"]
 
         if !choose_image.nil? and !choose_image.empty?
           invoke :set_image, [choose_image]
         end
+
         image = is_project_with_docker(Dir.pwd)
+        if !image
+          jupyter_installed = `which jupyter`
+          if !$?.success?
+            say "Could not find jupyter, Is it installed?", Thor::Shell::Color::RED
+            exit(1)
+          end
+
+
+          cmd = "jupyter-notebook --port=8888"
+          PTY.spawn(cmd) do |stdout, stdin, pid, stderr|
+            begin
+              notebooks_pid = pid
+              stdout.each do |line|
+                puts line
+
+                  end
+
+            rescue Errno::EIO => e
+              # break
+            rescue Errno::ENOENT
+              log_end(1, "command #{cmd} isn't valid")
+
+
+              say "command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED
+            rescue PTY::ChildExited
+              log_end(1, "proccess exited")
+              say "The process exited!", Thor::Shell::Color::RED
+            rescue => e
+              log_end(-1, e.message)
+              say "Error occurred,aborting", Thor::Shell::Color::RED
+              exit(0)
+
+            end
+
+
+          end
+
+        end
+
         if image and image.is_docker and !remote
           container= image.get_container
           if !container
-
-            say "Couldn't create container with image #{image.image_name}:#{image.image_tag}", Thor::Shell::Color::RED
+            say "Couldn't start docker container", Thor::Shell::Color::RED
             exit(1)
+
           end
-        else
-          say "Project is not configured with any image", Thor::Shell::Color::RED
-          exit(1)
 
-        end
         if options["verbose"]
           say "Syncing project before running", Thor::Shell::Color::BLUE
           say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
@@ -3116,7 +3161,7 @@ module Cnvrg
         container.exec(command, tty: true)
         container.stop()
         container.start()
-
+        sleep(7)
         @note = Experiment.new(@project.owner, @project.slug)
         port = image.container_port()
 
@@ -3128,8 +3173,13 @@ module Cnvrg
           exit(1)
         end
 
-        url = list[list.size-1].split(" ::")
-        token = url[0][(url[0].index("=")+1)..-1]
+          result = ""
+          list.each do |r|
+            if r.include? "http"
+              result = r
+            end
+          end
+          token = result.to_s.split("::")[0].to_s.match(/(token=)(.+)\s/)[2]
 
         # machine_activity = @note.get_machine_activity(project_dir)
 
@@ -3149,20 +3199,9 @@ module Cnvrg
           log_end(1, "can't start notebook server")
           exit(1)
         end
+        end
+
 
-          # jup =verify_software_installed("jupyter-notebook")
-          # logs = `#{jup} --no-browser --ip=0.0.0.0  --notebook-dir=#{notebook_dir}`
-          # url = URI.extract(logs).reject { |x| x if !x.include? "http" }.uniq![0]
-          # if !url.empty?
-          #   check = Helpers.checkmark()
-          #
-          #   say "#{check} Notebook server started successfully, view notebook in url: #{url}", Thor::Shell::Color::GREEN
-          #   log_end(0)
-          # else
-          #   say "Couldn't start notebook server", Thor::Shell::Color::RED
-          #   log_end(1, "can't start notebook server")
-          #   exit(1)
-          # end
       rescue => e
         log_end(-1, e.message)
         say "Error occurd, aborting", Thor::Shell::Color::RED
@@ -3170,12 +3209,20 @@ module Cnvrg
           container.stop()
         end
       rescue SignalException
-        log_end(-1)
-        if container
-          container.stop()
+        if !notebooks_pid.nil?
+          ::Process.kill(0,notebooks_pid)
+          say "#{check} Notebook has stopped successfully", Thor::Shell::Color::GREEN
+
+
+          invoke :sync, [false], []
+        else
+          log_end(-1)
+
+          if container
+            container.stop()
+          end
         end
-        say "Aborting"
-        exit(1)
+
       end
 
 
@@ -3676,9 +3723,9 @@ module Cnvrg
     end
 
     desc '', '', :hide => true
-    method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
-    method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
-    method_option :cmd, :type => :string, :aliases => ["-c", "--c"], :default => "/usr/local/cnvrg/run_ipython.sh"
+    method_option :login, :type => :string, :aliases => ["-l"], :default => ""
+    method_option :app_dir, :type => :string, :aliases => ["-d"], :default => "/home/ds/notebooks"
+    method_option :cmd, :type => :string, :aliases => ["-c"], :default => "/usr/local/cnvrg/run_ipython.sh"
 
 
     def config_remote(image_name, port=7654, tensport=6006)
@@ -3719,11 +3766,11 @@ module Cnvrg
         container.start()
         command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
         container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.cnvrg /home/ds/.netrc"]
+        # command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
+        # container.exec(command, tty: true)
+        # command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
+        # container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
         container.exec(command, tty: true)
         command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
         container.exec(command, tty: true)
@@ -3740,6 +3787,25 @@ module Cnvrg
       end
     end
 
+
+    desc '', '', :hide => true
+    method_option :login, :type => :string, :aliases => ["-l"], :default => ""
+
+    def config_netrc(container)
+
+      login_content = options["login"]
+
+        container = Docker::Container.get(container)
+        command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
+        container.exec(command, tty: true)
+        say "OK"
+
+    end
+
     desc '', '', :hide => true
     method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
     method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
@@ -3950,7 +4016,6 @@ module Cnvrg
               end
             end
 
-
             return dir_path
 
           else
@@ -4072,7 +4137,7 @@ module Cnvrg
 
     desc 'pull_image', 'downloads and loads an image'
 
-    def pull(image_name)
+    def pull_image(image_name)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
@@ -4085,8 +4150,18 @@ module Cnvrg
         path = download_image(image_name, image["slug"])
         if path
           say "Building image", Thor::Shell::Color::BLUE
-          Docker.options[:read_timeout]=1200
-          image = Docker::Image.build_from_dir(path, {'dockerfile' => 'Dockerfile', 't' => "#{image_name}:lastest"})
+          Docker.options[:read_timeout]=216000
+          image = Docker::Image.build_from_dir(path, {'dockerfile' => 'Dockerfile.cpu', 't' => "#{image_name}:latest"}) do |v|
+            begin
+              if (log = JSON.parse(v)) && log.has_key?("stream")
+                next if log["stream"].starts_with? "Step"
+                $stdout.puts log["stream"]
+              end
+            rescue
+            end
+
+          end
+
           if not image.nil?
             FileUtils.rm_rf(path)
             checks = Helpers.checkmark()
@@ -4095,11 +4170,13 @@ module Cnvrg
             return image
             log_end(0)
           else
+
             say "Could not build image", Thor::Shell::Color::RED
             log_end(1, "Could build  image")
             return false
           end
         else
+
           say "Could not download image", Thor::Shell::Color::RED
           log_end(1, "Could build  image")
           return false
@@ -4127,7 +4204,9 @@ module Cnvrg
           #
           #   end
           # end
-      rescue
+      rescue => e
+
+
         say "Couldn't build image", Thor::Shell::Color::RED
 
       rescue SignalException

data/lib/cnvrg/datafiles.rb

@@ -254,7 +254,7 @@ module Cnvrg
 
         end
 
-        if !s4cmd_suc or !s4cmd_install_suc
+        if !s4cmd_suc or !s4cmd_suc or !s4cmd_install_suc
           s3 = Aws::S3::Resource.new(
               :access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
               :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),

data/lib/cnvrg/files.rb

@@ -229,11 +229,11 @@ module Cnvrg
 
           s4cmd=`pip freeze |grep s4cmd > /dev/null 2>&1` ;  s4cmd_suc=$?.success?
           if !s4cmd_suc
-            `pip install s4cmd  > /dev/null 2>&1`
+            `pip install s4cmd  > /dev/null 2>&1` ; s4cmd_install_suc=$?.success?
           end
 
         end
-        if !is_python
+        if !s4cmd_suc or !s4cmd_suc or !s4cmd_install_suc
            s3 = Aws::S3::Resource.new(
                :access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
                :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
@@ -251,7 +251,7 @@ module Cnvrg
           s4cmd_body = ""
           s4cmd_http_object.start do |http|
             response = http.request s4cmd_request
-            s4cmd_body = response.read_body
+            s4cmd_body = response.read_bodyo
           end
           s4cmd_new_body = s4cmd_body.gsub(" self.client = self.boto3.client('s3',
                                       aws_access_key_id=aws_access_key_id,

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-    VERSION = '0.0.1414'
+    VERSION = '0.0.1418'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.0.1414
+  version: 0.0.1418
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-30 00:00:00.000000000 Z
+date: 2017-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -359,7 +359,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.3
+rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.