cnvrg 0.0.147 → 0.0.148

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a089a8118adb94a9ae9b8c37e75b6bdf8690eaf9
-  data.tar.gz: aef1c463d671068e6c7dcaa288f40a4d8b3af303
+  metadata.gz: 9889a94a7cbc7cf8e729daf363e61bd38072960a
+  data.tar.gz: e46f0fed0b9b6a3007daa399dc4e75b81d402aa6
 SHA512:
-  metadata.gz: a12c9b7ad5001f6c8e323ba82851cf3a23398761a6e0c90d72a09d38712a34b9a008e59256006e521f4b498e3bebe572de51ead008b0b75fc877b41f6fbc9c34
-  data.tar.gz: d0859793528290bee4e56f8ce6f1adb70cc7e1ffeead7cc5dab33e2a85df4d59e43eb0319856ced6f031831d5609c52a75dcb563b2f0a3a1b31238c14da6c6f7
+  metadata.gz: 8bc2c1dca91ad6f2ff9e4248b95b34231bef97be4eb9d7ba52b5cd73c9def6ee36ad672beaf45c8581e868d0b89fc7d8cf6e6f6babfd35ed6f5feb5bf610c222
+  data.tar.gz: 02d1c07bec47cada6285f3df813b6fc67b0c3f2700df70bc89be691e72f20593da9672ce2fa03430dc2d548e5ba9ac930a7f3d2711b2a36e6fe32f022367e589

data/lib/cnvrg/cli.rb

@@ -70,11 +70,11 @@ module Cnvrg
       #         },
       #     },
       # }
-      container = Docker::Container.get('b4d64bf83f41')
-      s = "/leah/1/2/3/4/5"
-      command = ["/bin/bash","-lc","sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{s}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
-      puts container.exec(command, tty: true)
-
+      # container = Docker::Container.get('b4d64bf83f41')
+      # s = "/leah/1/2/3/4/5"
+      # command = ["/bin/bash","-lc","sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{s}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
+      # puts container.exec(command, tty: true)
+      puts File.read("s4cmd.py")
     end
 
 
@@ -815,9 +815,11 @@ module Cnvrg
       end
 
     end
-    desc 'unlink','Unlink a project from current directory'
+
+
+    desc 'unlink','Unlink a project from current directory', :hide=>true
     def create_volume
-      verify_logged_in(true)
+      verify_logged_in(false)
       log_start(__method__, args, options)
       dataset_dir = is_cnvrg_dir(Dir.pwd)
       @dataset = Dataset.new(dataset_dir)
@@ -825,6 +827,20 @@ module Cnvrg
 
     end
 
+    desc 'list data','Unlink a project from current directory', :hide=>true
+    def list_data
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+      dataset_dir = is_cnvrg_dir(Dir.pwd)
+      @dataset = Dataset.new(dataset_dir)
+      result = @dataset.list()
+      list = result["result"]["list"]
+
+      print_table(list)
+
+    end
+
+
     desc 'unlink','Unlink a project from current directory'
     def unlink
       verify_logged_in(false)
@@ -1083,7 +1099,6 @@ module Cnvrg
         #
         # end
         check = Helpers.checkmark()
-
         if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
           log_end(0)
           say "#{check} Project is up to date", Thor::Shell::Color::GREEN unless ((options["sync"] or sync) and !direct)
@@ -3056,6 +3071,7 @@ module Cnvrg
     method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
     method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
     method_option :cmd, :type => :string, :aliases => ["-c", "--c"], :default => "/usr/local/cnvrg/run_ipython.sh"
+    method_option :volume, :type => :string, :aliases => ["-v", "--v"], :default => "/home/ds/notebooks/data"
 
 
     def config_remote(image_name, port=7654,tensport=6006)
@@ -3071,15 +3087,17 @@ module Cnvrg
         login_content = options["login"]
         app_dir = options["app_dir"]
         cmd= options["cmd"]
+        volume_from = options["volume"]
 
         image_settings = {
             'Image' => "#{image_name}:latest",
-            'User' => 'ds',
+
             'Cmd' => cmd,
             'WorkingDir' => app_dir,
             'ExposedPorts' => {
                 '8888/tcp' => {},
             },
+            'VolumesFrom' =>volume_from,
             'HostConfig' => {
                 'PortBindings' => {
                     '8888/tcp' => [

data/lib/cnvrg/dataset.rb

@@ -30,6 +30,12 @@ module Cnvrg
       CLI.is_response_success(response)
       return response
 
+    end
+    def list
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/list", 'GET')
+      CLI.is_response_success(response)
+      return response
+
     end
     def url
       url = Cnvrg::Helpers.remote_url

data/lib/cnvrg/files.rb

@@ -1,11 +1,14 @@
 require 'mimemagic'
 require 'aws-sdk'
 require 'URLcrypt'
+require 'tempfile'
 
 module Cnvrg
   class Files
 
     LARGE_FILE=1024*1024*5
+    MULTIPART_SPLIT=10000000
+
     attr_reader :base_resource
 
     def initialize(owner, project_slug)
@@ -19,9 +22,11 @@ module Cnvrg
       file_size = File.size(absolute_path).to_f
       mime_type = MimeMagic.by_path(absolute_path)
       content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
+      sha1 = Digest::SHA1.file(absolute_path).hexdigest
+
       upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
                                                                                      commit_sha1: commit_sha1, file_name: file_name,
-                                                                                     file_size: file_size, file_content_type: content_type})
+                                                                                     file_size: file_size, file_content_type: content_type, sha1:sha1})
       if Cnvrg::CLI.is_response_success(upload_resp, false)
         path = upload_resp["result"]["path"]
         if file_size.to_f>= Cnvrg::Files::LARGE_FILE.to_f
@@ -29,11 +34,13 @@ module Cnvrg
         else
           s3_res = upload_small_files_s3(path, absolute_path, content_type)
         end
-        if s3_res
-          Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
-                                                                    blob_id: upload_resp["result"]["id"]})
-          return true
-        end
+        return s3_res
+        # if s3_res
+          # update_s3_resp = Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
+          #                                                           blob_id: upload_resp["result"]["id"]})
+          # is_suc = Cnvrg::CLI.is_response_success(update_s3_resp, false)
+          #
+          # return is_suc
       end
       return false
     end
@@ -199,9 +206,11 @@ module Cnvrg
 
     end
 
-    def upload_large_files_s3(upload_resp, file_path)
+    def   upload_large_files_s3(upload_resp, file_path)
       begin
         sts_path = upload_resp["result"]["path_sts"]
+        s4cmd_path = upload_resp["result"]["path_s4cmd"]
+
         uri = URI.parse(sts_path)
         http_object = Net::HTTP.new(uri.host, uri.port)
         http_object.use_ssl = true if uri.scheme == 'https'
@@ -212,18 +221,62 @@ module Cnvrg
           response = http.request request
           body = response.read_body
         end
+
         URLcrypt::key = [body].pack('H*')
-        s3 = Aws::S3::Resource.new(
-            :access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
-            :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
-            :session_token => URLcrypt.decrypt(upload_resp["result"]["sts_st"]),
-            :region => URLcrypt.decrypt(upload_resp["result"]["region"]))
+
+        python_version=`python --version  > /dev/null 2>&1` ;  is_python=$?.success?
+        if !is_python
+           s3 = Aws::S3::Resource.new(
+               :access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
+               :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
+               :session_token => URLcrypt.decrypt(upload_resp["result"]["sts_st"]),
+               :region => URLcrypt.decrypt(upload_resp["result"]["region"]))
         resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
             object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
             upload_file(file_path,{:use_accelerate_endpoint=>true})
+        else
+          s4cmd_uri = URI.parse(s4cmd_path)
+          s4cmd_http_object = Net::HTTP.new(s4cmd_uri.host, s4cmd_uri.port)
+          s4cmd_http_object.use_ssl = true if s4cmd_uri.scheme == 'https'
+          s4cmd_request = Net::HTTP::Get.new(s4cmd_path)
+
+          s4cmd_body = ""
+          s4cmd_http_object.start do |http|
+            response = http.request s4cmd_request
+            s4cmd_body = response.read_body
+          end
+
+          s4cmd_new_body = s4cmd_body.gsub(" self.client = self.boto3.client('s3',
+                                      aws_access_key_id=aws_access_key_id,
+                                      aws_secret_access_key=aws_secret_access_key)"," self.client = self.boto3.client('s3',
+                                      aws_access_key_id='#{ URLcrypt.decrypt(upload_resp["result"]["sts_a"])}',
+                                      aws_secret_access_key='#{URLcrypt.decrypt(upload_resp["result"]["sts_s"])}',
+                                      aws_session_token='#{URLcrypt.decrypt(upload_resp["result"]["sts_st"])}')")
+
+          tmp = Tempfile.new('s4cmd.py')
+          tmp << s4cmd_new_body
+          tmp.flush
+          tmp.close
+
+          is_success = false
+          count = 0
+          while !is_success and count <3
+
+            resp = `python #{tmp.path} --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)}  > /dev/null 2>&1`
+            is_success =$?.success?
+            count +=1
+
+          end
+          resp= is_success
+
+        end
 
         return resp
+
       rescue =>e
+        if File.exist? tmp
+          FileUtils.rm_rf [tmp]
+        end
         puts e
         return false
 

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-    VERSION = '0.0.147'
+    VERSION = '0.0.148'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 0.0.147
+  version: 0.0.148
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-30 00:00:00.000000000 Z
+date: 2017-05-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler