cnvrg 1.4.8 → 1.4.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 33a363bbe4dbd47341175ffb1882e7ba60e64b3453295853593d3486f5a96d40
-  data.tar.gz: 3c4726d96cc5f67becbcb8fc09edd674528da06367f8fdb662b57ed2c29d471b
+  metadata.gz: 8139db865f88ce341ad6bf2330c622f995cb2b8ef6d581e2c45f817bbbfad8a3
+  data.tar.gz: 8bee4a637a7d39bb38fd19a5bf68b9f0bee00fa39d7fac4d4293b9cf6750d477
 SHA512:
-  metadata.gz: 355ce293882a1e5c72bc30efd321b073c0e06d9fac1af465832e13042623f6c3fd0982de90f8be0026da312097385c737981e49173ee8f46d16bfeb173b00e33
-  data.tar.gz: d9d3512d039eea9c1f46eff643d2930545b1e194c57af797d7cb615a9026283890bc3f10bf99b56031336461474829fc69edbd571ad9368ccae4624921aff9b5
+  metadata.gz: ee28520a8afaeb9551e8ab8762f46bc1f0171450032d858e04b0b42f4223a2377c854d004ed9228db94938812bb4ac4f9f5d2a989e9243fb992a160fc867d5d9
+  data.tar.gz: 860fc186eb8828bfe450ba84670c0b7e3913ff0662d4cc266e15b692386f39881bfefded59b6844c6efd380fd8926baa8ccc116301a22997355056af50b9030d

data/cnvrg.gemspec

@@ -44,4 +44,5 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'ruby-progressbar'
     spec.add_runtime_dependency 'net-ssh'
     spec.add_runtime_dependency 'down'
+    spec.add_runtime_dependency 'thread'
 end

data/lib/cnvrg/cli.rb

@@ -488,14 +488,18 @@ module Cnvrg
       job_type, job_id = ENV['CNVRG_JOB_TYPE'], ENV['CNVRG_JOB_ID']
       machine_activity = @project.get_machine_activity
 
+      notify_thread = Thread.new do
+        res = @project.send_restart_request(job_type: job_type, job_id: job_id, ma_id: machine_activity)
+        while res.blank?
+          res = @project.send_restart_request(job_type: job_type, job_id: job_id, ma_id: machine_activity)
+          sleep(10)
+        end
+      end
+
       sync_force = job_type == "NotebookSession" ? false : true
       upload(false, false, true, '', true, sync_force , ENV['CNVRG_OUTPUT_DIR'], job_type, job_id)
       log_message('Spot instance is going to be terminated', Thor::Shell::Color::YELLOW)
-      res = nil
-      while res.blank?
-        res = @project.send_restart_request(job_type: job_type, job_id: job_id, ma_id: machine_activity)
-        sleep(10)
-      end
+      notify_thread.join
     end
 
 
@@ -594,6 +598,20 @@ module Cnvrg
 
     def me()
       begin
+
+        home_dir = File.expand_path('~')
+        config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
+
+        api = config[:api]
+        verify_ssl = config[:verify_ssl]
+        if api.present?
+          log_message("API: #{api}", Thor::Shell::Color::BLUE)
+        end
+        if verify_ssl.present?
+          log_message("SSL Verification: #{verify_ssl}", Thor::Shell::Color::BLUE)
+        end
+        log_message("Logs file located at: #{$LOG.device.io.path}", Thor::Shell::Color::BLUE)
+
         verify_logged_in(false)
         log_start(__method__, args, options)
         auth = Cnvrg::Auth.new
@@ -610,6 +628,25 @@ module Cnvrg
       end
     end
 
+    desc 'logs', 'Prints cnvrg cli logs file to screen'
+    method_option :lines, :type => :numeric, :aliases => ["-l","-n","--lines"], :default => nil
+    def logs()
+      begin
+        lines = options["lines"]
+        if lines.present?
+          puts open($LOG.device.io.path).readlines.last(lines)
+        else
+          puts open($LOG.device.io.path).readlines.last(100)
+        end
+
+
+      rescue SignalException
+        say "\nAborting", Thor::Shell::Color::RED
+        exit(1)
+      end
+    end
+
+
     ## Projects
     desc 'new', 'Create a new cnvrg project'
     method_option :clean, :type => :boolean, :aliases => ["-c"], :default => false
@@ -836,23 +873,22 @@ module Cnvrg
         project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
         slug = url_parts[project_index + 1]
         owner = url_parts[project_index - 1]
-        response = Cnvrg::API.request("users/#{owner}/datasets/#{slug}/clone", 'POST',{ commit: commit, query:query})
-        Cnvrg::CLI.is_response_success(response,true)
+        @dataset = Dataset.new(dataset_url: dataset_url)
+        response = {}
+        response["result"] = @dataset.get_dataset(commit: commit, query: query)
         dataset_name = response["result"]["name"]
         dataset_home = Dir.pwd+"/"+dataset_name
-        if !remote and (Dir.exists? dataset_name)
-          log_message("Error: Conflict with dir #{dataset_name}", Thor::Shell::Color::RED)
-          if no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
-            log_message("Remove dir in order to clone #{dataset_name}", Thor::Shell::Color::RED)
-            exit(1)
-          end
-        end
         check = Helpers.checkmark
-        if Dataset.clone(owner, dataset_name, slug, remote)
+        if @dataset.init_home
           log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
-          @files = Cnvrg::Datafiles.new(owner, slug, dataset: Cnvrg::Dataset.new(dataset_home, dataset_url: dataset_url))
+          @files = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
           log_message("Downloading files", Thor::Shell::Color::BLUE)
+          if @dataset.softlinked?
+            return @dataset.download_softlink
+          end
+
           if only_tree
+
             success = Dataset.clone_tree(commit: commit, dataset_home: dataset_home)
             return if success
           end
@@ -888,6 +924,9 @@ module Cnvrg
           log_message("Error: Couldn't create directory: #{dataset_name}", Thor::Shell::Color::RED)
           exit(1)
         end
+      rescue Interrupt
+        say "\nAborting", Thor::Shell::Color::RED
+        exit(1)
       rescue SignalException
         say "\nAborting", Thor::Shell::Color::RED
         exit(1)

data/lib/cnvrg/cli/flow.rb

@@ -34,23 +34,15 @@ module Cnvrg
 
 
       desc "flow run", "run a flow file"
-
-      def run(path)
+      def run(flow_slug)
         unless @curr_dir.present?
           @cli.log_message("Cant run this command because you are not in project directory", Thor::Color::RED)
           return false
         end
-        begin
-          path = "#{path}.flow.yaml" unless path.end_with? '.flow.yaml'
-          @flow = Cnvrg::Flows.new(@curr_dir, path)
-          url = @flow.run
-        rescue => e
-          @cli.log_message("Cant run the flow, please run 'cnvrg flow verify #{path}' to check it", Thor::Color::RED)
-          @cli.log_error(e)
-          return
-        end
-        check = Cnvrg::Helpers.checkmark
-        @cli.log_message("#{check} The flow is running, you can track the status in: #{url}", Thor::Color::GREEN)
+        @flow = Cnvrg::Flows.new(flow_slug)
+        resp = @flow.run
+        flow_version_href = resp["flow_version"]["href"]
+        Cnvrg::CLI.log_message("Flow Live results: #{flow_version_href}")
         true
       end
 

data/lib/cnvrg/data.rb

@@ -1,3 +1,6 @@
+require 'thread/pool'
+require 'thread/future'
+
 class SubCommandBase < Thor
  def self.banner(command, namespace = nil, subcommand = false)
   "#{basename}  #{command.usage}"
@@ -188,14 +191,16 @@ module Cnvrg
     end
 
 
-    desc 'data testclient', 'test client'
-    def test
+    desc 'data test', 'test client'
+    def test(data_url)
       cli = Cnvrg::CLI.new
       cli.verify_logged_in(true)
       cli.log_start(__method__, args, options)
-      dataset_dir = cli.is_cnvrg_dir(Dir.pwd)
-      @dataset = Project.new(dataset_dir)
+      @dataset = Dataset.new(dataset_url: data_url)
       resp = @dataset.get_storage_client
+      @dataset.init_home(remote: false)
+      @dataset.download_softlink
+
     end
 
   end

data/lib/cnvrg/dataset.rb

@@ -28,6 +28,52 @@ module Cnvrg
       end
     end
 
+    def init_home(remote: false)
+      dataset_home = Dir.pwd+"/"+@slug
+      if !remote and (Dir.exists? dataset_home)
+        Cnvrg::CLI.log_message("Error: Conflict with dir #{@slug}", Thor::Shell::Color::RED)
+        if Thor::Shell::Basic.new.no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
+          Cnvrg::CLI.log_message("Remove dir in order to clone #{@slug}", Thor::Shell::Color::RED)
+          exit(1)
+        end
+        FileUtils.rm_rf(dataset_home)
+      end
+      # if Dataset.clone(owner, dataset_name, slug, remote)
+      Dataset.clone(@owner, @slug, @slug, remote)
+      @local_path = dataset_home
+      Cnvrg::CLI.log_message('')
+      true
+    end
+
+    def get_dataset(commit: nil, query: nil)
+      if @dataset_call
+        return @dataset_call
+      end
+      response = Cnvrg::API.request("users/#{owner}/datasets/#{slug}/clone", 'POST',{ commit: commit, query:query})
+      Cnvrg::CLI.is_response_success(response,true)
+      @dataset_call = response["result"]
+      @dataset_call
+    end
+
+    def softlinked?
+      get_dataset["dataset_type"] == "soft_link_dataset"
+    end
+
+
+    def download_softlink
+      @storage = self.get_storage_client
+      @progressbar = ProgressBar.create(:title => "Download Progress",
+                                        :progress_mark => '=',
+                                        :format => "%b%i| %c Files downloaded",
+                                        :starting_at => 0,
+                                        :total => nil,
+                                        :autofinish => true)
+      @storage.cp(get_dataset["bucket_prefix"], @local_path, progress: @progressbar)
+      @progressbar.finish
+      Cnvrg::CLI.log_message("#{Cnvrg::Helpers.checkmark} Clone finished successfully", Thor::Shell::Color::GREEN)
+      write_success
+    end
+
 
     def get_storage_client
       response = Cnvrg::API.request("users/#{@owner}/datasets/#{@slug}/client", 'GET')
@@ -754,8 +800,6 @@ module Cnvrg
       FileUtils.rm_rf working_dir
       # response     = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/revert", 'GET')
       # CLI.is_response_success(response)
-
-
     end
 
   end

data/lib/cnvrg/downloader/clients/s3_client.rb

@@ -34,15 +34,17 @@ module Cnvrg
           end
         end
 
-        def download(storage_path, local_path)
+        def download(storage_path, local_path, decrypt: true)
           prepare_download(local_path)
-          storage_path = Cnvrg::Helpers.decrypt(@key, @iv, storage_path)
+          storage_path = Cnvrg::Helpers.decrypt(@key, @iv, storage_path) if decrypt
           resp = nil
           File.open(local_path, 'w+') do |file|
-            resp = @client.get_object({bucket: @bucket_name,
-                                       key: storage_path}, target: file)
+            resp = @client.get_object({bucket: @bucket_name, key: storage_path}, target: file)
           end
           resp
+        rescue => e
+          p e.message
+          p e.backtrace
         end
 
         def upload(storage_path, local_path)
@@ -51,6 +53,40 @@ module Cnvrg
           o = @bucket.object(storage_path)
           o.upload_file(local_path, @upload_options)
         end
+
+
+        def cp(prefix, local_path, progress: nil)
+          files = []
+          files_thread = Thread.new do
+            current_marker = nil
+            while true
+              batch_files = @bucket.objects(prefix: prefix, marker: current_marker).first(500)
+              batch_files = batch_files.to_a
+              break if batch_files.size === 0
+              files += batch_files
+              current_marker = batch_files.last.key
+            end
+          end
+          thread_pool = Thread.pool(Cnvrg::Helpers.parallel_threads)
+          while true
+            while files.present?
+              Thread.future thread_pool do
+                f = files.pop
+                cutted_key = cut_prefix(prefix, f.key)
+                dest_path = File.join(local_path, cutted_key)
+                download(f.key, dest_path, decrypt: false)
+                progress.increment
+              end
+            end
+            break if files.size == 0 and files_thread.status == false
+            sleep(1)
+          end
+          thread_pool.wait(:done)
+        end
+
+        def cut_prefix(prefix, file)
+          file.gsub(prefix, '').gsub(/^\/*/, '')
+        end
       end
     end
   end

data/lib/cnvrg/flow.rb

@@ -1,15 +1,15 @@
 module Cnvrg
   class Flows
-    def initialize(project_path, fullpath)
-      @project = Cnvrg::Project.new(project_path)
-      @fullpath = fullpath
+    def initialize(flow_slug)
+      @project = Cnvrg::Project.new(Cnvrg::CLI.get_project_home)
+      @flow_slug = flow_slug
       @tasks = {}
       @relations = {}
       @title = nil
       @slug = nil
       @base_resource = @project.base_resource + "flows"
 
-      self.reload_flow
+      # self.reload_flow
     end
 
     def self.create_flow(path, flow)
@@ -50,15 +50,25 @@ module Cnvrg
       end
     end
 
+
     def run
-      resp = Cnvrg::API.request(@base_resource, 'POST', {data: to_api})
-      Cnvrg::CLI.is_response_success(resp, true)
-      flow_slug = resp['result']['flow']
-      self.set_flow_slug(flow_slug)
-      url = Cnvrg::Helpers.remote_url + resp['result']['url']
-      return url
+      resp = Cnvrg::API.request("#{@base_resource}/#{@flow_slug}/run", 'POST')
+      if Cnvrg::CLI.is_response_success(resp)
+        return resp
+      end
+      Cnvrg::CLI.log_message("Cant run flow #{@flow_slug}")
     end
 
+    ### in use for yaml file
+    # def run
+    #   resp = Cnvrg::API.request(@base_resource, 'POST', {data: to_api})
+    #   Cnvrg::CLI.is_response_success(resp, true)
+    #   flow_slug = resp['result']['flow']
+    #   self.set_flow_slug(flow_slug)
+    #   url = Cnvrg::Helpers.remote_url + resp['result']['url']
+    #   return url
+    # end
+
 
     private
     def to_api

data/lib/cnvrg/project.rb

@@ -29,8 +29,7 @@ module Cnvrg
 
 
     def last_local_commit
-      idx = YAML.load_file(@local_path + "/.cnvrg/idx.yml")
-      return idx[:commit]
+      YAML.load_file(@local_path + "/.cnvrg/idx.yml")[:commit] rescue nil
     end
 
 

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '1.4.8'
+  VERSION = '1.4.9'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.4.8
+  version: 1.4.9
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-28 00:00:00.000000000 Z
+date: 2019-06-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -372,6 +372,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: thread
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A CLI tool for interacting with cnvrg.io.
 email:
 - info@cnvrg.io