cnvrg 0.7.7 → 0.7.9

data/lib/cnvrg/cli/flow.rb

@@ -0,0 +1,116 @@
+require 'cnvrg/cli'
+require 'cnvrg/data'
+require 'cnvrg/task'
+require 'cnvrg/flow'
+require 'cnvrg/hyper'
+require 'cnvrg/cli/subcommand'
+require 'cnvrg/cli/task'
+require 'thor'
+module Cnvrg
+  module Commands
+    class Flow < SubCommand
+
+      def initialize(*args)
+        super
+        unless @curr_dir.present?
+          @cli.log_message("Not On Project Dir, exiting", Thor::Color::RED)
+          exit(1)
+        end
+      end
+
+      desc "task", "Running Flow tasks", :hide => true
+      subcommand 'task', Cnvrg::Commands::Task
+
+      desc "flow verify", "verify that the flow is well formated"
+
+      def verify(path)
+        unless @curr_dir.present?
+          @cli.log_message("Cant run this command because you are not in project directory", Thor::Color::RED)
+          return false
+        end
+        @flow = Cnvrg::Flows.new(@curr_dir, path)
+        @cli.log_message("The Flow is Valid", Thor::Color::GREEN)
+      end
+
+
+      desc "flow run", "run a flow file"
+
+      def run(path)
+        unless @curr_dir.present?
+          @cli.log_message("Cant run this command because you are not in project directory", Thor::Color::RED)
+          return false
+        end
+        begin
+          @flow = Cnvrg::Flows.new(@curr_dir, path)
+          url = @flow.run
+        rescue => e
+          @cli.log_message("Cant run the flow, please run 'cnvrg flow verify #{path}' to check it", Thor::Color::RED)
+          @cli.log_error(e)
+          return
+        end
+        check = Cnvrg::Helpers.checkmark
+        @cli.log_message("#{check} The flow is running, you can track the status in: #{url}", Thor::Color::GREEN)
+        true
+      end
+
+      desc "flow create", "create a flow file"
+
+      def create
+        title = ask("Flow Title: ")
+        title = title.presence || "cnvrg-flow"
+        relations = []
+        task = ask_for_relation
+        while task.compact.size == 2
+          from, to = task
+          relations << {from: from, to: to}
+          task = ask_for_relation
+        end
+        start_commit = ask("Flow Starting Commit (empty will set this value to 'latest')")
+        Cnvrg::Flows.create_flow("#{title}.flow.yaml", {title: title, relations: relations, start_commit: start_commit})
+      end
+
+
+      # desc "flow resolve", "Resolve flow parameters"
+      # def resolve(path)
+      #   @hyper = Cnvrg::Hyper.new(@curr_dir, path)
+      #   @hyper.resolve_params
+      # end
+      #
+
+      private
+
+      def init_tasks
+        @tasks = Dir.glob("**/*.task*")
+      end
+
+      def ask_for_relation
+        init_tasks if @tasks.blank?
+        to = nil
+        from = ask_for_task("Task To Start From: [#{@tasks.join(', ')}]")
+        to = ask_for_task("Task To Go To: [#{@tasks.join(', ')}]") if from.present?
+        [from, to]
+      end
+
+      def ask_for_task(text)
+        verified = false
+        task = nil
+        while !verified
+          task = ask(text)
+          if task.blank?
+            return nil
+          end
+          begin
+            Cnvrg::Task.new(@curr_dir, path: task).verify_task
+            verified = true
+          rescue => e
+          end
+        end
+        task
+      end
+
+      def get_all_tasks
+        @tasks = Dir.glob("*/**.task.yaml")
+      end
+    end
+  end
+end

data/lib/cnvrg/cli/subcommand.rb

@@ -0,0 +1,26 @@
+class SubCommand < Thor
+  @cli = nil
+  @curr_dir = nil
+
+  def initialize(*args)
+    super
+    @cli = Cnvrg::CLI.new
+    @curr_dir = @cli.is_cnvrg_dir
+  end
+
+  def self.banner(command, namespace = nil, subcommand = false)
+    "#{basename}  #{command.usage}"
+  end
+
+  def self.subcommand_prefix
+    self.name.gsub(%r{.*::}, '').gsub(%r{^[A-Z]}) {|match| match[0].downcase}.gsub(%r{[A-Z]}) {|match| "-#{match[0].downcase}"}
+  end
+
+  class << self
+    # Hackery.Take the run method away from Thor so that we can redefine it.
+    def is_thor_reserved_word?(word, type)
+      return false if word == "run"
+      super
+    end
+  end
+end

data/lib/cnvrg/cli/task.rb

@@ -0,0 +1,116 @@
+module Cnvrg
+  module Commands
+    class Task < SubCommand
+
+      def initialize(*args)
+        super
+        unless @curr_dir.present?
+          @cli.log_message("Not On Project Dir, exiting", Thor::Color::RED)
+          exit(1)
+        end
+      end
+
+      desc "flow task verify", "verify that the task is well formated"
+      def verify(path)
+        unless @curr_dir.present?
+          @cli.log_message("Cant run this command because you are not in project directory", Thor::Color::RED)
+          return false
+        end
+        @task = Cnvrg::Task.new(@curr_dir, path: path)
+        begin
+          @task.verify_task
+        rescue StandardError => e
+          @cli.log_message("An error during parsing task: #{e.message}", Thor::Color::RED)
+          @cli.log_error(e)
+          return false
+        rescue => e
+          @cli.log_error(e)
+          return false
+        end
+        @cli.log_message("The task verified successfuly", Thor::Color::GREEN)
+        return @task
+      end
+
+
+      desc "flow task create", "Creates new task"
+      def create
+
+        @project = Cnvrg::Project.new(@curr_dir)
+
+        task = {}
+        task[:title] = ask("Task title", Thor::Color::GREEN)
+        task[:type] = ask("Task Type?", Thor::Color::GREEN, :limited_to => ["exec", "data", "library", "deploy"])
+        case task[:type]
+        when "exec"
+          task[:cmd] = ask("Command to run", Thor::Color::GREEN)
+          task[:params], task[:params_path] = ask_for_params
+          task[:machine] = ask_for_machine
+        when "data"
+          task[:dataset] = ask("Dataset slug", Thor::Color::GREEN)
+          task[:query] = ask("Dataset query", Thor::Color::GREEN)
+        when "library"
+          task[:library] = ask("Library Slug", Thor::Color::GREEN)
+          task[:params], task[:params_path] = ask_for_params
+          task[:machine] = ask_for_machine
+        when "deploy"
+          task[:cmd] = ask("Command to run", Thor::Color::GREEN)
+          task[:function] = ask("Function to run", Thor::Color::GREEN)
+        end
+        @task = Cnvrg::Task.new(@curr_dir, content: task)
+        @task.save
+        @cli.log_message("Task #{@task.title} Saved Successfuly", Thor::Color::GREEN)
+      end
+
+
+      desc "flow task run", "Running specific task"
+      def run(path)
+        begin
+          @task = Cnvrg::Task.new(@curr_dir, path: path)
+          url = @task.run
+          @cli.log_message("Task: #{@task.title} is running, you can track its performance on #{url}", Thor::Color::GREEN)
+        rescue => e
+          @cli.log_message(e.message, Thor::Color::RED)
+          @cli.log_error(e)
+        end
+      end
+
+      private
+
+      def ask_for_params
+        params = []
+        param_key = ask("Param key: (enter for continue)", Thor::Color::GREEN)
+        while param_key.present?
+          param_value = ask("#{param_key} value: (seperated by commas)", Thor::Color::GREEN)
+          params << {key: param_key, value: param_value}
+          param_key = ask("Param key: (enter for continue)", Thor::Color::GREEN)
+        end
+        if params.blank?
+          params_path = ask("Params from file: ")
+          if params_path.present?
+            while params_path.present? and !File.exists? params_path
+              params_path = ask("Cant find file, please write it again.")
+            end
+          end
+        end
+        return params, params_path
+      end
+
+      def ask_for_machine
+        options = {}
+        all_machines = @project.get_machines
+        options[:limited_to] = all_machines if all_machines.present?
+        ask("Kind of machine to execute on?", Thor::Color::GREEN, options).presence || "medium"
+      end
+
+
+      def on_project
+        unless @curr_dir.present?
+          @cli.log_message("Cant run this command because you are not in project directory", Thor::Color::RED)
+          return false
+        end
+      end
+
+
+    end
+  end
+end

data/lib/cnvrg/data.rb

@@ -11,7 +11,6 @@ class SubCommandBase < Thor
 end
 
 module Cnvrg
-
   class Data < SubCommandBase
     class_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
     desc "data init", "init data folder"

data/lib/cnvrg/datafiles.rb

@@ -81,46 +81,62 @@ module Cnvrg
     end
 
 
-    def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, progressbar: nil, prefix: '')
-      random_file_name = (0...8).map { (65 + rand(26)).chr }.join #needed to create a file for post_file..
-      # each file have: {sha1: sha1, file_name: file_name, file_size: file_size, content_type: content_type, absolute_path, relative_path} 
-
-      #this call should also implement compare_idx...
+    def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, progressbar: nil, prefix: '', partial_commit: nil)
       begin
-      upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {commit_sha1: commit_sha1, tree: tree, force: force, is_branch: new_branch})
-      return Cnvrg::Result.new(false, "Failed to upload files") unless Cnvrg::CLI.is_response_success(upload_resp, false)
-      results = upload_resp['result'].with_indifferent_access
-      if results['files'].blank?
-        progressbar.progress += tree.keys.length if progressbar.present?
-        return 0
-      end
-      props = Cnvrg::Helpers.get_s3_props(results)
-      client = props[:client]
-      bucket = props[:bucket]
-      upload_options = props[:upload_options]
-      s3_bucket = Aws::S3::Resource.new(client: client).bucket(bucket)
-      files = results['files']
-      progressbar.progress += tree.keys.length - files.length if progressbar.present?
-      Parallel.map((files.keys), {in_threads: threads}) do |k|
-        o = tree[k].merge(files[k])
-        upload_single_file(o, s3_bucket, upload_options)
-        progressbar.progress += 1 if progressbar.present?
-      end
-      blob_ids = files.values.map{|f| f['bv_id']}
-      dirs = tree.keys.select{|k| tree[k].nil? } || []
-      upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
-      return files.keys.length
+        Cnvrg::Logger.log_info("Sending Upload Files request")
+        upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {commit_sha1: commit_sha1, tree: tree, force: force, is_branch: new_branch, partial_commit: partial_commit})
+        unless Cnvrg::CLI.is_response_success(upload_resp, false)
+          Cnvrg::Logger.log_method(bind: binding)
+          raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+        end
+        Cnvrg::Logger.log_info("Uploading files")
+        results = upload_resp['result'].with_indifferent_access
+        if results['files'].blank?
+          progressbar.progress += tree.keys.length if progressbar.present?
+          return 0
+        end
+        props = Cnvrg::Helpers.get_s3_props(results)
+        client = props[:client]
+        bucket = props[:bucket]
+        upload_options = props[:upload_options]
+        s3_bucket = Aws::S3::Resource.new(client: client).bucket(bucket)
+        files = results['files']
+        progressbar.progress += tree.keys.length - files.length if progressbar.present?
+        Parallel.map((files.keys), {in_threads: threads}) do |k|
+          o = tree[k].merge(files[k])
+          upload_single_file(o, s3_bucket, upload_options)
+          progressbar.progress += 1 if progressbar.present?
+        end
+        blob_ids = files.values.map {|f| f['bv_id']}
+        dirs = tree.keys.select {|k| tree[k].nil?} || []
+        Cnvrg::Logger.info("Sending Upload files save")
+        upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
+        unless Cnvrg::CLI.is_response_success(upload_resp, false)
+          Cnvrg::Logger.log_method(bind: binding)
+          raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+        end
+        Cnvrg::Logger.log_info("Upload Success")
+        return files.keys.length
       rescue => e
-        return Cnvrg::Result.new(false, "Failed to upload files")
+        Cnvrg::Logger.log_method(bind: binding)
+        Cnvrg::Logger.log_error(e)
+        raise e
       end
 
     end
 
     def upload_single_file(file, s3_bucket, upload_options={})
+      begin
       file = file.with_indifferent_access
-      resp = s3_bucket.
-        object(file[:path]).
-        upload_file(file[:absolute_path], upload_options)
+      Cnvrg::Logger.log_info("Uploading #{file[:absolute_path]}")
+      obj = s3_bucket.
+        object(file[:path])
+      obj.upload_file(file[:absolute_path], upload_options)
+      Cnvrg::Logger.log_info("#{file[:absolute_path]} uploaded.")
+      rescue => e
+        Cnvrg::Logger.log_error_message("Error while upload single file #{file[:path]}")
+        Cnvrg::Logger.log_error(e)
+      end
     end
 
     def upload_file(absolute_path, relative_path, commit_sha1)
@@ -809,10 +825,13 @@ module Cnvrg
       Cnvrg::CLI.is_response_success(response, true)
       return response
     end
-    def start_commit(new_branch,force=false,delete_commit=nil, chunks: 0, current_commit: nil)
+    def start_commit(new_branch,force=false,delete_commit=nil, chunks: 0, dataset: nil)
       begin
+        idx = dataset.get_idx
+        commit = idx[:commit]
+        next_commit = idx[:next_commit]
       response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch,force:force,
-                                                                              username: @owner,current_commit:current_commit, total_chunks: chunks})
+                                                                              username: @owner,current_commit: commit, next_commit: next_commit, total_chunks: chunks})
       Cnvrg::CLI.is_response_success(response, true)
       return response
       rescue => e
@@ -944,11 +963,12 @@ module Cnvrg
             file_path = "#{file_path}.conflict" if conflict
             dirname = File.dirname file_path
             download_dir(project_home, dirname) unless dirname.eql? "."
-
+            Cnvrg::Logger.info("Download #{file_path}")
             File.open(project_home+"/"+file_path, 'w+') do |file|
               resp = client.get_object({bucket:bucket,
                                         key:file_key}, target: file)
             end
+            Cnvrg::Logger.log_info("Download #{file_path} success")
 
             if resp
               download_succ_count +=1

data/lib/cnvrg/dataset.rb

@@ -20,6 +20,20 @@ module Cnvrg
       end
     end
 
+
+    def backup_idx
+      Cnvrg::Logger.log_info("Backup idx")
+      idx = self.get_idx
+      File.open("#{self.local_path}/.cnvrg/idx.yml.backup", 'w') { |f| f.write idx.to_yaml }
+    end
+
+    def restore_idx
+      Cnvrg::Logger.log_info("Restore idx because an error.")
+      Cnvrg::Logger.log_method(bind: binding)
+      idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml.backup")
+      self.set_idx(idx)
+    end
+
     def change_url(owner: '', slug: '', title: '')
       config = {dataset_home: title, dataset_slug: slug, owner: owner}
       File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
@@ -293,9 +307,18 @@ module Cnvrg
     end
 
 
-      def get_idx
-        YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+    def get_idx
+      if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
+        return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
       end
+      {commit: nil, tree: {}}
+
+    end
+
+    def set_idx(idx)
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w+') { |f| f.write idx.to_yaml }
+    end
+
     def url
       url = Cnvrg::Helpers.remote_url
       "#{url}/#{self.owner}/datasets/#{self.slug}"
@@ -342,8 +365,9 @@ module Cnvrg
       self.revert_next_commit
     end
 
-    def list_all_files
+    def list_all_files(with_ignore=false)
       list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~/^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~/^#{self.local_path}\/\.cnvrgignore/) }
+      return list if with_ignore
       list_ignore = self.get_ignore_list.map{|ignore_file| "#{self.local_path}/#{ignore_file}"}
       list.select{|file| !list_ignore.include? file}
     end
@@ -357,6 +381,12 @@ module Cnvrg
       File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
     end
 
+    def write_tree(tree)
+      idx = self.get_idx
+      idx[:tree] = tree
+      self.set_idx(idx)
+    end
+
     def generate_idx(show_progress=false)
       if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
         old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
@@ -454,10 +484,23 @@ module Cnvrg
     
     def compare_idx_download(all_files: false, desired_commit: nil)
       current_commit = self.last_local_commit
+      next_commit = self.get_next_commit
       ignore_list = self.send_ignore_list()
-      return Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/download_status", 'POST', {current_commit: current_commit, ignore:ignore_list,all_files:all_files, desired_commit: desired_commit.presence})
+      return Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/download_status", 'POST', {current_commit: current_commit, next_commit: next_commit, ignore:ignore_list,all_files:all_files, desired_commit: desired_commit.presence})
     end
-    
+
+    def set_partial_commit(commit_sha1)
+      idx = self.get_idx
+      idx[:partial_commit] = commit_sha1
+      self.set_idx(idx)
+    end
+
+    def get_partial_commit
+      idx = self.get_idx
+      idx.try(:fetch, :partial_commit)
+    end
+
+
     def current_status(new_branch)
         commit=last_local_commit
         response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status_current", 'POST', {current_commit: commit,new_branch: new_branch})
@@ -533,10 +576,8 @@ module Cnvrg
         return nil
       end
       idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
-      idx = Hash.new()
-      idx[:commit] = idx_hash[:commit]
-      idx[:tree] = idx_hash[:tree]
-      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
+      idx_hash = idx_hash.except(:next_commit)
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
     end
     def get_current_commit()
       if  !File.exist? "#{self.local_path}/.cnvrg/idx.yml"