cnvrg 0.7.7 → 0.7.9

data/lib/cnvrg/helpers.rb

@@ -19,6 +19,42 @@ module Cnvrg
       end
     end
 
+    def try_until_success(tries: 5)
+      (0..tries).each do |i|
+        begin
+          yield
+          return true
+        rescue Exception => e
+          Cnvrg::Logger.log_info("Error while trying for the #{i} time")
+          Cnvrg::Logger.log_error(e)
+          sleep(5)
+        rescue => e
+          Cnvrg::Logger.log_info("Error while trying for the #{i} time")
+          Cnvrg::Logger.log_error(e)
+          sleep(5)
+        end
+      end
+      raise StandardError.new("Failed")
+    end
+
+    def get_config
+      home_dir = File.expand_path('~')
+      config = {}
+      begin
+        if File.exist? home_dir+"/.cnvrg/config.yml"
+          config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
+        end
+      end
+      return config
+    end
+
+    def set_config(config)
+      home_dir = File.expand_path('~')
+      File.open("#{home_dir}/.cnvrg/config.yml", "w"){|f| f.write config.to_yaml }
+      return config
+    end
+
+
     def remote_url
       home_dir = File.expand_path('~')
       config = ""
@@ -38,6 +74,21 @@ module Cnvrg
         return config.to_h[:api].gsub("/api", "")
       end
     end
+
+    def server_version
+      config = self.get_config
+      config[:version].try(:to_i) || 0
+    end
+
+    def update_version(version)
+      config = self.get_config
+      if config[:version].to_s.eql? version
+        return
+      end
+      config[:version] = version
+      self.set_config(config)
+    end
+
     def is_verify_ssl
       home_dir = File.expand_path('~')
       config = ""
@@ -94,6 +145,7 @@ module Cnvrg
     end
 
     def cnvrgignore_content
+      #TODO: cnvrg ignore add .conflict
       %{
 # cnvrg ignore: Ignore the following directories and files
 # for example:
@@ -101,6 +153,8 @@ module Cnvrg
 # some_file.txt
 .git*
 .gitignore
+*.conflict
+*.deleted
             }.strip
     end
 

data/lib/cnvrg/hyper.rb

@@ -0,0 +1,21 @@
+module Cnvrg
+  class Hyper
+    def initialize(project_path, path)
+      @project = Cnvrg::Project.new(project_path)
+      @content = YAML.load_file(path)
+      @base_resource = "users/#{@project.owner}"
+      @params = []
+    end
+
+    def resolve_params
+      resp = Cnvrg::API.request(@base_resource + "/resolve_grid", "POST", {hyper_search: @content})
+      unless Cnvrg::CLI.is_response_success(resp, false)
+        return nil
+      end
+      resp['result']['params'].each do |param|
+        @params << {key: param.first.keys.first, value: param.map{|p| p.values}.flatten.join(',')}
+      end
+      @params
+    end
+  end
+end

data/lib/cnvrg/logger.rb

@@ -0,0 +1,102 @@
+module Cnvrg
+  module Logger
+    extend self
+    def log_handler
+      begin
+        date = DateTime.now.strftime("%m_%d_%Y")
+        home_dir = File.expand_path('~')
+
+        if !File.directory? home_dir + "/.cnvrg"
+          FileUtils.mkdir_p([home_dir + "/.cnvrg", home_dir + "/.cnvrg/tmp"])
+        end
+        if !File.exist?(home_dir + "/.cnvrg/config.yml")
+          FileUtils.touch [home_dir + "/.cnvrg/config.yml"]
+        end
+        logfile = File.expand_path('~') + "/.cnvrg/log_#{date}.log"
+        if !File.exist? logfile
+          FileUtils.touch([logfile])
+          yesterday = get_start_day - 86399
+          date = yesterday.strftime("%m_%d_%Y")
+
+          logfile_old = File.expand_path('~') + "/.cnvrg/log_#{date}.log"
+          count = 0
+          while not File.exist? logfile_old and count < 60
+            yesterday = yesterday - 86399
+            date = yesterday.strftime("%m_%d_%Y")
+            logfile_old = File.expand_path('~') + "/.cnvrg/log_#{date}.log"
+            count += 1
+          end
+          if File.exist? logfile_old
+            @files = Cnvrg::Files.new(Cnvrg::CLI.get_owner, "")
+            @files.upload_log_file(logfile_old, "log_#{date}.log", yesterday)
+            FileUtils.remove logfile_old
+          end
+
+        end
+        config = LogStashLogger.configure do |config|
+          config.customize_event do |event|
+            event.remove('@version')
+            event.remove('severity')
+          end
+        end
+        $log = LogStashLogger.new(type: :file, path: logfile, sync: true, config: config)
+        self.remove_old_log_files
+      rescue
+      end
+    end
+
+    def remove_old_log_files()
+      begin
+        last_week = (Time.now - (7 * 24 * 60 * 60)).strftime("%Y-%m-%d")
+        home = File.expand_path('~')
+        log_files = Dir["#{home}/.cnvrg/tmp/*.log"]
+        log_files.each do |l|
+          if File.mtime(l).strftime("%Y-%m-%d") < last_week
+            FileUtils.rm_rf(l)
+          end
+        end
+      end
+    end
+
+    def info(msg)
+      self.log_info(msg)
+    end
+
+    def log_method(bind: nil)
+      return if bind.blank?
+      Cnvrg::Logger.log_handler if $log.blank?
+      arg = {}
+      bind.local_variables.map do |name|
+        arg[name] = bind.local_variable_get(name).try(:to_s)
+      end
+      method = bind.eval('__method__')
+      $log.info method: method, args: arg
+    end
+
+    def log_error(e)
+      Cnvrg::Logger.log_handler if $log.blank?
+      bc = ActiveSupport::BacktraceCleaner.new
+      bc.add_silencer{|line| line =~ /thor/}
+      $log.error message: "An exception #{e.class} was logged during running", type: "error"
+      backtrace = bc.clean(e.backtrace).slice(0,6).map.with_index{|backtrace,idx| "(#{idx}) - #{backtrace}"}.join("; ")
+      $log.error message: e.message, type: "error", backtrace: backtrace
+    end
+
+    def log_error_message(msg)
+      Cnvrg::Logger.log_handler if $log.blank?
+      $log.error message: msg, type: "error"
+    end
+
+    def log_info(msg)
+      Cnvrg::Logger.log_handler if $log.blank?
+      $log.info message: msg, type: "info"
+    end
+
+    def jsonify_message(msg: '', success: true)
+      puts JSON[{
+          "msg": msg,
+          "success": success.to_s
+      }]
+    end
+  end
+end

data/lib/cnvrg/project.rb

@@ -1,7 +1,7 @@
 require 'fileutils'
 module Cnvrg
   class Project
-    attr_reader :slug, :owner, :title, :local_path, :working_dir, :is_git, :is_branch
+    attr_reader :slug, :owner, :title, :local_path, :working_dir, :is_git, :is_branch, :machines
 
     RemoteURL ||= "https://cnvrg.io"
     IDXParallelThreads ||= 15
@@ -17,10 +17,16 @@ module Cnvrg
         @owner = config[:owner]
         @is_branch = config[:is_branch]
         @is_git = config[:git] || false
+        @base_resource = "users/#{@owner}/projects/#{@slug}/"
+        @machines = nil
       rescue => e
       end
     end
 
+    def base_resource
+      "users/#{@owner}/projects/#{@slug}/"
+    end
+
 
     def last_local_commit
       idx = YAML.load_file(@local_path + "/.cnvrg/idx.yml")
@@ -164,7 +170,7 @@ module Cnvrg
         File.open(project_name + "/.cnvrgignore", "w+") { |f| f.write cnvrgignore }
         File.open(project_name + "/README.md", "w+") { |f| f.write cnvrgreadme }
         File.open(project_name + "/src/hyper.yaml", "w+") { |f| f.write cnvrghyper }
-        
+
       rescue
         return false
       end
@@ -343,8 +349,7 @@ module Cnvrg
         if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
           old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
         else
-
-          old_idx = nil
+          old_idx = {:tree => {}, :commit => nil}
         end
 
         tree_idx = Hash.new(0)
@@ -376,15 +381,25 @@ module Cnvrg
           end
         end
 
-        idx = {commit: old_idx.to_h[:commit], tree: tree_idx}
+        old_idx[:tree] = tree_idx
 
-        File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
+        File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write old_idx.to_yaml }
         return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
       end
       def get_idx
+        unless File.exists? "#{self.local_path}/.cnvrg/idx.yml"
+          empty_idx = {:commit => nil, :tree => {}}
+          File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write empty_idx.to_yaml }
+          return empty_idx
+        end
         YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
       end
 
+    def set_idx(idx)
+      FileUtils.mkdir_p("#{self.local_path}/.cnvrg")
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
+    end
+
       def clone(remote=0, commit)
         response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/clone", 'POST', {project_slug: self.slug, remote: remote, commit: commit})
         return response
@@ -395,34 +410,70 @@ module Cnvrg
       return response
     end
 
-      def compare_idx(new_branch, commit:last_local_commit,force:false, deploy: false, in_exp:false, specific_files: [])
-        if commit.nil?
-          local_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+      def compare_idx(new_branch, force:false, deploy: false, in_exp:false, specific_files: [], download: false)
+        is_download = download
+        if is_download
+          local_idx = self.get_idx
         else
+          #upload
           local_idx = self.generate_idx(deploy: deploy)
         end
+        commit = local_idx[:commit]
+        tree = local_idx[:tree]
         ignore_list = self.send_ignore_list()
-        if force or !specific_files.blank?
+        if force or specific_files.present?
           added = []
-          if !specific_files.blank?
+          if specific_files.present?
             added = specific_files
-          elsif local_idx[:tree]
+          elsif tree.present?
             added << local_idx[:tree].keys
             added.flatten!
           end
           response ={"result"=> {"commit"=>nil,"tree"=> {"added"=> added,
                      "updated_on_server"=> [],
                      "updated_on_local"=> [],
+                     "update_local" => [],
                      "deleted"=> [],
                      "conflicts"=> []} } }
           return response
-
         end
-        response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/status", 'POST', {idx: local_idx, new_branch: new_branch,
-                                                                                                   current_commit: commit,ignore:ignore_list, force:force,in_exp:in_exp})
+        #we dont want to send it on download - we only compare between commits sha1 in download.
+        if is_download
+          #the new server doesnt need the tree, but the old probably needs :X
+          local_idx[:tree] = {} if Cnvrg::Helpers.server_version > 0
+        end
+        response = Cnvrg::API.request(@base_resource + "status", 'POST', {idx: local_idx, new_branch: new_branch,
+                                                                                                   current_commit: commit,ignore:ignore_list, force:force,in_exp:in_exp, download: download})
+
         CLI.is_response_success(response,true)
+        if is_download
+          if Cnvrg::Helpers.server_version > 0
+            #trying to optimize the download using resolver
+            resolve = response['result']['tree']['resolver'] || tree #tree of file -> sha1 from current commit to check conflicts
+            destination_files = response['result']['tree']['destination'] || {} #tree of file -> sha1 from last commit to check files that already downloaded
+          else
+            resolve = tree
+            destination_files = {}
+          end
+          @files = self.get_files
+          local_tree = @files.calculate_sha1(resolve.keys)
+          changed_files = resolve.keys.select{|file| local_tree[file] != resolve[file]}
+
+          # means that the user changed the file locally
+          response['result']['tree']['update_local'] = changed_files
+
+          # means that we already downloaded this file and we dont need it anymore
+          downloaded_files = destination_files.keys.select{|file| local_tree[file] == destination_files[file]}
+          response['result']['tree']['added'] -= downloaded_files
+          response['result']['tree']['updated_on_server'] -= downloaded_files
+        end
         return response
       end
+
+    def get_files
+      Cnvrg::Files.new(self.owner, self.slug, project_home: @local_path)
+    end
+
     def jump_idx(destination: self.last_local_commit)
       local_idx = self.generate_idx
       ignore_list = self.send_ignore_list
@@ -497,7 +548,7 @@ module Cnvrg
       return response
     end
       def update_idx_with_commit!(commit)
-        idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+        idx_hash = self.get_idx
         idx_hash[:commit] = commit
 
         File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
@@ -506,11 +557,27 @@ module Cnvrg
 
       def revert(working_dir)
         FileUtils.rm_rf working_dir
-        # response     = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/revert", 'GET')
-        # CLI.is_response_success(response)
+      end
 
+    def init_machines
+      Cnvrg::Logger.log_info("Init machines.")
+      resp = Cnvrg::API.request("users/#{@owner}/machines", "GET")
+      return unless Cnvrg::CLI.is_response_success(resp, false)
+      @machines = resp['result']['machines']
+    end
 
-      end
+    def get_machines
+      init_machines if @machines.nil?
+      @machines = false if @machines.nil?
+      @machines || []
+    end
+
+    def check_machine(machine)
+      Cnvrg::Logger.log_info("Check if #{machine} machine exists")
+      machines = get_machines
+      return true if machines.blank?
+      machines.include? machine
+    end
 
     end
   end

data/lib/cnvrg/task.rb

@@ -0,0 +1,165 @@
+module Cnvrg
+  class Task
+    attr_accessor :title, :path, :source
+    def initialize(project_path, path: nil, content: {})
+      @path = path
+      @content = content
+      @source = 'flow' if @content.present?
+      @source = 'file' if @path.present?
+      @project = Cnvrg::Project.new(project_path)
+      @project_path = project_path
+      #generic
+      @type = nil #type can be exec, data, library, deploy
+      @compute = 'medium'
+      @title = nil
+      @uid = nil
+      @params = {}
+
+      #exec
+      @cmd = nil
+      @params_path = nil
+
+      #library
+      @library = nil
+
+      #dataset
+      @dataset = nil
+      @query = nil
+
+      #deploy
+      @function = nil
+
+
+      @base_resource = @project.base_resource + "/tasks"
+
+      self.reload_task
+    end
+
+    def save
+      path = @path || gen_path
+      File.open(path, 'w'){|f| f.write(get_content.to_yaml)}
+    end
+
+    def reload_task
+      task_raw = get_content
+      @title = task_raw[:title]
+      @type = task_raw[:type]
+      @uid = task_raw[:uid]
+      @title = task_raw[:title] || @uid
+      @compute = task_raw[:compute] || @compute
+
+      case @type
+      when 'exec'
+        @cmd = task_raw[:cmd]
+        init_params(task_raw)
+      when 'library'
+        @library = task_raw[:library]
+        init_params(task_raw)
+      when 'data'
+        @dataset = task_raw[:dataset]
+        @query = task_raw[:query]
+      when 'deploy'
+        @cmd = task_raw[:cmd]
+        @function = task_raw[:function]
+      else
+        error("Cant parse task of type #{@type}")
+      end
+    end
+
+    def verify_task
+      case @type
+      when 'exec'
+        verify_exec
+      when 'data'
+        verify_data
+      when 'deploy'
+        verify_deploy
+      when 'library'
+        verify_library
+      else
+        error("Cant parse task of type #{@type}")
+      end
+    end
+
+
+    def to_api
+      get_content.merge(params: @params)
+    end
+
+    def run
+      verify_task
+      if @type == 'data'
+        raise StandardError.new("Data Tasks are not runnable")
+      end
+      resp = Cnvrg::API.request(@base_resource, "POST", {task: to_api})
+      Cnvrg::CLI.is_response_success(resp, true)
+      Cnvrg::Helpers.remote_url + resp['result']['url']
+    end
+
+    private
+    def verify_compute
+      unless @project.check_machine(@compute)
+        raise StandardError.new("Cant find #{@compute} machine in project.")
+      end
+    end
+
+    def init_params(task_raw)
+      @params_path = task_raw[:params_path].presence
+      @params = task_raw[:params] || @params
+      if @params_path.present?
+        @hyper = Cnvrg::Hyper.new(@project_path, @params_path)
+        @params = @hyper.resolve_params
+      end
+    end
+
+    def verify_exec
+      if @cmd.blank?
+        error("Cant find command")
+      end
+      verify_compute
+    end
+
+    def verify_data
+      if @dataset.blank?
+        error("Cant find dataset slug")
+      end
+    end
+
+    def verify_deploy
+      error("Cant find command") if @cmd.blank?
+      error("Cant find function") if @function.blank?
+      verify_compute
+    end
+
+    def verify_library
+      error("Cant find library") if @library.blank?
+    end
+
+    def get_content
+      return @content if @source == 'flow'
+      unless File.exists? @path
+        raise StandardError.new("Cant find task in #{@path}")
+      end
+      YAML.load_file(@path)
+    end
+
+    def gen_path
+      @title ||= "#{@type.capitalize}Task"
+      unless File.exists? @title
+        @path = "#{@title}.task.yaml"
+        return @path
+      end
+      i = 0
+      while File.exists? "#{@title}_#{i}.task.yaml"
+        i += 1
+      end
+      @path = "#{@title}.task.yaml"
+      return @path
+    end
+
+    def error(msg)
+      raise StandardError.new("task: #{@uid} - #{msg}")
+    end
+
+  end
+end