cnvrg 1.6.0.8 → 1.6.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 319b1e08fd56c2a26834b3a2e2fc94565e2abad1d0244c72e4e45c11a8cc7d3d
-  data.tar.gz: 60edde88d619c8aae0afded8314c2688545712445718e99576a7ec20222d795e
+  metadata.gz: 3a578311b787d26b7a510ce7c6ff9b845d55300b82a5c75e70a787e334693779
+  data.tar.gz: 7bea7b961282523b23dfbe01014d658147dfa354240246ff4722e165d5603895
 SHA512:
-  metadata.gz: 9dc2c22f8c1f4c696c0580f2ab2c3122d57061a95591bff13403a2e93412f6268b50a23119ba5ccbaa0830bd8f87474490be50ccf6d128c378b39d4a1cffaca3
-  data.tar.gz: 4628fee3352fb89fb639ed3aef3e76ce2cadff9c916d153a940416d25ac81f8e700b2891cca6682c95e9526342c089ef631f2c16d9df9ca84cc4faccaac70e28
+  metadata.gz: f8d23528d8fe271b4f4dde1c585b2a0991e8aaf4445a4b33436355caaed5dff153dd72afa1574123de627a894d38ba78157caa79ba77ba4e52f13ba92e0c6c45
+  data.tar.gz: 1cfa9cddbd3af489be308bf496f0e4b1079a9a007eafe97cf5255b31187fbb15ab9978797b2a8fbec3b8f4b08c8a74ec042a7088f6e3b799cf0a41e80792c5f3

data/lib/cnvrg/api.rb

@@ -30,6 +30,7 @@ module Cnvrg
             end
         end
         def self.request(resource, method = 'GET', data = {}, parse_request = true)
+            resource = URI::encode resource
             begin
                 n = Netrc.read
             rescue => e
@@ -163,22 +164,25 @@ module Cnvrg
 
                 # what if windows?
                 # data[:file] = Faraday::UploadIO.new(data[:absolute_path], content_type)
+                if not File.exists? data[:relative_path]
                   file_base = File.basename(data[:relative_path])
-                begin
-                    temp_path = File.expand_path('~')+"/.cnvrg/tmp_files/#{file_base}"
-                    FileUtils.touch(temp_path)
-                rescue
-                    temp_path ="/tmp/#{file_base}"
-                    FileUtils.touch(temp_path)
-                end
 
+                  begin
+                      temp_path = File.expand_path('~')+"/.cnvrg/tmp_files/#{file_base}"
+                      FileUtils.touch(temp_path)
+                  rescue
+                      temp_path ="/tmp/#{file_base}"
+                      FileUtils.touch(temp_path)
+                  end
+                else
+                  temp_path = data[:relative_path]
+                end
 
 
-                data[:file] = Faraday::UploadIO.new("#{temp_path}", "plain/text")
+                data[:file] = Faraday::UploadIO.new("#{temp_path}", "application/tar+gzip")
 
                 response = conn.post "#{endpoint_uri}/#{resource}", data
                 Cnvrg::API.parse_version(response)
-
                 FileUtils.rm_rf(temp_path)
                 if response.to_hash[:status] == 404
                   return false

data/lib/cnvrg/cli.rb

@@ -41,6 +41,7 @@ require 'cnvrg/org_helpers'
 require 'cnvrg/cli/subcommand'
 require 'cnvrg/cli/flow'
 require 'cnvrg/cli/task'
+require 'cnvrg/cli/library_cli'
 require 'cnvrg/image_cli'
 require 'cnvrg/helpers/executer'
 require 'cnvrg/downloader/client'
@@ -177,6 +178,9 @@ module Cnvrg
     desc "image [COMMAND]", "build existing images", :hide => true
     subcommand "image", ImageCli
 
+    desc "library [COMMAND]", "Upload and manage datasets", :hide => false
+    subcommand "library", LibraryCli
+
 
     desc "flow", "mange project flows", :hide => true
     subcommand "flow", Cnvrg::Commands::Flow
@@ -944,8 +948,7 @@ module Cnvrg
         Cnvrg::CLI.is_response_success(response,true)
         dataset_name = response["results"]["name"]
         dataset_slug = response["results"]["slug"]
-        # dataset_home = Dir.pwd+"/"+dataset_name
-        dataset_home = Dir.pwd
+        dataset_home = File.join(Dir.pwd, dataset_name)
 
         if Dataset.blank_clone(owner, dataset_name, dataset_slug)
           dataset = Dataset.new(dataset_home)
@@ -1990,6 +1993,7 @@ module Cnvrg
         end
       end
       rescue => e
+        Cnvrg::CLI.log_message(e.message, 'red')
         Cnvrg::Logger.log_error(e)
         say "\nAborting", Thor::Shell::Color::BLUE
         dataset_dir = is_cnvrg_dir(Dir.pwd)
@@ -3124,7 +3128,7 @@ module Cnvrg
                 end
               end
               start_time = Time.now
-              PTY.spawn(cmd) do |stdout, stdin, pid, stderr|
+              PTY.spawn(@exp.as_env, cmd) do |stdout, stdin, pid, stderr|
                 begin
                   stdout.each do |line|
                     cur_time = Time.now
@@ -3189,7 +3193,7 @@ module Cnvrg
               end
 
                 if sync_after
-                  @project.job_log(["Syncing Experiment"])
+                  @exp.job_log(["Syncing Experiment"])
                 # Sync after run
                   if @project.is_git
                     output_dir = output_dir || @exp.output_dir
@@ -3206,7 +3210,7 @@ module Cnvrg
                 end
               end_commit = @project.last_local_commit
               if end_commit.present?
-                @project.job_log(["Experiment end commit: #{end_commit}"])
+                @exp.job_log(["Experiment end commit: #{end_commit}"])
               end
 
                 # log_thread.join

data/lib/cnvrg/cli/flow.rb

@@ -18,92 +18,149 @@ module Cnvrg
         end
       end
 
-      desc "task", "Running Flow tasks", :hide => true
-      subcommand 'task', Cnvrg::Commands::Task
-
-      desc "flow verify", "verify that the flow is well formatted"
-
-      def verify(path)
-        unless @curr_dir.present?
-          @cli.log_message("Cant run this command because you are not in project directory", Thor::Color::RED)
-          return false
-        end
-        @flow = Cnvrg::Flows.new(@curr_dir, path)
-        @cli.log_message("The Flow is Valid", Thor::Color::GREEN)
+      desc "flow import --file='flow.yaml'", "Import flow to the web"
+      method_option :file, :type => :string, :aliases => ["-f", "--file"]
+      def import()
+        run_flow_internal(file: options[:file], run: false)
       end
 
+      desc "flow run --file='flow.yaml'", "Import flow to the web"
+      method_option :file, :type => :string, :aliases => ["-f", "--file"]
+      def run()
+        run_flow_internal(file: options[:file], run: true)
+      end
 
-      desc "flow run", "run a flow file"
-      def run(flow_slug)
-        unless @curr_dir.present?
-          @cli.log_message("Cant run this command because you are not in project directory", Thor::Color::RED)
-          return false
+      no_commands {
+      def run_flow_internal(file: nil, run: false)
+        if options[:file].blank?
+          Cnvrg::CLI.log_message("Flow title is required for export. please use --title='Flow 1'")
+          return
+        end
+        if not File.exists? options[:file]
+          Cnvrg::CLI.log_message("Cant find file in path #{options[:file]}.")
+          return
+        end
+        payload = YAML.safe_load(File.open(file).read)
+        @project = Cnvrg::Project.new(Cnvrg::CLI.get_project_home)
+        begin
+          @flow, version_slug = Flows.create_flow(@project, payload, run: run)
+        rescue => e
+          Cnvrg::CLI.log_message("Error while validating flow: #{e.message}", 'error')
+          return
         end
-        @flow = Cnvrg::Flows.new(flow_slug)
-        resp = @flow.run
-        flow_version_href = resp["flow_version"]["href"]
-        Cnvrg::CLI.log_message("Flow Live results: #{flow_version_href}")
-        true
+        Cnvrg::CLI.log_message("New flow version was created successfuly!", 'green')
+        Cnvrg::CLI.log_message("you can find the flow in #{@flow.edit_version_href(version_slug)}", 'green')
+      rescue => e
+        Cnvrg::CLI.log_message("Failed while creating flow")
+        Cnvrg::Logger.log_error(e)
       end
+      }
 
-      desc "flow create", "create a flow file"
-
-      def create
-        title = ask("Flow Title: ")
-        title = title.presence || "cnvrg-flow"
-        relations = []
-        task = ask_for_relation
-        while task.compact.size == 2
-          from, to = task
-          relations << {from: from, to: to}
-          task = ask_for_relation
+      desc "flow export --file='flow.yaml' --title='Flow 1' --version='version 1'", "Export flow to the web"
+      method_option :file, :type => :string, :aliases => ["-f", "--file"]
+      method_option :title, :type => :string, :aliases => ["-t", "--title"]
+      method_option :version, :type => :string, :aliases => ["-v", "--version"], :default => "latest"
+      def export()
+        if options[:title].blank?
+          Cnvrg::CLI.log_message("Flow title is required for export. please use --title='Flow 1'", "red")
+          return
         end
-        start_commit = ask("Flow Starting Commit (empty will set this value to 'latest')")
-        Cnvrg::Flows.create_flow("#{title}.flow.yaml", {title: title, relations: relations, start_commit: start_commit})
-      end
 
+        @project = Cnvrg::Project.new(Cnvrg::CLI.get_project_home)
+        @flow = Flows.new(options[:title], project: @project)
+        filename = @flow.export(options[:version], file: options[:file])
+        Cnvrg::CLI.log_message("Flow was saved successfuly to: #{filename}", 'green')
+      rescue => e
+        Cnvrg::CLI.log_message(e.message, 'red')
+        Cnvrg::Logger.log_error(e)
+      end
 
-      # desc "flow resolve", "Resolve flow parameters"
-      # def resolve(path)
-      #   @hyper = Cnvrg::Hyper.new(@curr_dir, path)
-      #   @hyper.resolve_params
+      # desc "task", "Running Flow tasks", :hide => true
+      # subcommand 'task', Cnvrg::Commands::Task
+      #
+      # desc "flow verify", "verify that the flow is well formatted"
+      #
+      # def verify(path)
+      #   unless @curr_dir.present?
+      #     @cli.log_message("Cant run this command because you are not in project directory", Thor::Color::RED)
+      #     return false
+      #   end
+      #   @flow = Cnvrg::Flows.new(@curr_dir, path)
+      #   @cli.log_message("The Flow is Valid", Thor::Color::GREEN)
       # end
       #
-
-      private
-
-      def init_tasks
-        @tasks = Dir.glob("**/*.task*")
-      end
-
-      def ask_for_relation
-        init_tasks if @tasks.blank?
-        to = nil
-        from = ask_for_task("Task To Start From: [#{@tasks.join(', ')}]")
-        to = ask_for_task("Task To Go To: [#{@tasks.join(', ')}]") if from.present?
-        [from, to]
-      end
-
-      def ask_for_task(text)
-        verified = false
-        task = nil
-        while !verified
-          task = ask(text)
-          if task.blank?
-            return nil
-          end
-          begin
-            Cnvrg::Task.new(@curr_dir, path: task).verify_task
-            verified = true
-          rescue => e
-          end
-        end
-        task
-      end
-
-      def get_all_tasks
-        @tasks = Dir.glob("*/**.task.yaml")
-      end
-    end
+      #
+      # desc "flow run", "run a flow file"
+      # def run(flow_slug)
+      #   unless @curr_dir.present?
+      #     @cli.log_message("Cant run this command because you are not in project directory", Thor::Color::RED)
+      #     return false
+      #   end
+      #   @flow = Cnvrg::Flows.new(flow_slug)
+      #   resp = @flow.run
+      #   flow_version_href = resp["flow_version"]["href"]
+      #   Cnvrg::CLI.log_message("Flow Live results: #{flow_version_href}")
+      #   true
+      # end
+      #
+      # desc "flow create", "create a flow file"
+      #
+      # def create
+      #   title = ask("Flow Title: ")
+      #   title = title.presence || "cnvrg-flow"
+      #   relations = []
+      #   task = ask_for_relation
+      #   while task.compact.size == 2
+      #     from, to = task
+      #     relations << {from: from, to: to}
+      #     task = ask_for_relation
+      #   end
+      #   start_commit = ask("Flow Starting Commit (empty will set this value to 'latest')")
+      #   Cnvrg::Flows.create_flow("#{title}.flow.yaml", {title: title, relations: relations, start_commit: start_commit})
+      # end
+      #
+      #
+      # # desc "flow resolve", "Resolve flow parameters"
+      # # def resolve(path)
+      # #   @hyper = Cnvrg::Hyper.new(@curr_dir, path)
+      # #   @hyper.resolve_params
+      # # end
+      # #
+      #
+      # private
+      #
+      # def init_tasks
+      #   @tasks = Dir.glob("**/*.task*")
+      # end
+      #
+      # def ask_for_relation
+      #   init_tasks if @tasks.blank?
+      #   to = nil
+      #   from = ask_for_task("Task To Start From: [#{@tasks.join(', ')}]")
+      #   to = ask_for_task("Task To Go To: [#{@tasks.join(', ')}]") if from.present?
+      #   [from, to]
+      # end
+      #
+      # def ask_for_task(text)
+      #   verified = false
+      #   task = nil
+      #   while !verified
+      #     task = ask(text)
+      #     if task.blank?
+      #       return nil
+      #     end
+      #     begin
+      #       Cnvrg::Task.new(@curr_dir, path: task).verify_task
+      #       verified = true
+      #     rescue => e
+      #     end
+      #   end
+      #   task
+      # end
+      #
+      # def get_all_tasks
+      #   @tasks = Dir.glob("*/**.task.yaml")
+      # end
   end
-end
+end
+end

data/lib/cnvrg/cli/library_cli.rb

@@ -0,0 +1,33 @@
+module Cnvrg
+  class LibraryCli < SubCommandBase
+
+    desc "library import", description: ""
+    def import
+      unless File.exists? "library.yml"
+        Cnvrg::CLI.log_message("Can't find library.yml", 'red')
+        exit(1)
+      end
+      library = YAML.safe_load(File.open("library.yml").read)
+      Cnvrg::CLI.log_message("Archiving library #{library["title"]}")
+      files = Dir["**/*"].select{|file| not File.directory?(file)}
+      File.open("archive.tar.gz", "wb") do |file|
+        Zlib::GzipWriter.wrap(file) do |gzip|
+          Gem::Package::TarWriter.new(gzip) do |tar|
+            files.each do |filename|
+              f = File.open(filename)
+              tar.add_file_simple(filename, 0644, f.size) do |io|
+                io.write(f.read)
+              end
+            end
+          end
+        end
+      end
+      response = Cnvrg::API.request(['users', Cnvrg::CLI.get_owner, 'libraries'].join("/"), "POST_FILE", {relative_path: "archive.tar.gz"})
+      if response["status"] != 200
+        Cnvrg::CLI.log_message("Can't create library: #{response["message"]}")
+        exit(1)
+      end
+      Cnvrg::CLI.log_message("Library Created successfuly", "green")
+    end
+  end
+end

data/lib/cnvrg/dataset.rb

@@ -518,6 +518,9 @@ module Cnvrg
         #check if prefix exists do prefix/path otherwise path
         label = file.gsub(self.local_path + "/", "")
         label = "#{prefix}/#{label}" if prefix.present?
+        if not Cnvrg::Files.valid_file_name?(label)
+          raise StandardError.new("#{label} is not a valid file name.")
+        end
         if File.directory? file
           tree[label + "/"] = nil
         else

data/lib/cnvrg/experiment.rb

@@ -16,6 +16,15 @@ module Cnvrg
       @output_dir = nil
     end
 
+    def as_env
+      return {
+          CNVRG_JOB_ID: @slug,
+          CNVRG_JOB_TYPE: "Experiment",
+          CNVRG_PROJECT: @project_slug,
+          CNVRG_OWNER: @owner,
+      }.as_json
+    end
+
     def start(input, platform, machine_name, start_commit, name, email_notification, machine_activity,script_path,
               sync_before_terminate, periodic_sync)
 
@@ -88,6 +97,14 @@ module Cnvrg
 
     end
 
+    def job_log(logs, level: 'info', step: nil, job_type: nil, job_id: nil)
+      logs = [logs].flatten
+      logs.each_slice(10).each do |temp_logs|
+        Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/jobs/experiment/#{@slug}/log", "POST", {job_type: "Experiment", job_id: @slug, logs: temp_logs, log_level: level, step: step, timestamp: Time.now})
+        sleep(1)
+      end
+    end
+
     def exec_remote(command, commit_to_run, instance_type, image_slug,schedule,local_timestamp, grid,path_to_cmd,data, data_commit,periodic_sync,
                     sync_before_terminate, max_time, ds_sync_options=0,output_dir=nil,data_query=nil,
                     git_commit=nil, git_branch=nil, restart_if_stuck=nil, local_folders=nil,title=nil, datasets=nil, prerun: true, requirements: true, recurring: nil)

data/lib/cnvrg/files.rb

@@ -6,7 +6,7 @@ require 'net/http'
 require 'cnvrg/result'
 module Cnvrg
   class Files
-
+    VALID_FILE_NAME = /[\x00\\:\*\?\"<>\|]/
     LARGE_FILE=1024*1024*5
     MULTIPART_SPLIT=10000000
 
@@ -29,6 +29,11 @@ module Cnvrg
     end
 
 
+    def self.valid_file_name?(fullpath)
+      VALID_FILE_NAME.match(fullpath).blank?
+    end
+
+
     def download_commit(sha1)
       response = @project.clone(false, sha1)
       log_error("Cant download commit #{sha1}") unless Cnvrg::CLI.is_response_success response, false

data/lib/cnvrg/flow.rb

@@ -1,19 +1,71 @@
 module Cnvrg
   class Flows
-    def initialize(flow_slug)
-      @project = Cnvrg::Project.new(Cnvrg::CLI.get_project_home)
-      @flow_slug = flow_slug
+    def initialize(flow_slug, project: nil)
+      @project = project || Cnvrg::Project.new(Cnvrg::CLI.get_project_home)
+      @flow_info= Flows.resolve_flow_title(flow_slug, project)
+      @slug = @flow_info["slug"]
       @tasks = {}
       @relations = {}
       @title = nil
-      @slug = nil
-      @base_resource = @project.base_resource + "flows"
-
+      @base_resource = @project.base_resource + "flows/#{@slug}"
+      @public_url = "#{@project.url}/flows/#{@slug}"
       # self.reload_flow
     end
 
-    def self.create_flow(path, flow)
-      File.open(path, "w"){|file| file.write flow.to_yaml}
+    def self.resolve_flow_title(title, project)
+      resp = Cnvrg::API.request("#{project.base_resource}/flows", 'GET')
+      if resp.blank?
+        raise StandardError.new("Can't resolve flow")
+      end
+      res = resp["result"].find{|flow| flow["slug"].downcase == title.downcase}
+      res ||= resp["result"].find{|flow| flow["title"].downcase == title.downcase}
+      if res.blank?
+        raise StandardError.new("Can't find flow with title #{title}")
+      end
+      res
+    end
+
+    def edit_href
+      "#{@public_url}/flow_versions/new"
+    end
+
+    def edit_version_href(version)
+      return "#{edit_href}?flow_version_slug=#{version}"
+    end
+
+    def version_href(version=nil)
+      "#{@base_resource}/flow_versions/#{version || 'latest'}"
+    end
+
+    def export(version, file: nil)
+      resp = Cnvrg::API.request(version_href(version), 'GET')
+      if resp["status"] != 200
+        raise StandardError.new("Cant find flow version: #{version} for flow: #{@slug}")
+      end
+      flow_version = resp["flow_version"]
+      api_recipe = flow_version["api_recipe"]
+      file = file.presence  || "flow-#{@slug.downcase.gsub("\s", "_")}.yml"
+      File.open(file, "w"){|f| f.write api_recipe.to_yaml}
+      file
+    end
+
+    def get_version(version)
+
+
+    end
+
+    def self.create_flow(project, recipe, run: false)
+      url = "#{project.base_resource}flows"
+      if run
+        url += "/run"
+      end
+      resp = Cnvrg::API.request(url, 'POST', {flow_version: recipe.to_json}) || {}
+      if resp["status"] == 200
+        return [Flows.new(resp["flow_version"]["flow_id"], project: project), resp["flow_version"]["id"]]
+      elsif resp["status"] == 400
+        raise StandardError.new(resp["message"])
+      end
+      raise StandardError.new("Can't create new flow")
     end
 
     def get_flow
@@ -52,11 +104,11 @@ module Cnvrg
 
 
     def run
-      resp = Cnvrg::API.request("#{@base_resource}/#{@flow_slug}/run", 'POST')
+      resp = Cnvrg::API.request("#{@base_resource}/#{@slug}/run", 'POST')
       if Cnvrg::CLI.is_response_success(resp)
         return resp
       end
-      Cnvrg::CLI.log_message("Cant run flow #{@flow_slug}")
+      Cnvrg::CLI.log_message("Cant run flow #{@slug}")
     end
 
     ### in use for yaml file

data/lib/cnvrg/project.rb

@@ -428,7 +428,9 @@ module Cnvrg
       # list.each do |e|
       Parallel.map(list, in_threads: IDXParallelThreads) do |e|
         label = e.gsub(self.local_path + "/", "")
-
+        if not Cnvrg::Files.valid_file_name?(label)
+          raise StandardError.new("#{label} is not a valid file name")
+        end
         if list_ignore_new.include? label
           next
         end

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '1.6.0.8'
+  VERSION = '1.6.0.9'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.6.0.8
+  version: 1.6.0.9
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-01 00:00:00.000000000 Z
+date: 2019-12-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -430,6 +430,7 @@ files:
 - lib/cnvrg/auth.rb
 - lib/cnvrg/cli.rb
 - lib/cnvrg/cli/flow.rb
+- lib/cnvrg/cli/library_cli.rb
 - lib/cnvrg/cli/subcommand.rb
 - lib/cnvrg/cli/task.rb
 - lib/cnvrg/colors.rb