cnvrg 1.9.9.9.7

data/lib/cnvrg/cli/flow.rb

@@ -0,0 +1,166 @@
+require 'cnvrg/cli'
+require 'cnvrg/data'
+require 'cnvrg/task'
+require 'cnvrg/flow'
+require 'cnvrg/hyper'
+require 'cnvrg/cli/subcommand'
+require 'cnvrg/cli/task'
+require 'thor'
+module Cnvrg
+  module Commands
+    class Flow < SubCommand
+
+      def initialize(*args)
+        super
+        unless @curr_dir.present?
+          @cli.log_message("Not On Project Dir, exiting", Thor::Color::RED)
+          exit(1)
+        end
+      end
+
+      desc "flow import --file='flow.yaml'", "Import flow to the web"
+      method_option :file, :type => :string, :aliases => ["-f", "--file"]
+      def import()
+        run_flow_internal(file: options[:file], run: false)
+      end
+
+      desc "flow run --file='flow.yaml'", "Import flow to the web"
+      method_option :file, :type => :string, :aliases => ["-f", "--file"]
+      def run()
+        run_flow_internal(file: options[:file], run: true)
+      end
+
+      no_commands {
+      def run_flow_internal(file: nil, run: false)
+        if options[:file].blank?
+          Cnvrg::CLI.log_message("Flow title is required for export. please use --title='Flow 1'")
+          return
+        end
+        if not File.exists? options[:file]
+          Cnvrg::CLI.log_message("Cant find file in path #{options[:file]}.")
+          return
+        end
+        payload = YAML.safe_load(File.open(file).read)
+        @project = Cnvrg::Project.new(Cnvrg::CLI.get_project_home)
+        begin
+          @flow, version_slug = Flows.create_flow(@project, payload, run: run)
+        rescue => e
+          Cnvrg::CLI.log_message("Error while validating flow: #{e.message}", 'error')
+          return
+        end
+        Cnvrg::CLI.log_message("New flow version was created successfuly!", 'green')
+        Cnvrg::CLI.log_message("you can find the flow in #{@flow.edit_version_href(version_slug)}", 'green')
+      rescue => e
+        Cnvrg::CLI.log_message("Failed while creating flow")
+        Cnvrg::Logger.log_error(e)
+      end
+      }
+
+      desc "flow export --file='flow.yaml' --title='Flow 1' --version='version 1'", "Export flow to the web"
+      method_option :file, :type => :string, :aliases => ["-f", "--file"]
+      method_option :title, :type => :string, :aliases => ["-t", "--title"]
+      method_option :version, :type => :string, :aliases => ["-v", "--version"], :default => "latest"
+      def export()
+        if options[:title].blank?
+          Cnvrg::CLI.log_message("Flow title is required for export. please use --title='Flow 1'", "red")
+          return
+        end
+
+        @project = Cnvrg::Project.new(Cnvrg::CLI.get_project_home)
+        @flow = Flows.new(options[:title], project: @project)
+        filename = @flow.export(options[:version], file: options[:file])
+        Cnvrg::CLI.log_message("Flow was saved successfuly to: #{filename}", 'green')
+      rescue => e
+        Cnvrg::CLI.log_message(e.message, 'red')
+        Cnvrg::Logger.log_error(e)
+      end
+
+      # desc "task", "Running Flow tasks", :hide => true
+      # subcommand 'task', Cnvrg::Commands::Task
+      #
+      # desc "flow verify", "verify that the flow is well formatted"
+      #
+      # def verify(path)
+      #   unless @curr_dir.present?
+      #     @cli.log_message("Cant run this command because you are not in project directory", Thor::Color::RED)
+      #     return false
+      #   end
+      #   @flow = Cnvrg::Flows.new(@curr_dir, path)
+      #   @cli.log_message("The Flow is Valid", Thor::Color::GREEN)
+      # end
+      #
+      #
+      # desc "flow run", "run a flow file"
+      # def run(flow_slug)
+      #   unless @curr_dir.present?
+      #     @cli.log_message("Cant run this command because you are not in project directory", Thor::Color::RED)
+      #     return false
+      #   end
+      #   @flow = Cnvrg::Flows.new(flow_slug)
+      #   resp = @flow.run
+      #   flow_version_href = resp["flow_version"]["href"]
+      #   Cnvrg::CLI.log_message("Flow Live results: #{flow_version_href}")
+      #   true
+      # end
+      #
+      # desc "flow create", "create a flow file"
+      #
+      # def create
+      #   title = ask("Flow Title: ")
+      #   title = title.presence || "cnvrg-flow"
+      #   relations = []
+      #   task = ask_for_relation
+      #   while task.compact.size == 2
+      #     from, to = task
+      #     relations << {from: from, to: to}
+      #     task = ask_for_relation
+      #   end
+      #   start_commit = ask("Flow Starting Commit (empty will set this value to 'latest')")
+      #   Cnvrg::Flows.create_flow("#{title}.flow.yaml", {title: title, relations: relations, start_commit: start_commit})
+      # end
+      #
+      #
+      # # desc "flow resolve", "Resolve flow parameters"
+      # # def resolve(path)
+      # #   @hyper = Cnvrg::Hyper.new(@curr_dir, path)
+      # #   @hyper.resolve_params
+      # # end
+      # #
+      #
+      # private
+      #
+      # def init_tasks
+      #   @tasks = Dir.glob("**/*.task*")
+      # end
+      #
+      # def ask_for_relation
+      #   init_tasks if @tasks.blank?
+      #   to = nil
+      #   from = ask_for_task("Task To Start From: [#{@tasks.join(', ')}]")
+      #   to = ask_for_task("Task To Go To: [#{@tasks.join(', ')}]") if from.present?
+      #   [from, to]
+      # end
+      #
+      # def ask_for_task(text)
+      #   verified = false
+      #   task = nil
+      #   while !verified
+      #     task = ask(text)
+      #     if task.blank?
+      #       return nil
+      #     end
+      #     begin
+      #       Cnvrg::Task.new(@curr_dir, path: task).verify_task
+      #       verified = true
+      #     rescue => e
+      #     end
+      #   end
+      #   task
+      # end
+      #
+      # def get_all_tasks
+      #   @tasks = Dir.glob("*/**.task.yaml")
+      # end
+  end
+end
+end

data/lib/cnvrg/cli/library_cli.rb

@@ -0,0 +1,33 @@
+module Cnvrg
+  class LibraryCli < SubCommandBase
+
+    desc "library import", ''
+    def import
+      unless File.exists? "library.yml"
+        Cnvrg::CLI.log_message("Can't find library.yml", 'red')
+        exit(1)
+      end
+      library = YAML.safe_load(File.open("library.yml").read)
+      Cnvrg::CLI.log_message("Archiving library #{library["title"]}")
+      files = Dir["**/*"].select{|file| not File.directory?(file)}
+      File.open("archive.tar.gz", "wb") do |file|
+        Zlib::GzipWriter.wrap(file) do |gzip|
+          Gem::Package::TarWriter.new(gzip) do |tar|
+            files.each do |filename|
+              f = File.open(filename)
+              tar.add_file_simple(filename, 0644, f.size) do |io|
+                io.write(f.read)
+              end
+            end
+          end
+        end
+      end
+      response = Cnvrg::API.request(['users', Cnvrg::CLI.get_owner, 'libraries'].join("/"), "POST_FILE", {relative_path: "archive.tar.gz"})
+      if response["status"] != 200
+        Cnvrg::CLI.log_message("Can't create library: #{response["message"]}")
+        exit(1)
+      end
+      Cnvrg::CLI.log_message("Library Created successfuly", "green")
+    end
+  end
+end

data/lib/cnvrg/cli/subcommand.rb

@@ -0,0 +1,28 @@
+class SubCommand < Thor
+  @cli = nil
+  @curr_dir = nil
+
+  def initialize(*args)
+    super
+    @cli = Cnvrg::CLI.new
+    @curr_dir = @cli.is_cnvrg_dir
+    @config = YAML.load_file(@curr_dir + "/.cnvrg/config.yml") if @curr_dir
+    @helpers = Cnvrg::OrgHelpers.new();
+  end
+
+  def self.banner(command, namespace = nil, subcommand = false)
+    "#{basename}  #{command.usage}"
+  end
+
+  def self.subcommand_prefix
+    self.name.gsub(%r{.*::}, '').gsub(%r{^[A-Z]}) {|match| match[0].downcase}.gsub(%r{[A-Z]}) {|match| "-#{match[0].downcase}"}
+  end
+
+  class << self
+    # Hackery.Take the run method away from Thor so that we can redefine it.
+    def is_thor_reserved_word?(word, type)
+      return false if word == "run"
+      super
+    end
+  end
+end

data/lib/cnvrg/cli/task.rb

@@ -0,0 +1,116 @@
+module Cnvrg
+  module Commands
+    class Task < SubCommand
+
+      def initialize(*args)
+        super
+        unless @curr_dir.present?
+          @cli.log_message("Not On Project Dir, exiting", Thor::Color::RED)
+          exit(1)
+        end
+      end
+
+      desc "flow task verify", "verify that the task is well formatted"
+      def verify(path)
+        unless @curr_dir.present?
+          @cli.log_message("Cant run this command because you are not in project directory", Thor::Color::RED)
+          return false
+        end
+        @task = Cnvrg::Task.new(@curr_dir, path: path)
+        begin
+          @task.verify_task
+        rescue StandardError => e
+          @cli.log_message("An error during parsing task: #{e.message}", Thor::Color::RED)
+          @cli.log_error(e)
+          return false
+        rescue => e
+          @cli.log_error(e)
+          return false
+        end
+        @cli.log_message("The task verified successfuly", Thor::Color::GREEN)
+        return @task
+      end
+
+
+      desc "flow task create", "Creates new task"
+      def create
+        @project = Cnvrg::Project.new(@curr_dir)
+
+        task = {}
+        task[:title] = ask("Task title", Thor::Color::GREEN)
+        task[:type] = ask("Task Type?", Thor::Color::GREEN, :limited_to => ["exec", "data", "library", "deploy"])
+        case task[:type]
+        when "exec"
+          task[:cmd] = ask("Command to run", Thor::Color::GREEN)
+          task[:params], task[:params_path] = ask_for_params
+          task[:machine] = ask_for_machine
+        when "data"
+          task[:dataset] = ask("Dataset slug", Thor::Color::GREEN)
+          task[:query] = ask("Dataset query", Thor::Color::GREEN)
+        when "library"
+          task[:library] = ask("Library Slug", Thor::Color::GREEN)
+          task[:params], task[:params_path] = ask_for_params
+          task[:machine] = ask_for_machine
+        when "deploy"
+          task[:cmd] = ask("Command to run", Thor::Color::GREEN)
+          task[:function] = ask("Function to run", Thor::Color::GREEN)
+        end
+        @task = Cnvrg::Task.new(@curr_dir, content: task)
+        @task.save
+        @cli.log_message("Task #{@task.title} Saved Successfuly", Thor::Color::GREEN)
+      end
+
+
+      desc "flow task run", "Running specific task"
+      def run(path)
+        begin
+          path = "#{path}.task.yaml" unless path.end_with? '.task.yaml'
+          @task = Cnvrg::Task.new(@curr_dir, path: path)
+          url = @task.run
+          @cli.log_message("Task: #{@task.title} is running, you can track its performance on #{url}", Thor::Color::GREEN)
+        rescue => e
+          @cli.log_message(e.message, Thor::Color::RED)
+          @cli.log_error(e)
+        end
+      end
+
+      private
+
+      def ask_for_params
+        params = []
+        param_key = ask("Param key: (enter for continue)", Thor::Color::GREEN)
+        while param_key.present?
+          param_value = ask("#{param_key} value: (seperated by commas)", Thor::Color::GREEN)
+          params << {key: param_key, value: param_value}
+          param_key = ask("Param key: (enter for continue)", Thor::Color::GREEN)
+        end
+        if params.blank?
+          params_path = ask("Params from file: ")
+          if params_path.present?
+            while params_path.present? and !File.exists? params_path
+              params_path = ask("Cant find file, please write it again.")
+            end
+          end
+        end
+        return params, params_path
+      end
+
+      def ask_for_machine
+        options = {}
+        all_machines = @project.get_machines
+        options[:limited_to] = all_machines if all_machines.present?
+        ask("Kind of machine to execute on?", Thor::Color::GREEN, options).presence || "medium"
+      end
+
+
+      def on_project
+        unless @curr_dir.present?
+          @cli.log_message("Cant run this command because you are not in project directory", Thor::Color::RED)
+          return false
+        end
+      end
+
+
+    end
+  end
+end

data/lib/cnvrg/colors.rb

@@ -0,0 +1,8 @@
+module Cnvrg
+  class Colors
+    RED = 0
+    BLUE = 1
+    YELLOW = 2
+    GREEN = 3
+  end
+end

data/lib/cnvrg/connect_job_ssh.rb

@@ -0,0 +1,31 @@
+module Cnvrg
+  class ConnectJobSsh
+    def initialize(job_id)
+      home_dir = File.expand_path('~')
+      config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
+      @owner = config.to_h[:owner]
+      @job_id = job_id
+    rescue => e
+      @owner = ""
+      Cnvrg::Logger.log_info("cnvrg is not configured")
+    end
+
+    def start(username, password)
+      Cnvrg::API_V2.request("#{@owner}/job_ssh/#{@job_id}/start" , 'POST', {username: username, password: password})
+    end
+
+    def status()
+      Cnvrg::API_V2.request("#{@owner}/job_ssh/#{@job_id}/status" , 'GET', nil)
+    end
+
+    def run_portforward_command(pod_name, port, kubeconfig, namespace)
+      command = "kubectl"
+      if kubeconfig.present?
+        command = "kubectl --kubeconfig=#{kubeconfig}"
+      end
+      bashCommand = "#{command} -n #{namespace} port-forward #{pod_name} #{port}:22"
+      puts("\nrunning command #{bashCommand}")
+      `#{bashCommand}`
+    end
+  end
+end

data/lib/cnvrg/data.rb

@@ -0,0 +1,335 @@
+require 'cnvrg/cli'
+require 'thor'
+class SubCommandBase < Thor
+ def self.banner(command, namespace = nil, subcommand = false)
+  "#{basename}  #{command.usage}"
+ end
+
+ def self.subcommand_prefix
+  self.name.gsub(%r{.*::}, '').gsub(%r{^[A-Z]}) { |match| match[0].downcase }.gsub(%r{[A-Z]}) { |match| "-#{match[0].downcase}" }
+ end
+end
+
+module Cnvrg
+  class Data < SubCommandBase
+    class_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
+
+    module ConfigValidation
+      SUCCESS = "success"
+      FAILED = "failed"
+    end
+
+    desc "data init", "Set current directory as dataset directory"
+    method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
+    method_option :bucket, :type => :string, :aliases => ["-b", "--bucket"], :default => ""
+    method_option :title, :type => :string, :aliases => ["-t", "--title"], :default => ""
+
+    def init
+      cli = Cnvrg::CLI.new()
+      public = options["public"]
+      bucket = options["bucket"]
+      title = options["title"]
+      cli.init_data(public, bucket: bucket, title: title)
+    end
+
+    desc "data link DATASET_SLUG", "Set current directory as dataset directory"
+    def link(dataset=nil)
+      begin
+        cli = Cnvrg::CLI.new()
+        cli.verify_logged_in(false)
+        cli.log_start(__method__, args, options)
+        
+        if dataset.include? "/"
+          ## this case is to support DATASET_URL expected example: domain.com/organization_name/datasets/dataset_name
+          url_parts = dataset.split("/")
+          dataset_index = Cnvrg::Helpers.look_for_in_path(dataset, "datasets")
+          dataset_slug = url_parts[dataset_index + 1]
+          owner = url_parts[dataset_index - 1]
+          raise Exception.new("Can't find all dataset information please check the URL") if dataset_slug.blank? or owner.blank?
+        else
+          ## this case is to support DATASET_SLUG expected example: dataset_name
+          # in this case it will take the organization from the config file
+          dataset_slug = dataset
+          raise Exception.new("Please enter dataset name or dataset full url") if dataset_slug.blank?
+          owner = CLI.get_owner
+        end
+
+        config_validation = Dataset.validate_config
+        if config_validation[:validation] == Data::ConfigValidation::SUCCESS
+          cli.log_message(config_validation[:message])
+          return
+        else
+          Cnvrg::Logger.log_error_message(config_validation)
+          FileUtils.rmtree(".cnvrg")
+        end
+
+        cli.log_message("Linking dataset: #{dataset_slug} to the current directory", Thor::Shell::Color::BLUE)
+        success = Dataset.link_dataset(owner: owner, slug: dataset_slug)
+        if success
+          cli.log_message("Dataset: #{dataset_slug} linked successfully to the current directory")
+        else
+          cli.log_message("Linking failed\nAborting", Thor::Shell::Color::RED)
+        end
+      rescue => e
+        Cnvrg::Logger.log_error(e)
+        cli.log_message("Aborting\n#{e.message}", Thor::Shell::Color::RED)
+        exit(1)
+      rescue Exception => e
+        Cnvrg::Logger.log_error(e)
+        cli.log_message("Aborting\n#{e.message}", Thor::Shell::Color::RED)
+        exit(1)
+      end
+    end
+
+
+    desc "data upload", "Upload files from local dataset directory to remote server"
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
+    method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s","--sync"], :default => false
+    method_option :tags, :type => :boolean, :aliases => ["--tags"], :desc => "upload file tags", :default => false
+    method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :desc => "upload file tags", :default => 1000
+    method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
+
+    def upload
+      cli = Cnvrg::CLI.new()
+      verbose = options["verbose"]
+      sync = options["sync"]
+      force = options["force"]
+      new_branch = options["new_branch"]
+      chunk_size = options["chunk_size"]
+      tags = options["tags"]
+      message = options["message"]
+      cli.upload_data_new(new_branch, verbose, sync, force, tags, chunk_size, message:message)
+    end
+    desc 'data sync', 'Synchronise local dataset directory with remote server'
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
+    method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :commit, :type => :string, :aliases => ["-c"], :desc => "download specified commit", :default => nil
+    method_option :all_files, :type => :boolean, :aliases => ["--all"], :desc => "download specified commit", :default => false
+    method_option :tags, :type => :boolean, :aliases => ["--tags"], :desc => "upload file tags", :default => false
+    method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
+    method_option :chunk_size, :type => :numeric, :aliases => ["--chunk_size"], :desc => "chunk size to communicate with the server", :default => 1000
+    method_option :init, :type => :boolean, :aliases => ["--initial"], :desc => "initial upload of dataset", :default => false
+    method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
+
+    def sync_data_new()
+      cli = Cnvrg::CLI.new()
+      force = options["force"]
+      new_branch = options["new_branch"]
+      verbose = options["verbose"]
+      commit = options["commit"]
+      all_files = options["all_files"]
+      tags = options["tags"]
+      parallel=options["parallel"]
+      chunk_size = options["chunk_size"]
+      init = options["init"]
+      message = options["message"]
+      cli.sync_data_new(new_branch, force, verbose, commit, all_files, tags, parallel, chunk_size, init, message)
+    end
+
+    desc 'data download', 'Download files from remote server'
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits", :default => false
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+    method_option :commit, :type => :string, :aliases => ["-c"], :desc => "download specified commit", :default => nil
+    method_option :all_files, :type => :boolean, :aliases => ["--all"], :desc => "download specified commit", :default => false
+
+    def download()
+      cli = Cnvrg::CLI.new()
+      verbose = options["verbose"]
+      sync = options["sync"]
+      new_branch = options["new_branch"]
+      commit = options["commit"]
+      all_files = options["all_files"]
+      cli.download_data_new(verbose,sync,new_branch, commit,all_files)
+    end
+    desc 'data clone DATASET_URL', 'Clone dataset'
+    method_option :only_tree, :type => :boolean, :aliases => ["-t", "--tree"], :default => false
+    method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => nil
+    method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => nil
+    method_option :read, :type => :boolean, :aliases => ["-r", "--read"], :default => false
+    method_option :remote, :type => :boolean, :aliases => ["-h", "--remote"], :default => false
+    method_option :relative, :type => :boolean, :aliases => ["-rel", "--relative"], :default => false
+    method_option :flatten, :type => :boolean, :aliases => ["-f", "--flatten"], :default => false
+    method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
+    def clone(dataset_url)
+      cli = Cnvrg::CLI.new()
+      only_tree =options[:only_tree]
+      commit =options[:commit]
+      query =options[:query]
+      read = options[:read]
+      remote = options[:remote]
+      soft = options[:soft]
+      flatten = options[:flatten]
+      cli.clone_data(
+          dataset_url,
+          only_tree=only_tree,
+          commit=commit,
+          query=query,
+          read=read,
+          remote=remote,
+          flatten: flatten,
+          relative: options[:relative],
+          soft: soft
+      )
+    end
+
+    desc 'data verify DATASETS_TITLES', 'verify datasets', :hide => true
+    method_option :timeout, :type => :numeric, :aliases => ["-t", "--timeout"], :desc => "Time to wait before returning final answer", :default => nil
+    def verify(*dataset_titles)
+      cli = Cnvrg::CLI.new()
+      timeout =options[:timeout]
+      cli.verify_datasets(dataset_titles, timeout)
+    end
+
+    desc 'data scan', 'lookup datasets', :hide => true
+    def scan()
+      cli = Cnvrg::CLI.new()
+      cli.scan_datasets()
+    end
+
+    desc "data block DATASET_TITLES", 'verifying that datasets exists', hide: true
+    def block(*dataset_slugs)
+      not_verified = true
+      while not_verified
+        not_verified = dataset_slugs.select{|slug| not Dataset.verify_dataset(slug)}.present?
+      end
+    end
+
+    desc 'data set --url=DATASET_URL', 'Set dataset url to other url'
+    method_option :url, :type => :string, :aliases => ["--url"], :default => ''
+    def set
+      cli = Cnvrg::CLI.new
+      cli.log_start(__method__)
+      cli.log_handler
+      if options['url'].present?
+        cli.set_data_url(options['url'])
+      end
+    end
+
+    desc 'data put DATASET_URL FILES_PREFIX', 'Upload selected files from local dataset directory to remote server'
+    method_option :dir, :type => :string, :aliases => ["-d", "--dir"], :default => ''
+    method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ''
+    method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
+    method_option :override, :type => :boolean, :aliases => ["--override"], :default => false
+    method_option :threads, :type => :numeric, :aliases => ["-t","--threads"], :default => 15
+    method_option :chunk_size, :type => :numeric, :aliases => ["-cs","--chunk"], :default => 1000
+    method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
+
+    def put(dataset_url, *files)
+      cli = Cnvrg::CLI.new()
+      dir = options[:dir]
+      force = options[:force]
+      override = options[:override]
+      commit = options[:commit]
+      message = options[:message]
+      threads = options[:threads]
+      chunk_size = options[:chunk_size]
+      cli.data_put(
+          dataset_url,
+          files: files,
+          dir: dir,
+          commit: commit,
+          force: force,
+          override: override,
+          threads: threads,
+          chunk_size: chunk_size,
+          message: message
+      )
+    end
+
+    desc 'data rm DATASET_URL FILES_PREFIX', 'Delete selected files from remote server'
+    method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
+    def rm(dataset_url, *regex_list)
+      cli = Cnvrg::CLI.new()
+      message = options[:message]
+      cli.data_rm(dataset_url, regex_list: regex_list, message: message)
+    end
+
+    desc 'data clone_query --query=QUERY_SLUG DATASET_URL', 'Clone dataset with specific query'
+    method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => nil
+    method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
+    method_option :flatten, :type => :boolean, :aliases => ["-f", "--flatten"], :default => false
+    def clone_query(dataset_url)
+      cli = Cnvrg::CLI.new()
+      query = options[:query]
+      flatten = options[:flatten]
+      soft =options[:soft]
+      cli.clone_data_query(dataset_url,query=query, flatten, soft: soft)
+    end
+
+    desc 'data delete DATASET_SLUG', 'Delete dataset'
+    def delete(dataset_slug)
+      cli = Cnvrg::CLI.new()
+      cli.delete_data(dataset_slug)
+
+    end
+    desc 'data list', 'Show list of all datasets'
+    def list()
+      cli = Cnvrg::CLI.new()
+
+      cli.list_dataset()
+
+    end
+
+    desc 'data commits URL/SLUG', 'List all commits for a given dataset'
+    method_option :commit_sha1, :type => :string, :aliases => ["-c", "--commit"], :default => nil
+    def commits(dataset_url)
+      cli = Cnvrg::CLI.new()
+      commit_sha1 = options[:commit_sha1]
+      cli.list_dataset_commits(dataset_url, commit_sha1:commit_sha1)
+    end
+
+    desc 'data files DATASET_URL', 'Show list of dataset files'
+    method_option :offset, :type => :numeric, :aliases => ["-o", "--offset"], :default => 0
+    method_option :limit, :type => :numeric, :aliases => ["-l", "--limit"], :default => 1000
+    method_option :expires, :type => :numeric, :aliases => ["-ex", "--expires"], :default => 3600
+    method_option :commit_sha1, :type => :string, :aliases => ["-c", "--commit"], :default => nil
+    def files(dataset_url)
+      cli = Cnvrg::CLI.new()
+      cli.verify_logged_in(false)
+      cli.log_start(__method__, args, options)
+      @dataset = Dataset.new(dataset_url: dataset_url)
+      files = @dataset.list_files(
+          commit_sha1: options[:commit_sha1],
+          limit: options[:limit],
+          expires: options[:expires],
+          offset: options[:offset])
+      cli.log_message(files)
+    end
+
+    desc 'data queries', 'List all dataset queries related to current dataset'
+    def queries()
+      cli = Cnvrg::CLI.new()
+      cli.queries()
+    end
+
+    desc 'data query_files QUERY_NAME', 'Show list of all files in specific query'
+    def query_files(query)
+      cli = Cnvrg::CLI.new()
+      cli.query_files(query)
+    end
+
+    desc 'data download_tags_yaml', 'Download dataset tags yml files in current dataset directory'
+    def download_tags_yaml
+      cli = Cnvrg::CLI.new()
+      cli.download_tags_yaml()
+    end
+
+
+    desc 'data test', 'test client'
+    def test(data_url)
+      cli = Cnvrg::CLI.new
+      cli.verify_logged_in(true)
+      cli.log_start(__method__, args, options)
+      @dataset = Dataset.new(dataset_url: data_url)
+      resp = @dataset.get_storage_client
+      @dataset.init_home(remote: false)
+      @dataset.download_softlink
+
+    end
+
+  end
+end