cnvrg 0.0.2

data/lib/cnvrg/experiment.rb

@@ -0,0 +1,26 @@
+module Cnvrg
+    class Experiment
+        attr_reader :slug
+        def initialize(owner, project_slug)
+            @project_slug = project_slug
+            @owner = owner
+            @base_resource = "users/#{owner}/projects/#{project_slug}/"
+            @slug = nil
+        end
+
+        def start(input, platform, machine_name, start_commit,name)
+            res = Cnvrg::API.request(@base_resource + "experiment/start", 'POST', { input: input, platform: platform, machine_name: machine_name, start_commit: start_commit , title:name})
+            Cnvrg::CLI.is_response_success(res)
+
+            @slug = res.to_h["result"].to_h["slug"]
+
+            return res
+            
+        end
+
+        def end(output, exit_status, end_commit,cpu_average, memory_average)
+        response = Cnvrg::API.request(@base_resource + "experiment/end", 'POST', { output: output, exp_slug: @slug, exit_status: exit_status, end_commit: end_commit, cpu_average:cpu_average,memory_average:memory_average })
+         Cnvrg::CLI.is_response_success(response)
+        end
+    end
+end

data/lib/cnvrg/files.rb

@@ -0,0 +1,115 @@
+require 'mimemagic'
+require 'pry'
+module Cnvrg
+    class Files
+        
+        attr_reader :base_resource
+
+        def initialize(owner, project_slug)
+            @project_slug = project_slug
+            @owner = owner
+            @base_resource = "users/#{owner}/projects/#{project_slug}/"
+        end
+
+        def upload_file(absolute_path, relative_path, commit_sha1)
+            file_name = File.basename relative_path
+            file_size = File.size(relative_path).to_f
+            upload_resp  = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', { absolute_path: absolute_path, relative_path: relative_path,
+                                                                                         commit_sha1: commit_sha1, file_name: file_name ,file_size:file_size})
+            if Cnvrg::CLI.is_response_success(upload_resp, false)
+                path = upload_resp["result"]["path"]
+                s3_res = upload_s3(path,relative_path)
+                    if s3_res
+                        Cnvrg::API.request(@base_resource + "update_s3", 'POST', { path:path,commit_id:upload_resp["result"]["commit_id"],
+                                                                                    blob_id:upload_resp["result"]["blob_id"]})
+                        return true
+                    end
+            end
+            return false
+        end
+        def upload_s3(url,file)
+            url = URI.parse(url)
+            mime_type = MimeMagic.by_path(file)
+            content_type =  !mime_type.nil? ?  mime_type.type : ""
+            file = File.open(file, "rb")
+            body = file.read
+            begin
+                Net::HTTP.start(url.host) do |http|
+                    http.send_request("PUT", url.request_uri, body, {
+                        "content-type" => content_type,
+                    })
+               end
+                return true
+            rescue Interrupt
+                return false
+            rescue => e
+                return false
+            end
+        end
+        def upload_url(file_path)
+            response  = Cnvrg::API.request(@base_resource + "upload_url", 'POST', { file_s3_path: file_path})
+            if Cnvrg::CLI.is_response_success(response, false)
+                return response
+            else
+              return nil
+            end
+
+        end
+        def delete_file(absolute_path, relative_path, commit_sha1)
+            response = Cnvrg::API.request(@base_resource + "delete_file", 'DELETE',{ absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1 })
+            return Cnvrg::CLI.is_response_success(response, false)
+        end
+
+        def delete_dir(absolute_path, relative_path, commit_sha1)
+            response = Cnvrg::API.request(@base_resource + "delete_dir", 'DELETE', { absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1 })
+            return Cnvrg::CLI.is_response_success(response, false)
+        end
+
+        def create_dir(absolute_path, relative_path, commit_sha1)
+            response = Cnvrg::API.request(@base_resource + "create_dir", 'POST',{ absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1 })
+            return Cnvrg::CLI.is_response_success(response, false)
+        end
+
+        def download_file(absolute_path, relative_path, project_home, conflict=false)
+            res = Cnvrg::API.request(@base_resource + "download_file", 'POST', { absolute_path: absolute_path, relative_path: relative_path })
+            Cnvrg::CLI.is_response_success(res, false)
+            if res["result"]
+                res = res["result"]
+                return false if res["link"].empty? or res["filename"].empty?
+                filename = res["filename"]
+                file_location = absolute_path.gsub(/#{filename}\/?$/, "")
+
+                FileUtils.mkdir_p project_home + "/" + file_location
+                filename += ".conflict" if conflict
+
+                File.open("#{project_home}/#{file_location}/#{filename}", "wb") do |file|
+                  file.write open(res["link"]).read
+                end
+            else
+                return false
+            end
+            return true
+        end
+
+        def download_dir(absolute_path, relative_path, project_home)
+            FileUtils.mkdir_p("#{project_home}/#{absolute_path}")
+        end
+
+        def start_commit
+
+            response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', { project_slug: @project_slug,
+                                                              username: @owner} )
+            Cnvrg::CLI.is_response_success(response)
+          return response
+        end
+
+        def end_commit(commit_sha1)
+            response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', { commit_sha1: commit_sha1 } )
+            return response
+        end
+        def rollback_commit(commit_sha1)
+            response = Cnvrg::API.request("#{base_resource}/commit/rollback", 'POST', { commit_sha1: commit_sha1 } )
+            Cnvrg::CLI.is_response_success(response, false)
+        end
+    end
+end

data/lib/cnvrg/helpers.rb

@@ -0,0 +1,90 @@
+module Cnvrg
+  module Helpers
+
+    extend self
+    def checkmark
+      checkmark = "\u2713"
+      return checkmark.encode('utf-8')
+      end
+
+    def remote_url
+        "https://cnvrg.io"
+    end
+
+    def windows?
+      !!(RUBY_PLATFORM =~ /mswin32|mingw32/)
+    end
+
+    def mac?
+      !!(RUBY_PLATFORM =~ /-darwin\d/)
+    end
+
+    def linux?
+      not mac? and not windows?
+    end
+
+    def cnvrgignore_content
+      %{
+               # cnvrg ignore: Ignore the following directories and files
+# for example:
+# some_dir/
+# some_file.txt
+            }.strip
+    end
+
+    def readme_content
+      %{
+            # README
+
+            This README would normally contain some context and description about the project. 
+
+            Things you may want to cover:
+
+            * Data description
+
+            * Benchmark and measurement guidelines
+
+            * Used algorithms
+
+            * Scores
+
+            * Configurations
+
+            * Requirements
+
+            * How to run the experiments
+
+            * ...}.strip
+    end
+
+    def netrc_domain
+      "cnvrg.io"
+    end
+
+    def look_for_in_path(path, name)
+      url_split = path.split("/")
+      url_split.each_with_index do |u, i|
+        if u == name
+          return i
+        end
+      end
+      return -1
+    end
+
+    # cpu
+
+    def cpu_time
+      Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :microsecond)
+    end
+
+    def wall_time
+      Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond)
+    end
+
+    # memory
+    #
+    def get_mem(pid)
+    end
+
+  end
+end

data/lib/cnvrg/old_cli.rb

@@ -0,0 +1,332 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'commander/import'
+require "open4"
+require 'netrc'
+require 'net/http'
+require 'uri'
+require 'json'
+require 'yaml'
+require 'digest' # sha1
+
+# DEV VERSION
+#require 'pry'
+#
+
+
+program :name, 'cnvrg'
+program :version, '0.0.1'
+program :description, 'cnvrg\'s cli for running scripts'
+
+command :idx do |c|
+    c.action do |args,options|
+        generate_idx
+    end
+end
+command :status do |c|
+    c.action do |args,options|
+        status
+    end
+end
+command :run do |c|
+    c.syntax = 'cnvrg run script [options]'
+    c.summary = 'monitor run of script'
+    c.description = 'Runs scripts and sending notification when finished'
+    c.option '--script_args script arguments', String, 'script arguments if exist'
+    c.option '--sms', String, 'set sms if you want to be notify by sms when the run is over'
+    c.option '--email', String, 'set email if you want to be notify by email when the run is over'
+    c.option '--silent', String, 'set silent if you want to run script in non-blocking'
+    c.action do |args, options|
+        is_exist = verify_auth_exist()
+        unless is_exist
+            puts("You have to be logged in to continue. Please login.\n")
+            loged_in = login_process()
+            if !loged_in
+                puts("Error: Couldn't logged you in, please run cnvr login separately")
+            end
+        end
+
+        silent = ''
+        if options.silent
+            silent = '&'
+        end
+        script = args[0]
+        if args.length != 1
+            puts "Error: only one script should be listed to run"
+            found = false
+            args.each do |arg|
+                ans = agree("should I run #{arg}?")
+                if ans
+                    found = true
+                    script = arg
+                    break
+                end
+            end
+            if not found
+                exit(0)
+            end
+        end
+        #check if file exist
+        exist = File.file?(script)
+        if not exist
+            puts "Error: Can't find #{script}, try using the full path"
+            exit(0)
+        end
+        puts "Running #{script} with args: #{options.script_args}"
+        body = {try: 'yay'}.to_json
+        puts run(true,body)
+        pid, stdin, stdout, stderr = Open4::popen4 "sh"
+        stdin.puts "./#{script} #{options.script_args}"
+        stdin.close
+
+        ignored, status = Process::waitpid2 pid
+
+        #try to run
+        if stderr.read.strip.include? "Permission denied"
+            ans = agree("Your script #{script} isn't executable, should I make it executable?")
+            if ans
+                puts "Making #{script} executable"
+                run_result = %x(chmod +x ./#{script} 2>&1)
+                if run_result.include? "Operation not permitted"
+                    puts "Can't make script executable,you should first run sudo chmod +x #{script}"
+                    exit(0)
+                end
+                puts "Running again #{script} with args: #{options.script_args}"
+
+                pid, stdin, stdout, stderr = Open4::popen4 "sh"
+                stdin.puts "./#{script} #{options.script_args}"
+                stdin.close
+                ignored, status = Process::waitpid2 pid
+
+                if !stderr.read.strip.empty?
+                    puts "Error: #{stderr.read.strip}"
+                end
+            end
+        end
+        puts stdout.read.strip
+        # send results to the server
+        puts "Finished running with status #{status.exitstatus}"
+        puts run(false, body)
+
+    end
+end
+
+command :new do |c|
+    c.syntax = "cnvrg new project-name"
+    c.summary = "Create a new data science project"
+
+    c.action do |args, options|
+        is_exist = verify_auth_exist()
+        unless is_exist
+            "You have to login"
+        else
+            if args[0].to_s.size == 0 
+                exit(0)
+            else
+                new(args[0])
+                link_dir(args[0], "")
+            end
+        end
+    end
+end
+
+command :login do |c|
+    c.syntax = 'cnvrg login'
+    c.summary = 'login to cnvrg app'
+    c.description = 'Login to the CNVRG system'
+    c.option '--reset', String, 'set reset if you want to change current login credentials'
+    c.action do |args, options|
+        n = Netrc.read
+        user, pass = n["cnvrg.io"]
+        if user and pass
+            relogin = agree("You are already authenticated with email: #{user}, do you want to login again? (This will delete you're current login credentials")
+            if !relogin
+                exit(0)
+            end
+        end
+        if options.reset
+            n["cnvrg.io"] = "", ""
+
+        end
+        login_process()
+
+
+    end
+end
+
+def login_process()
+    n = Netrc.read
+
+    email = ask("Please enter your email")
+    email_regex = /\A[\w+\-.]+@[a-z\d\-]+(\.[a-z\d\-]+)*\.[a-z]+\z/i
+    is_valid = email_regex.match(email)
+    while !is_valid do
+        email = ask("Please enter a valid email address:")
+        is_valid = email_regex.match(email)
+    end
+
+    password = ask("Please enter your password:  ") { |q| q.echo = false }
+    #send API to login
+    token = sign_in(email, password)
+    count = 0
+    while token.nil? and count<3 do
+        do_retry = agree("Wrong email or password, try again?")
+        if do_retry
+            count +=1
+            email = ask("Please enter your email")
+            password = ask("Please enter your password:  ") { |q| q.echo = false }
+            token = sign_in(email, password)
+        end
+    end
+    if token.nil?
+        puts "Error: Too many sign in retries"
+        exit(0)
+    else
+
+
+        n.new_item_prefix = "# This entry was added automatically by CNVRG\n"
+        n["cnvrg.io"] = email, token
+        n.save
+        puts "Great, you're in."
+        return true
+    end
+end
+
+def sign_in(email, password)
+    url = URI.parse("http://localhost:3000/api/v1/users/sign_in")
+
+    req = Net::HTTP::Post.new(url.path)
+    req.add_field("EMAIL", email)
+    req.add_field("PASSWORD", password)
+
+    response = Net::HTTP.new(url.host, url.port).start do |http|
+        http.request(req)
+    end
+
+    result = JSON.parse(response.body)
+    if result["status"] == 200
+        return result["token"]
+    else
+        return nil
+    end
+end
+
+def verify_auth_exist
+    n = Netrc.read
+    user, pass = n["cnvrg.io"]
+    if user.nil? or pass.nil?
+        puts "Please login first, using \"cnvrg login\" command"
+        return false
+    end
+    return true
+end
+
+def get_logged_token
+    n = Netrc.read
+    user, token = n["cnvrg.io"]
+    return token
+
+end
+def run(is_start, body)
+    token = get_logged_token()
+    if is_start
+        url = URI.parse("http://localhost:3000/api/v1/cli/start_run")
+    else
+        url = URI.parse("http://localhost:3000/api/v1/cli/end_run")
+    end
+
+
+    req = Net::HTTP::Post.new(url.path)
+    req.add_field("AUTH_TOKEN", token)
+    req.body = body
+    response = Net::HTTP.new(url.host, url.port).start do |http|
+        http.request(req)
+    end
+
+    result = JSON.parse(response.body)
+    puts result
+end
+
+def new(project_name)
+    begin
+        `mkdir -p #{project_name} #{project_name}/data #{project_name}/models #{project_name}/notebooks #{project_name}/src #{project_name}/src/data #{project_name}/src/features  #{project_name}/src/models #{project_name}/src/visualizations #{project_name}/.cnvrg`
+        `touch #{project_name}/README.md`
+        `touch #{project_name}/.cnvrgignore`
+    rescue
+        "Error"
+    end
+end
+
+def link_dir(project_name, user, host="cnvrg.io")
+    `touch #{project_name}/.cnvrg/project`
+    `echo "owner: #{user}\nproject_slug: #{project_name}\nhost: #{host}" >> #{project_name}/.cnvrg/project`
+end
+
+def download
+
+    status_output = status
+    downloadables = status_output["updated_on_server"].merge(status_output["deleted"])
+
+
+    downloadables.each do |d|
+        if d[-1] == "/"
+            # make dir
+        else
+            files << #download(d)
+        end
+end
+
+def upload
+
+end
+
+def generate_idx
+
+    idx = Hash.new(0)
+    list = Dir['**/**']
+
+    # remove files that are in CNVRGIGNORE
+    # list.select { |x| ! cnvrgignore.include? x }
+
+    list.each do |e|
+        if File.directory? e
+            idx[e+"/"] = nil
+        else
+            idx[e] = { sha1: Digest::SHA1.hexdigest(File.read(e)), 
+                       commit_time: nil }
+        end
+    end
+
+    File.open("hello/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
+
+
+end
+
+def status
+    url = URI.parse("http://localhost:3000/api/v1/users/yochze/projects/porject/status")
+    token = get_logged_token()
+    load_idx = YAML.load(File.read("hello/.cnvrg/idx.yml"))
+    req = Net::HTTP::Get.new(url, 'Content-Type' => 'application/json')
+    req.add_field("AUTH_TOKEN", token)
+    req.body = { snapshot: load_idx }.to_json
+    response = Net::HTTP.new(url.host, url.port).start do |http|
+        http.request(req)
+    end
+
+    result = JSON.parse(response.body)
+
+    result["status"]["added"].each do |a|
+        puts "A:\t#{a}"
+    end
+
+    result["status"]["updated_on_server"].each do |a|
+        puts "M:\t#{a}"
+    end
+    
+    result["status"]["deleted"].each do |a|
+        puts "D:\t#{a}"
+    end
+
+    return result["status"]
+end