cnvrg 2.0.1 → 2.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: ab82882b2bb6c9093751cd560eaa4ccb7540fad9b9a34a81245721538dd37a5b
4
- data.tar.gz: e35299be744d985a37794288a269ba2bb55cf64d3fcd702c6c6147bd4f5d740d
3
+ metadata.gz: 4092be1be35506977a7f1b4778b5bb1dfa234a02fc572812e773069f38e3d518
4
+ data.tar.gz: f8e954f4bbcbb4b735966c1391c3570c6ab7bf32036c825b7fd6a25460a7d5ef
5
5
  SHA512:
6
- metadata.gz: 52b51bb4942583e9ac3eceab24891d252ef7e31d7f1a2a87d5c8f4586f914f33f2d9cc7addc09ab5aa3652a9ba939ef5a37bdb9fc82d18e4e0e0a61126265d17
7
- data.tar.gz: df735e631778b5d36d296c33903719a2ba7832ce2a9e218eea033ad246f8c4ce044a7b7d5562f68d901a1ee32d62a88559f886da2a0341d8bf7c51b50c25660c
6
+ metadata.gz: 8e80a6d6d66eb5101eaa82c339c9c3862d4e39fbaaff7ba6e6d2278d0be5ef1cfc1b7b029e50dd567bf79af389911e2d27ad0d056e33110d1c56b4d613008b33
7
+ data.tar.gz: ab28db04b2d89b4170c1df63c7fec38c0027538f0e5620672a8ca358fd3234e4d177cdadaf46f5ebb8fe0c5c90f20cfd97179d1456201432b14b094b1555b666
data/Readme.md CHANGED
@@ -23,4 +23,46 @@
23
23
  2021-05-05
24
24
  * DEV-8868 - Bug: SDK - e.sync() in git project only creates empty "output" folder in commit
25
25
  ## Version v2.0.1
26
- 2021-06-13
26
+ 2021-06-13
27
+ ## Version v2.0.2
28
+ 2021-06-16
29
+ * DEV-9694 - Bug: Download artifacts fails on authorization error
30
+ ## Version v2.0.3
31
+ 2021-06-29
32
+ * DEV-9919 - Bug: clone artifacts fails on "Not Authorize, Are you logged in?"
33
+ ## Version v2.0.4
34
+ 2021-07-08
35
+ * DEV-9935 - Bug: CLI - cnvrg sync creates new commit but no blob versions
36
+ ## Version v2.0.5
37
+ 2021-07-11
38
+ * DEV-10171 - Bug: experiment randomly fails with error- "Couldn't clone artifacts"
39
+ * DEV-10189 - Bug: CLI Sync -file/folder with broken symlink will cause sync to fail
40
+ ## Version v2.0.6
41
+ 2021-07-18
42
+ * DEV-10209 - Bug: some experiments in grid failed on cnvrg-cli commands (docker container id was missing)
43
+ ## Version v2.0.7
44
+ 2021-07-27
45
+ * DEV-10186 - Bug: CLI/run an experiment with --local tag giver server error
46
+ ## Version v2.0.8
47
+ 2021-09-06
48
+ * DEV-10697 - Bug: Tensorboard not starting in workspace and experiment.
49
+ ## Version v2.0.9
50
+ 2021-09-12
51
+ * DEV-10502 - Bug: Periodic sync stuck
52
+ ## Version v2.0.10
53
+ 2021-09-12
54
+ * DEV-10502 - Bug: Periodic sync stuck
55
+ ## Version v2.0.11
56
+ 2021-10-21
57
+ ## Version v2.0.12
58
+ 2021-10-25
59
+ * DEV-11544 - Sub-bug: local experiment is failing to run
60
+ ## Version v2.0.13
61
+ 2021-10-27
62
+ * DEV-11054 - Task: Create organization and user by default
63
+ ## Version v2.0.14
64
+ 2021-11-11
65
+ * DEV-11834 - Sub-task: add device name to hpu metrics
66
+ ## Version v2.0.15
67
+ 2021-12-12
68
+ * DEV-12316 - Improvement: cli login should identify saas users automatically
data/lib/cnvrg/api.rb CHANGED
@@ -57,6 +57,7 @@ module Cnvrg
57
57
  conn = Faraday.new "#{endpoint_uri}"
58
58
  end
59
59
  conn.headers['Auth-Token'] = @pass
60
+ conn.headers['Authorization'] = "CAPI #{@pass}"
60
61
  conn.headers['User-Agent'] = "#{Cnvrg::API::USER_AGENT}"
61
62
  conn.options.timeout = 420
62
63
  conn.options.open_timeout=180
@@ -169,6 +170,7 @@ module Cnvrg
169
170
  when 'POST_FILE'
170
171
  conn = Faraday.new do |fr|
171
172
  fr.headers['Auth-Token'] = @pass
173
+ fr.headers['Authorization'] = "CAPI #{@pass}"
172
174
  fr.headers['User-Agent'] = "#{Cnvrg::API::USER_AGENT}"
173
175
  fr.headers["Content-Type"] = "multipart/form-data"
174
176
  if !Helpers.is_verify_ssl
data/lib/cnvrg/api_v2.rb CHANGED
@@ -22,6 +22,7 @@ module Cnvrg
22
22
 
23
23
  conn = Faraday.new endpoint_uri, :ssl => {:verify => !!Helpers.is_verify_ssl}
24
24
  conn.headers['Auth-Token'] = pass
25
+ conn.headers['Authorization'] = "CAPI #{pass}"
25
26
  conn.headers['User-Agent'] = Cnvrg::API::USER_AGENT
26
27
  conn.headers['Content-Type'] = "application/json"
27
28
  conn.options.timeout = 420
data/lib/cnvrg/auth.rb CHANGED
@@ -44,7 +44,7 @@ module Cnvrg
44
44
  end
45
45
  end
46
46
 
47
- def sign_in(email, password)
47
+ def sign_in(email, password, token: nil)
48
48
  url = Cnvrg::API.endpoint_uri()
49
49
  url = URI.parse(url+ "/users/sign_in")
50
50
  http = Net::HTTP.new(url.host, url.port)
@@ -61,6 +61,9 @@ module Cnvrg
61
61
 
62
62
  req.add_field("EMAIL", email)
63
63
  req.add_field("PASSWORD", password)
64
+ if token.present?
65
+ req.add_field("Authorization", "CAPI #{token}")
66
+ end
64
67
 
65
68
  response = http.request(req)
66
69
 
@@ -1,7 +1,7 @@
1
1
  module Cnvrg
2
2
  class LibraryCli < SubCommandBase
3
-
4
- desc "library import", ''
3
+ map push: :import
4
+ desc "library push", 'Push a new library to AI Library'
5
5
  def import
6
6
  unless File.exists? "library.yml"
7
7
  Cnvrg::CLI.log_message("Can't find library.yml", 'red')
data/lib/cnvrg/cli.rb CHANGED
@@ -496,8 +496,10 @@ module Cnvrg
496
496
 
497
497
 
498
498
  desc 'login', 'Authenticate with cnvrg.io platform'
499
+ method_option :sso, :type => :boolean, :aliases => ["-s", "--sso"], :default => false
499
500
 
500
501
  def login
502
+ use_token = options["sso"]
501
503
  begin
502
504
  log_handler()
503
505
  log_start(__method__, args, options)
@@ -515,12 +517,22 @@ module Cnvrg
515
517
  exit(0)
516
518
  end
517
519
  @email = ask("Enter your email:")
518
- password = cmd.ask("Enter your password (hidden):") {|q| q.echo = "*"}
519
- result = @auth.sign_in(@email, password)
520
+ url = Cnvrg::API.endpoint_uri()
521
+ if use_token or url.include?("cloud.cnvrg.io")
522
+ @token = cmd.ask("Enter your token (hidden):") {|q| q.echo = "*"}
523
+ netrc[Cnvrg::Helpers.netrc_domain] = @email, @token
524
+ netrc.save
525
+ password = ""
526
+ else
527
+ password = cmd.ask("Enter your password (hidden):") {|q| q.echo = "*"}
528
+ end
529
+ result = @auth.sign_in(@email, password, token: @token)
520
530
 
521
531
  if !result["token"].nil?
522
- netrc[Cnvrg::Helpers.netrc_domain] = @email, result["token"]
523
- netrc.save
532
+ unless use_token
533
+ netrc[Cnvrg::Helpers.netrc_domain] = @email, result["token"]
534
+ netrc.save
535
+ end
524
536
 
525
537
  log_message("Authenticated successfully as #{@email}", Thor::Shell::Color::GREEN)
526
538
 
@@ -3006,12 +3018,12 @@ module Cnvrg
3006
3018
  method_option :job_slug, :type => :string, :aliases => ["-j", "--job"], :default => nil
3007
3019
  method_option :job_type, :type => :string, :aliases => ["-jt", "--job_type"], :default => nil
3008
3020
  method_option :files, :type => :string, :aliases => ["--files"], :default => nil
3009
- method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
3021
+ method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => 'output'
3010
3022
  method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
3011
3023
  method_option :suppress_exceptions, :type => :boolean, :aliases => ["--suppress-exceptions"], :default => true
3012
3024
  method_option :debug_mode, :type => :boolean, :aliases => ["--debug-mode"], :default => false
3013
3025
  method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :default => 1000
3014
- method_option :local, :type => :boolean, :aliases => ["--local"], :default => false
3026
+ method_option :local, :type => :boolean, :aliases => ["--local"], :default => true
3015
3027
 
3016
3028
  def sync(direct = true)
3017
3029
  verify_logged_in(true) if direct
@@ -3147,7 +3159,7 @@ module Cnvrg
3147
3159
  invoke :exec, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title,
3148
3160
  :log => log, :email_notification => email_notification, :upload_output => upload_output,
3149
3161
  :commit => commit, :image => image, :data => data, :data_commit => data_commit,
3150
- :ignore => ignore, :force => force, :output_dir=>output_dir, :data_query=>data_query
3162
+ :ignore => ignore, :force => force, :output_dir=>output_dir, :data_query=>data_query, :local => local
3151
3163
  return
3152
3164
  end
3153
3165
  else
@@ -3204,6 +3216,7 @@ module Cnvrg
3204
3216
  method_option :data_query, :type => :string, :aliases => ["-q", "--query"], :default => nil
3205
3217
  method_option :use_bash, :type => :boolean, :aliases => ["-b", "--use_bash"], :default => false
3206
3218
  method_option :docker_stats, :type => :boolean, :aliases => ["--docker_stats"], :default => true
3219
+ method_option :local, :type => :boolean, :aliases => ["-l", "--local"], :default => false
3207
3220
 
3208
3221
  def exec(*cmd)
3209
3222
  log = []
@@ -3228,6 +3241,7 @@ module Cnvrg
3228
3241
  project_home = get_project_home
3229
3242
  data_query = options["data_query"]
3230
3243
  docker_stats = options["docker_stats"]
3244
+ local = options[:local] || false
3231
3245
  @project = Project.new(project_home)
3232
3246
  if @project.is_git
3233
3247
  sync_before = false
@@ -3323,43 +3337,49 @@ module Cnvrg
3323
3337
  if @exp.get_cmd.present?
3324
3338
  cmd = @exp.get_cmd
3325
3339
  end
3326
- command_slug = (0...18).map { (65 + rand(26)).chr }.join
3327
- result_file = "/conf/result-#{command_slug}"
3328
- data = {cmd: cmd, async: true, format: true, file_name: result_file, use_script: true, use_bash: options["use_bash"]}
3329
-
3330
- conn = Cnvrg::Helpers::Executer.get_main_conn
3331
- response = conn.post('command', data.to_json)
3332
- if response.to_hash[:status].to_i != 200
3333
- exit_status = 129
3334
- raise StandardError.new("Cant send command to slave")
3335
- end
3336
- t = FileWatch::Tail.new
3337
- filename = result_file
3338
- lines = []
3339
- t.tail(filename)
3340
- t.subscribe do |path, line|
3341
- begin
3342
- cur_log = JSON.parse(line)
3343
- if cur_log["type"] == "endMessage"
3344
- exit_status = cur_log["real"].to_i
3345
- break
3346
- else
3347
- puts(cur_log.to_json)
3348
- STDOUT.flush
3349
- cur_log["time"] = Time.parse(cur_log["timestamp"])
3350
- cur_log["message"] = cur_log["message"].to_s + "\r\n"
3351
- log << cur_log
3352
- end
3353
- if log.size >= 10
3354
- @exp.upload_temp_log(log)
3355
- log = []
3356
- elsif (start_time + 15.seconds) <= Time.now
3357
- @exp.upload_temp_log(log) unless log.empty?
3358
- log = []
3359
- start_time = Time.now
3340
+
3341
+ if local
3342
+ exec_local(cmd, print_log, start_commit, real, start_time)
3343
+ exit_status = $?.exitstatus
3344
+
3345
+ else
3346
+ command_slug = (0...18).map { (65 + rand(26)).chr }.join
3347
+ result_file = "/conf/result-#{command_slug}"
3348
+ data = {cmd: cmd, async: true, format: true, file_name: result_file, use_script: true, use_bash: options["use_bash"]}
3349
+ conn = Cnvrg::Helpers::Executer.get_main_conn
3350
+ response = conn.post('command', data.to_json)
3351
+ if response.to_hash[:status].to_i != 200
3352
+ exit_status = 129
3353
+ raise StandardError.new("Cant send command to slave")
3354
+ end
3355
+ t = FileWatch::Tail.new
3356
+ filename = result_file
3357
+ lines = []
3358
+ t.tail(filename)
3359
+ t.subscribe do |path, line|
3360
+ begin
3361
+ cur_log = JSON.parse(line)
3362
+ if cur_log["type"] == "endMessage"
3363
+ exit_status = cur_log["real"].to_i
3364
+ break
3365
+ else
3366
+ puts(cur_log.to_json)
3367
+ STDOUT.flush
3368
+ cur_log["time"] = Time.parse(cur_log["timestamp"])
3369
+ cur_log["message"] = cur_log["message"].to_s + "\r\n"
3370
+ log << cur_log
3371
+ end
3372
+ if log.size >= 10
3373
+ @exp.upload_temp_log(log)
3374
+ log = []
3375
+ elsif (start_time + 15.seconds) <= Time.now
3376
+ @exp.upload_temp_log(log) unless log.empty?
3377
+ log = []
3378
+ start_time = Time.now
3379
+ end
3380
+ rescue => e
3381
+ log_error(e)
3360
3382
  end
3361
- rescue => e
3362
- log_error(e)
3363
3383
  end
3364
3384
  end
3365
3385
  end_time = Time.now
@@ -4692,6 +4712,7 @@ module Cnvrg
4692
4712
  data_result.each_with_index do |res, i|
4693
4713
  timestamp, value = res["value"]
4694
4714
  uuid = res["metric"]["UUID"].presence || i
4715
+ uuid = res["metric"]["device"] if query_name == "gaudi"
4695
4716
  stat_value = value.present? ? ("%.2f" % value) : 0 # converting 34.685929244444445 to 34.69
4696
4717
  stat_value = stat_value.to_i == stat_value.to_f ? stat_value.to_i : stat_value.to_f # converting 34.00 to 34
4697
4718
  if query_name.include? 'block'
@@ -5876,6 +5897,57 @@ module Cnvrg
5876
5897
  end
5877
5898
  end
5878
5899
 
5900
+ def exec_local(cmd , print_log, start_commit, real, start_time)
5901
+ log = []
5902
+ PTY.spawn(@exp.as_env, cmd) do |stdout, stdin, pid, stderr|
5903
+ begin
5904
+ stdout.each do |line|
5905
+ cur_time = Time.now
5906
+ real_time = Time.now - real
5907
+ cur_log = {time: cur_time,
5908
+ message: line,
5909
+ type: "stdout",
5910
+ real: real_time
5911
+ }
5912
+ if print_log
5913
+ puts({log: line, timestamp: Time.now, exp_logs: true}.to_json)
5914
+ end
5915
+ log << cur_log
5916
+ if log.size >= 10
5917
+ @exp.upload_temp_log(log) unless log.empty?
5918
+ log = []
5919
+ elsif (start_time + 15.seconds) <= Time.now
5920
+ @exp.upload_temp_log(log) unless log.empty?
5921
+ log = []
5922
+ start_time = Time.now
5923
+ end
5924
+ end
5925
+ if stderr
5926
+ stderr.each do |err|
5927
+ log << {time: Time.now, message: err, type: "stderr"}
5928
+ end
5929
+ end
5930
+ rescue Errno::EIO => e
5931
+ log_error(e)
5932
+ if !log.empty?
5933
+ temp_log = log
5934
+ @exp.upload_temp_log(temp_log) unless temp_log.empty?
5935
+ log -= temp_log
5936
+ end
5937
+ rescue Errno::ENOENT => e
5938
+ exp_success = false
5939
+ log_message("command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED)
5940
+ log_error(e)
5941
+ rescue => e
5942
+ res = @exp.end(log, 1, start_commit, 0, 0)
5943
+ log_message("Error occurred,aborting", Thor::Shell::Color::RED)
5944
+ log_error(e)
5945
+ exit(0)
5946
+ end
5947
+ ::Process.wait pid
5948
+ end
5949
+ end
5950
+
5879
5951
  end
5880
5952
  end
5881
5953
 
data/lib/cnvrg/files.rb CHANGED
@@ -730,7 +730,11 @@ module Cnvrg
730
730
  end
731
731
  res = Cnvrg::API.request(@base_resource + "download_files", 'POST', {files: files, commit: commit})
732
732
  unless Cnvrg::CLI.is_response_success(res, false)
733
- raise SignalException.new("Cant download files from the server.")
733
+ begin
734
+ puts(res)
735
+ rescue
736
+ end
737
+ raise StandardError.new("Cant download files from the server.")
734
738
  end
735
739
  self.download_multiple_files_s3(res['result'], @project_home, postfix: postfix, progress: progress, threads: threads)
736
740
  end
@@ -3,7 +3,7 @@ require 'cnvrg/helpers/agent'
3
3
  class Cnvrg::Helpers::Executer
4
4
  attr_reader :machine_activity, :agent_id, :main_id
5
5
  MAIN_CONTAINER_PORT = ENV['MAIN_CONTAINER_PORT'].try(:to_i) || 4000
6
-
6
+ HAS_DOCKER = ENV['HAS_DOCKER'] == "true"
7
7
 
8
8
  ### this class represent a machine_activity. it will poll the commands, communicate with the
9
9
  # server (poll commands) and let the server know the status of this executer.
@@ -84,7 +84,7 @@ class Cnvrg::Helpers::Executer
84
84
  agent_id = nil
85
85
  main_id = nil
86
86
  timeout = 2
87
- timeout = nil if !@is_new_main
87
+ timeout = nil if (!@is_new_main || HAS_DOCKER)
88
88
  Timeout.timeout(timeout) do
89
89
  while agent_id.blank? or main_id.blank?
90
90
  grep_by = @job_id
data/lib/cnvrg/project.rb CHANGED
@@ -328,15 +328,21 @@ module Cnvrg
328
328
  end
329
329
 
330
330
  def get_storage_client
331
- response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/client", 'GET')
332
- if Cnvrg::CLI.is_response_success(response, false)
333
-
331
+ client_params = nil
332
+ i = 0
333
+ begin
334
+ response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/client", 'GET')
335
+ unless Cnvrg::CLI.is_response_success(response, false)
336
+ raise StandardError.new("Can't find project credentials")
337
+ end
334
338
  client_params = response['client']
335
- else
336
-
339
+ rescue StandardError
340
+ i += 1
341
+ sleep(5 * i)
342
+ retry if i < 10
337
343
  client_params = get_storage_client_fallback
338
344
  end
339
-
345
+ raise StandardError.new("Can't find project credentials") unless client_params
340
346
  Cnvrg::Downloader::Client.factory(client_params)
341
347
  end
342
348
 
@@ -381,10 +387,12 @@ module Cnvrg
381
387
  def generate_output_dir(output_dir, local: false)
382
388
  Cnvrg::Logger.log_info("Generating output dir for #{output_dir}")
383
389
  upload_list = []
390
+ list = []
384
391
  list = Dir.glob("/cnvrg/#{output_dir}/**/*", File::FNM_DOTMATCH)
385
392
  if local
386
393
  list += Dir.glob("#{output_dir}/**/*", File::FNM_DOTMATCH)
387
394
  end
395
+ list.uniq!
388
396
  Parallel.map(list, in_threads: IDXParallelThreads) do |e|
389
397
  next if e.end_with? "/."
390
398
  if File.directory? e
@@ -449,6 +457,10 @@ module Cnvrg
449
457
  if list_ignore_new.include? label
450
458
  next
451
459
  end
460
+ if File.symlink?(e)
461
+ Cnvrg::Logger.log_info("Skipping symlink #{e}")
462
+ next
463
+ end
452
464
  if File.directory? e
453
465
  dir_name = (label.ends_with? "/") ? label : (label + "/")
454
466
  tree_idx[dir_name] = nil
@@ -649,7 +661,11 @@ module Cnvrg
649
661
 
650
662
  def fetch_webapp_slugs(webapp_slug, slugs: nil)
651
663
  response = Cnvrg::API_V2.request("#{self.owner}/projects/#{self.slug}/webapps/#{webapp_slug}" , 'GET')
652
- return response["experiments"]
664
+
665
+ if response.key?("experiments")
666
+ return response["experiments"]
667
+ end
668
+ return response["data"]["attributes"]["experiments"]
653
669
  rescue
654
670
  slugs
655
671
  end
@@ -701,8 +717,11 @@ module Cnvrg
701
717
  res = JSON.parse(resp['result']) rescue nil
702
718
  return if res.blank?
703
719
  config = self.get_config
704
- config[:is_git] = res['git']
705
720
  config[:project_name] = res['title']
721
+ config[:project_slug] = @slug
722
+ config[:owner] = @owner
723
+ config[:git] = res['git'] || false
724
+ config[:is_git] = res['git'] || false
706
725
  self.set_config(config)
707
726
  end
708
727
 
data/lib/cnvrg/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Cnvrg
2
- VERSION = '2.0.1'
2
+ VERSION = '2.0.15'
3
3
  end
metadata CHANGED
@@ -1,16 +1,16 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cnvrg
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.1
4
+ version: 2.0.15
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yochay Ettun
8
8
  - Leah Kolben
9
9
  - Omer Shacham
10
- autorequire:
10
+ autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2021-06-16 00:00:00.000000000 Z
13
+ date: 2021-12-12 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: bundler
@@ -473,7 +473,7 @@ files:
473
473
  homepage: https://cnvrg.io
474
474
  licenses: []
475
475
  metadata: {}
476
- post_install_message:
476
+ post_install_message:
477
477
  rdoc_options: []
478
478
  require_paths:
479
479
  - lib
@@ -488,8 +488,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
488
488
  - !ruby/object:Gem::Version
489
489
  version: '0'
490
490
  requirements: []
491
- rubygems_version: 3.0.9
492
- signing_key:
491
+ rubygems_version: 3.2.32
492
+ signing_key:
493
493
  specification_version: 4
494
494
  summary: A CLI tool for interacting with cnvrg.io.
495
495
  test_files: []