cnvrg 0.7.7 → 0.7.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -19,6 +19,42 @@ module Cnvrg
19
19
  end
20
20
  end
21
21
 
22
+ def try_until_success(tries: 5)
23
+ (0..tries).each do |i|
24
+ begin
25
+ yield
26
+ return true
27
+ rescue Exception => e
28
+ Cnvrg::Logger.log_info("Error while trying for the #{i} time")
29
+ Cnvrg::Logger.log_error(e)
30
+ sleep(5)
31
+ rescue => e
32
+ Cnvrg::Logger.log_info("Error while trying for the #{i} time")
33
+ Cnvrg::Logger.log_error(e)
34
+ sleep(5)
35
+ end
36
+ end
37
+ raise StandardError.new("Failed")
38
+ end
39
+
40
+ def get_config
41
+ home_dir = File.expand_path('~')
42
+ config = {}
43
+ begin
44
+ if File.exist? home_dir+"/.cnvrg/config.yml"
45
+ config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
46
+ end
47
+ end
48
+ return config
49
+ end
50
+
51
+ def set_config(config)
52
+ home_dir = File.expand_path('~')
53
+ File.open("#{home_dir}/.cnvrg/config.yml", "w"){|f| f.write config.to_yaml }
54
+ return config
55
+ end
56
+
57
+
22
58
  def remote_url
23
59
  home_dir = File.expand_path('~')
24
60
  config = ""
@@ -38,6 +74,21 @@ module Cnvrg
38
74
  return config.to_h[:api].gsub("/api", "")
39
75
  end
40
76
  end
77
+
78
+ def server_version
79
+ config = self.get_config
80
+ config[:version].try(:to_i) || 0
81
+ end
82
+
83
+ def update_version(version)
84
+ config = self.get_config
85
+ if config[:version].to_s.eql? version
86
+ return
87
+ end
88
+ config[:version] = version
89
+ self.set_config(config)
90
+ end
91
+
41
92
  def is_verify_ssl
42
93
  home_dir = File.expand_path('~')
43
94
  config = ""
@@ -94,6 +145,7 @@ module Cnvrg
94
145
  end
95
146
 
96
147
  def cnvrgignore_content
148
+ #TODO: cnvrg ignore add .conflict
97
149
  %{
98
150
  # cnvrg ignore: Ignore the following directories and files
99
151
  # for example:
@@ -101,6 +153,8 @@ module Cnvrg
101
153
  # some_file.txt
102
154
  .git*
103
155
  .gitignore
156
+ *.conflict
157
+ *.deleted
104
158
  }.strip
105
159
  end
106
160
 
@@ -0,0 +1,21 @@
1
+ module Cnvrg
2
+ class Hyper
3
+ def initialize(project_path, path)
4
+ @project = Cnvrg::Project.new(project_path)
5
+ @content = YAML.load_file(path)
6
+ @base_resource = "users/#{@project.owner}"
7
+ @params = []
8
+ end
9
+
10
+ def resolve_params
11
+ resp = Cnvrg::API.request(@base_resource + "/resolve_grid", "POST", {hyper_search: @content})
12
+ unless Cnvrg::CLI.is_response_success(resp, false)
13
+ return nil
14
+ end
15
+ resp['result']['params'].each do |param|
16
+ @params << {key: param.first.keys.first, value: param.map{|p| p.values}.flatten.join(',')}
17
+ end
18
+ @params
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,102 @@
1
+ module Cnvrg
2
+ module Logger
3
+ extend self
4
+ def log_handler
5
+ begin
6
+ date = DateTime.now.strftime("%m_%d_%Y")
7
+ home_dir = File.expand_path('~')
8
+
9
+ if !File.directory? home_dir + "/.cnvrg"
10
+ FileUtils.mkdir_p([home_dir + "/.cnvrg", home_dir + "/.cnvrg/tmp"])
11
+ end
12
+ if !File.exist?(home_dir + "/.cnvrg/config.yml")
13
+ FileUtils.touch [home_dir + "/.cnvrg/config.yml"]
14
+ end
15
+ logfile = File.expand_path('~') + "/.cnvrg/log_#{date}.log"
16
+ if !File.exist? logfile
17
+ FileUtils.touch([logfile])
18
+ yesterday = get_start_day - 86399
19
+ date = yesterday.strftime("%m_%d_%Y")
20
+
21
+ logfile_old = File.expand_path('~') + "/.cnvrg/log_#{date}.log"
22
+ count = 0
23
+ while not File.exist? logfile_old and count < 60
24
+ yesterday = yesterday - 86399
25
+ date = yesterday.strftime("%m_%d_%Y")
26
+ logfile_old = File.expand_path('~') + "/.cnvrg/log_#{date}.log"
27
+ count += 1
28
+ end
29
+ if File.exist? logfile_old
30
+ @files = Cnvrg::Files.new(Cnvrg::CLI.get_owner, "")
31
+ @files.upload_log_file(logfile_old, "log_#{date}.log", yesterday)
32
+ FileUtils.remove logfile_old
33
+ end
34
+
35
+ end
36
+ config = LogStashLogger.configure do |config|
37
+ config.customize_event do |event|
38
+ event.remove('@version')
39
+ event.remove('severity')
40
+ end
41
+ end
42
+ $log = LogStashLogger.new(type: :file, path: logfile, sync: true, config: config)
43
+ self.remove_old_log_files
44
+ rescue
45
+ end
46
+ end
47
+
48
+ def remove_old_log_files()
49
+ begin
50
+ last_week = (Time.now - (7 * 24 * 60 * 60)).strftime("%Y-%m-%d")
51
+ home = File.expand_path('~')
52
+ log_files = Dir["#{home}/.cnvrg/tmp/*.log"]
53
+ log_files.each do |l|
54
+ if File.mtime(l).strftime("%Y-%m-%d") < last_week
55
+ FileUtils.rm_rf(l)
56
+ end
57
+ end
58
+ end
59
+ end
60
+
61
+ def info(msg)
62
+ self.log_info(msg)
63
+ end
64
+
65
+ def log_method(bind: nil)
66
+ return if bind.blank?
67
+ Cnvrg::Logger.log_handler if $log.blank?
68
+ arg = {}
69
+ bind.local_variables.map do |name|
70
+ arg[name] = bind.local_variable_get(name).try(:to_s)
71
+ end
72
+ method = bind.eval('__method__')
73
+ $log.info method: method, args: arg
74
+ end
75
+
76
+ def log_error(e)
77
+ Cnvrg::Logger.log_handler if $log.blank?
78
+ bc = ActiveSupport::BacktraceCleaner.new
79
+ bc.add_silencer{|line| line =~ /thor/}
80
+ $log.error message: "An exception #{e.class} was logged during running", type: "error"
81
+ backtrace = bc.clean(e.backtrace).slice(0,6).map.with_index{|backtrace,idx| "(#{idx}) - #{backtrace}"}.join("; ")
82
+ $log.error message: e.message, type: "error", backtrace: backtrace
83
+ end
84
+
85
+ def log_error_message(msg)
86
+ Cnvrg::Logger.log_handler if $log.blank?
87
+ $log.error message: msg, type: "error"
88
+ end
89
+
90
+ def log_info(msg)
91
+ Cnvrg::Logger.log_handler if $log.blank?
92
+ $log.info message: msg, type: "info"
93
+ end
94
+
95
+ def jsonify_message(msg: '', success: true)
96
+ puts JSON[{
97
+ "msg": msg,
98
+ "success": success.to_s
99
+ }]
100
+ end
101
+ end
102
+ end
@@ -1,7 +1,7 @@
1
1
  require 'fileutils'
2
2
  module Cnvrg
3
3
  class Project
4
- attr_reader :slug, :owner, :title, :local_path, :working_dir, :is_git, :is_branch
4
+ attr_reader :slug, :owner, :title, :local_path, :working_dir, :is_git, :is_branch, :machines
5
5
 
6
6
  RemoteURL ||= "https://cnvrg.io"
7
7
  IDXParallelThreads ||= 15
@@ -17,10 +17,16 @@ module Cnvrg
17
17
  @owner = config[:owner]
18
18
  @is_branch = config[:is_branch]
19
19
  @is_git = config[:git] || false
20
+ @base_resource = "users/#{@owner}/projects/#{@slug}/"
21
+ @machines = nil
20
22
  rescue => e
21
23
  end
22
24
  end
23
25
 
26
+ def base_resource
27
+ "users/#{@owner}/projects/#{@slug}/"
28
+ end
29
+
24
30
 
25
31
  def last_local_commit
26
32
  idx = YAML.load_file(@local_path + "/.cnvrg/idx.yml")
@@ -164,7 +170,7 @@ module Cnvrg
164
170
  File.open(project_name + "/.cnvrgignore", "w+") { |f| f.write cnvrgignore }
165
171
  File.open(project_name + "/README.md", "w+") { |f| f.write cnvrgreadme }
166
172
  File.open(project_name + "/src/hyper.yaml", "w+") { |f| f.write cnvrghyper }
167
-
173
+
168
174
  rescue
169
175
  return false
170
176
  end
@@ -343,8 +349,7 @@ module Cnvrg
343
349
  if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
344
350
  old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
345
351
  else
346
-
347
- old_idx = nil
352
+ old_idx = {:tree => {}, :commit => nil}
348
353
  end
349
354
 
350
355
  tree_idx = Hash.new(0)
@@ -376,15 +381,25 @@ module Cnvrg
376
381
  end
377
382
  end
378
383
 
379
- idx = {commit: old_idx.to_h[:commit], tree: tree_idx}
384
+ old_idx[:tree] = tree_idx
380
385
 
381
- File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
386
+ File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write old_idx.to_yaml }
382
387
  return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
383
388
  end
384
389
  def get_idx
390
+ unless File.exists? "#{self.local_path}/.cnvrg/idx.yml"
391
+ empty_idx = {:commit => nil, :tree => {}}
392
+ File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write empty_idx.to_yaml }
393
+ return empty_idx
394
+ end
385
395
  YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
386
396
  end
387
397
 
398
+ def set_idx(idx)
399
+ FileUtils.mkdir_p("#{self.local_path}/.cnvrg")
400
+ File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
401
+ end
402
+
388
403
  def clone(remote=0, commit)
389
404
  response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/clone", 'POST', {project_slug: self.slug, remote: remote, commit: commit})
390
405
  return response
@@ -395,34 +410,70 @@ module Cnvrg
395
410
  return response
396
411
  end
397
412
 
398
- def compare_idx(new_branch, commit:last_local_commit,force:false, deploy: false, in_exp:false, specific_files: [])
399
- if commit.nil?
400
- local_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
413
+ def compare_idx(new_branch, force:false, deploy: false, in_exp:false, specific_files: [], download: false)
414
+ is_download = download
415
+ if is_download
416
+ local_idx = self.get_idx
401
417
  else
418
+ #upload
402
419
  local_idx = self.generate_idx(deploy: deploy)
403
420
  end
421
+ commit = local_idx[:commit]
422
+ tree = local_idx[:tree]
404
423
  ignore_list = self.send_ignore_list()
405
- if force or !specific_files.blank?
424
+ if force or specific_files.present?
406
425
  added = []
407
- if !specific_files.blank?
426
+ if specific_files.present?
408
427
  added = specific_files
409
- elsif local_idx[:tree]
428
+ elsif tree.present?
410
429
  added << local_idx[:tree].keys
411
430
  added.flatten!
412
431
  end
413
432
  response ={"result"=> {"commit"=>nil,"tree"=> {"added"=> added,
414
433
  "updated_on_server"=> [],
415
434
  "updated_on_local"=> [],
435
+ "update_local" => [],
416
436
  "deleted"=> [],
417
437
  "conflicts"=> []} } }
418
438
  return response
419
-
420
439
  end
421
- response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/status", 'POST', {idx: local_idx, new_branch: new_branch,
422
- current_commit: commit,ignore:ignore_list, force:force,in_exp:in_exp})
440
+ #we dont want to send it on download - we only compare between commits sha1 in download.
441
+ if is_download
442
+ #the new server doesnt need the tree, but the old probably needs :X
443
+ local_idx[:tree] = {} if Cnvrg::Helpers.server_version > 0
444
+ end
445
+ response = Cnvrg::API.request(@base_resource + "status", 'POST', {idx: local_idx, new_branch: new_branch,
446
+ current_commit: commit,ignore:ignore_list, force:force,in_exp:in_exp, download: download})
447
+
423
448
  CLI.is_response_success(response,true)
449
+ if is_download
450
+ if Cnvrg::Helpers.server_version > 0
451
+ #trying to optimize the download using resolver
452
+ resolve = response['result']['tree']['resolver'] || tree #tree of file -> sha1 from current commit to check conflicts
453
+ destination_files = response['result']['tree']['destination'] || {} #tree of file -> sha1 from last commit to check files that already downloaded
454
+ else
455
+ resolve = tree
456
+ destination_files = {}
457
+ end
458
+ @files = self.get_files
459
+ local_tree = @files.calculate_sha1(resolve.keys)
460
+ changed_files = resolve.keys.select{|file| local_tree[file] != resolve[file]}
461
+
462
+ # means that the user changed the file locally
463
+ response['result']['tree']['update_local'] = changed_files
464
+
465
+ # means that we already downloaded this file and we dont need it anymore
466
+ downloaded_files = destination_files.keys.select{|file| local_tree[file] == destination_files[file]}
467
+ response['result']['tree']['added'] -= downloaded_files
468
+ response['result']['tree']['updated_on_server'] -= downloaded_files
469
+ end
424
470
  return response
425
471
  end
472
+
473
+ def get_files
474
+ Cnvrg::Files.new(self.owner, self.slug, project_home: @local_path)
475
+ end
476
+
426
477
  def jump_idx(destination: self.last_local_commit)
427
478
  local_idx = self.generate_idx
428
479
  ignore_list = self.send_ignore_list
@@ -497,7 +548,7 @@ module Cnvrg
497
548
  return response
498
549
  end
499
550
  def update_idx_with_commit!(commit)
500
- idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
551
+ idx_hash = self.get_idx
501
552
  idx_hash[:commit] = commit
502
553
 
503
554
  File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
@@ -506,11 +557,27 @@ module Cnvrg
506
557
 
507
558
  def revert(working_dir)
508
559
  FileUtils.rm_rf working_dir
509
- # response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/revert", 'GET')
510
- # CLI.is_response_success(response)
560
+ end
511
561
 
562
+ def init_machines
563
+ Cnvrg::Logger.log_info("Init machines.")
564
+ resp = Cnvrg::API.request("users/#{@owner}/machines", "GET")
565
+ return unless Cnvrg::CLI.is_response_success(resp, false)
566
+ @machines = resp['result']['machines']
567
+ end
512
568
 
513
- end
569
+ def get_machines
570
+ init_machines if @machines.nil?
571
+ @machines = false if @machines.nil?
572
+ @machines || []
573
+ end
574
+
575
+ def check_machine(machine)
576
+ Cnvrg::Logger.log_info("Check if #{machine} machine exists")
577
+ machines = get_machines
578
+ return true if machines.blank?
579
+ machines.include? machine
580
+ end
514
581
 
515
582
  end
516
583
  end
@@ -0,0 +1,165 @@
1
+ module Cnvrg
2
+ class Task
3
+ attr_accessor :title, :path, :source
4
+ def initialize(project_path, path: nil, content: {})
5
+ @path = path
6
+ @content = content
7
+ @source = 'flow' if @content.present?
8
+ @source = 'file' if @path.present?
9
+ @project = Cnvrg::Project.new(project_path)
10
+ @project_path = project_path
11
+ #generic
12
+ @type = nil #type can be exec, data, library, deploy
13
+ @compute = 'medium'
14
+ @title = nil
15
+ @uid = nil
16
+ @params = {}
17
+
18
+ #exec
19
+ @cmd = nil
20
+ @params_path = nil
21
+
22
+ #library
23
+ @library = nil
24
+
25
+ #dataset
26
+ @dataset = nil
27
+ @query = nil
28
+
29
+ #deploy
30
+ @function = nil
31
+
32
+
33
+ @base_resource = @project.base_resource + "/tasks"
34
+
35
+ self.reload_task
36
+ end
37
+
38
+ def save
39
+ path = @path || gen_path
40
+ File.open(path, 'w'){|f| f.write(get_content.to_yaml)}
41
+ end
42
+
43
+ def reload_task
44
+ task_raw = get_content
45
+ @title = task_raw[:title]
46
+ @type = task_raw[:type]
47
+ @uid = task_raw[:uid]
48
+ @title = task_raw[:title] || @uid
49
+ @compute = task_raw[:compute] || @compute
50
+
51
+ case @type
52
+ when 'exec'
53
+ @cmd = task_raw[:cmd]
54
+ init_params(task_raw)
55
+ when 'library'
56
+ @library = task_raw[:library]
57
+ init_params(task_raw)
58
+ when 'data'
59
+ @dataset = task_raw[:dataset]
60
+ @query = task_raw[:query]
61
+ when 'deploy'
62
+ @cmd = task_raw[:cmd]
63
+ @function = task_raw[:function]
64
+ else
65
+ error("Cant parse task of type #{@type}")
66
+ end
67
+ end
68
+
69
+ def verify_task
70
+ case @type
71
+ when 'exec'
72
+ verify_exec
73
+ when 'data'
74
+ verify_data
75
+ when 'deploy'
76
+ verify_deploy
77
+ when 'library'
78
+ verify_library
79
+ else
80
+ error("Cant parse task of type #{@type}")
81
+ end
82
+ end
83
+
84
+
85
+ def to_api
86
+ get_content.merge(params: @params)
87
+ end
88
+
89
+ def run
90
+ verify_task
91
+ if @type == 'data'
92
+ raise StandardError.new("Data Tasks are not runnable")
93
+ end
94
+ resp = Cnvrg::API.request(@base_resource, "POST", {task: to_api})
95
+ Cnvrg::CLI.is_response_success(resp, true)
96
+ Cnvrg::Helpers.remote_url + resp['result']['url']
97
+ end
98
+
99
+ private
100
+ def verify_compute
101
+ unless @project.check_machine(@compute)
102
+ raise StandardError.new("Cant find #{@compute} machine in project.")
103
+ end
104
+ end
105
+
106
+ def init_params(task_raw)
107
+ @params_path = task_raw[:params_path].presence
108
+ @params = task_raw[:params] || @params
109
+ if @params_path.present?
110
+ @hyper = Cnvrg::Hyper.new(@project_path, @params_path)
111
+ @params = @hyper.resolve_params
112
+ end
113
+ end
114
+
115
+ def verify_exec
116
+ if @cmd.blank?
117
+ error("Cant find command")
118
+ end
119
+ verify_compute
120
+ end
121
+
122
+ def verify_data
123
+ if @dataset.blank?
124
+ error("Cant find dataset slug")
125
+ end
126
+ end
127
+
128
+ def verify_deploy
129
+ error("Cant find command") if @cmd.blank?
130
+ error("Cant find function") if @function.blank?
131
+ verify_compute
132
+ end
133
+
134
+ def verify_library
135
+ error("Cant find library") if @library.blank?
136
+ end
137
+
138
+ def get_content
139
+ return @content if @source == 'flow'
140
+ unless File.exists? @path
141
+ raise StandardError.new("Cant find task in #{@path}")
142
+ end
143
+ YAML.load_file(@path)
144
+ end
145
+
146
+ def gen_path
147
+ @title ||= "#{@type.capitalize}Task"
148
+ unless File.exists? @title
149
+ @path = "#{@title}.task.yaml"
150
+ return @path
151
+ end
152
+ i = 0
153
+ while File.exists? "#{@title}_#{i}.task.yaml"
154
+ i += 1
155
+ end
156
+ @path = "#{@title}.task.yaml"
157
+ return @path
158
+ end
159
+
160
+ def error(msg)
161
+ raise StandardError.new("task: #{@uid} - #{msg}")
162
+ end
163
+
164
+ end
165
+ end