cnvrg 0.7.7 → 0.7.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,116 @@
1
+ require 'cnvrg/cli'
2
+ require 'cnvrg/data'
3
+ require 'cnvrg/task'
4
+ require 'cnvrg/flow'
5
+ require 'cnvrg/hyper'
6
+ require 'cnvrg/cli/subcommand'
7
+ require 'cnvrg/cli/task'
8
+ require 'thor'
9
+ module Cnvrg
10
+ module Commands
11
+ class Flow < SubCommand
12
+
13
+ def initialize(*args)
14
+ super
15
+ unless @curr_dir.present?
16
+ @cli.log_message("Not On Project Dir, exiting", Thor::Color::RED)
17
+ exit(1)
18
+ end
19
+ end
20
+
21
+ desc "task", "Running Flow tasks", :hide => true
22
+ subcommand 'task', Cnvrg::Commands::Task
23
+
24
+ desc "flow verify", "verify that the flow is well formated"
25
+
26
+ def verify(path)
27
+ unless @curr_dir.present?
28
+ @cli.log_message("Cant run this command because you are not in project directory", Thor::Color::RED)
29
+ return false
30
+ end
31
+ @flow = Cnvrg::Flows.new(@curr_dir, path)
32
+ @cli.log_message("The Flow is Valid", Thor::Color::GREEN)
33
+ end
34
+
35
+
36
+ desc "flow run", "run a flow file"
37
+
38
+ def run(path)
39
+ unless @curr_dir.present?
40
+ @cli.log_message("Cant run this command because you are not in project directory", Thor::Color::RED)
41
+ return false
42
+ end
43
+ begin
44
+ @flow = Cnvrg::Flows.new(@curr_dir, path)
45
+ url = @flow.run
46
+ rescue => e
47
+ @cli.log_message("Cant run the flow, please run 'cnvrg flow verify #{path}' to check it", Thor::Color::RED)
48
+ @cli.log_error(e)
49
+ return
50
+ end
51
+ check = Cnvrg::Helpers.checkmark
52
+ @cli.log_message("#{check} The flow is running, you can track the status in: #{url}", Thor::Color::GREEN)
53
+ true
54
+ end
55
+
56
+ desc "flow create", "create a flow file"
57
+
58
+ def create
59
+ title = ask("Flow Title: ")
60
+ title = title.presence || "cnvrg-flow"
61
+ relations = []
62
+ task = ask_for_relation
63
+ while task.compact.size == 2
64
+ from, to = task
65
+ relations << {from: from, to: to}
66
+ task = ask_for_relation
67
+ end
68
+ start_commit = ask("Flow Starting Commit (empty will set this value to 'latest')")
69
+ Cnvrg::Flows.create_flow("#{title}.flow.yaml", {title: title, relations: relations, start_commit: start_commit})
70
+ end
71
+
72
+
73
+ # desc "flow resolve", "Resolve flow parameters"
74
+ # def resolve(path)
75
+ # @hyper = Cnvrg::Hyper.new(@curr_dir, path)
76
+ # @hyper.resolve_params
77
+ # end
78
+ #
79
+
80
+ private
81
+
82
+ def init_tasks
83
+ @tasks = Dir.glob("**/*.task*")
84
+ end
85
+
86
+ def ask_for_relation
87
+ init_tasks if @tasks.blank?
88
+ to = nil
89
+ from = ask_for_task("Task To Start From: [#{@tasks.join(', ')}]")
90
+ to = ask_for_task("Task To Go To: [#{@tasks.join(', ')}]") if from.present?
91
+ [from, to]
92
+ end
93
+
94
+ def ask_for_task(text)
95
+ verified = false
96
+ task = nil
97
+ while !verified
98
+ task = ask(text)
99
+ if task.blank?
100
+ return nil
101
+ end
102
+ begin
103
+ Cnvrg::Task.new(@curr_dir, path: task).verify_task
104
+ verified = true
105
+ rescue => e
106
+ end
107
+ end
108
+ task
109
+ end
110
+
111
+ def get_all_tasks
112
+ @tasks = Dir.glob("*/**.task.yaml")
113
+ end
114
+ end
115
+ end
116
+ end
@@ -0,0 +1,26 @@
1
+ class SubCommand < Thor
2
+ @cli = nil
3
+ @curr_dir = nil
4
+
5
+ def initialize(*args)
6
+ super
7
+ @cli = Cnvrg::CLI.new
8
+ @curr_dir = @cli.is_cnvrg_dir
9
+ end
10
+
11
+ def self.banner(command, namespace = nil, subcommand = false)
12
+ "#{basename} #{command.usage}"
13
+ end
14
+
15
+ def self.subcommand_prefix
16
+ self.name.gsub(%r{.*::}, '').gsub(%r{^[A-Z]}) {|match| match[0].downcase}.gsub(%r{[A-Z]}) {|match| "-#{match[0].downcase}"}
17
+ end
18
+
19
+ class << self
20
+ # Hackery.Take the run method away from Thor so that we can redefine it.
21
+ def is_thor_reserved_word?(word, type)
22
+ return false if word == "run"
23
+ super
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,116 @@
1
+ module Cnvrg
2
+ module Commands
3
+ class Task < SubCommand
4
+
5
+ def initialize(*args)
6
+ super
7
+ unless @curr_dir.present?
8
+ @cli.log_message("Not On Project Dir, exiting", Thor::Color::RED)
9
+ exit(1)
10
+ end
11
+ end
12
+
13
+ desc "flow task verify", "verify that the task is well formated"
14
+ def verify(path)
15
+ unless @curr_dir.present?
16
+ @cli.log_message("Cant run this command because you are not in project directory", Thor::Color::RED)
17
+ return false
18
+ end
19
+ @task = Cnvrg::Task.new(@curr_dir, path: path)
20
+ begin
21
+ @task.verify_task
22
+ rescue StandardError => e
23
+ @cli.log_message("An error during parsing task: #{e.message}", Thor::Color::RED)
24
+ @cli.log_error(e)
25
+ return false
26
+ rescue => e
27
+ @cli.log_error(e)
28
+ return false
29
+ end
30
+ @cli.log_message("The task verified successfuly", Thor::Color::GREEN)
31
+ return @task
32
+ end
33
+
34
+
35
+ desc "flow task create", "Creates new task"
36
+ def create
37
+
38
+ @project = Cnvrg::Project.new(@curr_dir)
39
+
40
+ task = {}
41
+ task[:title] = ask("Task title", Thor::Color::GREEN)
42
+ task[:type] = ask("Task Type?", Thor::Color::GREEN, :limited_to => ["exec", "data", "library", "deploy"])
43
+ case task[:type]
44
+ when "exec"
45
+ task[:cmd] = ask("Command to run", Thor::Color::GREEN)
46
+ task[:params], task[:params_path] = ask_for_params
47
+ task[:machine] = ask_for_machine
48
+ when "data"
49
+ task[:dataset] = ask("Dataset slug", Thor::Color::GREEN)
50
+ task[:query] = ask("Dataset query", Thor::Color::GREEN)
51
+ when "library"
52
+ task[:library] = ask("Library Slug", Thor::Color::GREEN)
53
+ task[:params], task[:params_path] = ask_for_params
54
+ task[:machine] = ask_for_machine
55
+ when "deploy"
56
+ task[:cmd] = ask("Command to run", Thor::Color::GREEN)
57
+ task[:function] = ask("Function to run", Thor::Color::GREEN)
58
+ end
59
+ @task = Cnvrg::Task.new(@curr_dir, content: task)
60
+ @task.save
61
+ @cli.log_message("Task #{@task.title} Saved Successfuly", Thor::Color::GREEN)
62
+ end
63
+
64
+
65
+ desc "flow task run", "Running specific task"
66
+ def run(path)
67
+ begin
68
+ @task = Cnvrg::Task.new(@curr_dir, path: path)
69
+ url = @task.run
70
+ @cli.log_message("Task: #{@task.title} is running, you can track its performance on #{url}", Thor::Color::GREEN)
71
+ rescue => e
72
+ @cli.log_message(e.message, Thor::Color::RED)
73
+ @cli.log_error(e)
74
+ end
75
+ end
76
+
77
+ private
78
+
79
+ def ask_for_params
80
+ params = []
81
+ param_key = ask("Param key: (enter for continue)", Thor::Color::GREEN)
82
+ while param_key.present?
83
+ param_value = ask("#{param_key} value: (seperated by commas)", Thor::Color::GREEN)
84
+ params << {key: param_key, value: param_value}
85
+ param_key = ask("Param key: (enter for continue)", Thor::Color::GREEN)
86
+ end
87
+ if params.blank?
88
+ params_path = ask("Params from file: ")
89
+ if params_path.present?
90
+ while params_path.present? and !File.exists? params_path
91
+ params_path = ask("Cant find file, please write it again.")
92
+ end
93
+ end
94
+ end
95
+ return params, params_path
96
+ end
97
+
98
+ def ask_for_machine
99
+ options = {}
100
+ all_machines = @project.get_machines
101
+ options[:limited_to] = all_machines if all_machines.present?
102
+ ask("Kind of machine to execute on?", Thor::Color::GREEN, options).presence || "medium"
103
+ end
104
+
105
+
106
+ def on_project
107
+ unless @curr_dir.present?
108
+ @cli.log_message("Cant run this command because you are not in project directory", Thor::Color::RED)
109
+ return false
110
+ end
111
+ end
112
+
113
+
114
+ end
115
+ end
116
+ end
@@ -11,7 +11,6 @@ class SubCommandBase < Thor
11
11
  end
12
12
 
13
13
  module Cnvrg
14
-
15
14
  class Data < SubCommandBase
16
15
  class_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
17
16
  desc "data init", "init data folder"
@@ -81,46 +81,62 @@ module Cnvrg
81
81
  end
82
82
 
83
83
 
84
- def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, progressbar: nil, prefix: '')
85
- random_file_name = (0...8).map { (65 + rand(26)).chr }.join #needed to create a file for post_file..
86
- # each file have: {sha1: sha1, file_name: file_name, file_size: file_size, content_type: content_type, absolute_path, relative_path}
87
-
88
- #this call should also implement compare_idx...
84
+ def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, progressbar: nil, prefix: '', partial_commit: nil)
89
85
  begin
90
- upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {commit_sha1: commit_sha1, tree: tree, force: force, is_branch: new_branch})
91
- return Cnvrg::Result.new(false, "Failed to upload files") unless Cnvrg::CLI.is_response_success(upload_resp, false)
92
- results = upload_resp['result'].with_indifferent_access
93
- if results['files'].blank?
94
- progressbar.progress += tree.keys.length if progressbar.present?
95
- return 0
96
- end
97
- props = Cnvrg::Helpers.get_s3_props(results)
98
- client = props[:client]
99
- bucket = props[:bucket]
100
- upload_options = props[:upload_options]
101
- s3_bucket = Aws::S3::Resource.new(client: client).bucket(bucket)
102
- files = results['files']
103
- progressbar.progress += tree.keys.length - files.length if progressbar.present?
104
- Parallel.map((files.keys), {in_threads: threads}) do |k|
105
- o = tree[k].merge(files[k])
106
- upload_single_file(o, s3_bucket, upload_options)
107
- progressbar.progress += 1 if progressbar.present?
108
- end
109
- blob_ids = files.values.map{|f| f['bv_id']}
110
- dirs = tree.keys.select{|k| tree[k].nil? } || []
111
- upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
112
- return files.keys.length
86
+ Cnvrg::Logger.log_info("Sending Upload Files request")
87
+ upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {commit_sha1: commit_sha1, tree: tree, force: force, is_branch: new_branch, partial_commit: partial_commit})
88
+ unless Cnvrg::CLI.is_response_success(upload_resp, false)
89
+ Cnvrg::Logger.log_method(bind: binding)
90
+ raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
91
+ end
92
+ Cnvrg::Logger.log_info("Uploading files")
93
+ results = upload_resp['result'].with_indifferent_access
94
+ if results['files'].blank?
95
+ progressbar.progress += tree.keys.length if progressbar.present?
96
+ return 0
97
+ end
98
+ props = Cnvrg::Helpers.get_s3_props(results)
99
+ client = props[:client]
100
+ bucket = props[:bucket]
101
+ upload_options = props[:upload_options]
102
+ s3_bucket = Aws::S3::Resource.new(client: client).bucket(bucket)
103
+ files = results['files']
104
+ progressbar.progress += tree.keys.length - files.length if progressbar.present?
105
+ Parallel.map((files.keys), {in_threads: threads}) do |k|
106
+ o = tree[k].merge(files[k])
107
+ upload_single_file(o, s3_bucket, upload_options)
108
+ progressbar.progress += 1 if progressbar.present?
109
+ end
110
+ blob_ids = files.values.map {|f| f['bv_id']}
111
+ dirs = tree.keys.select {|k| tree[k].nil?} || []
112
+ Cnvrg::Logger.info("Sending Upload files save")
113
+ upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
114
+ unless Cnvrg::CLI.is_response_success(upload_resp, false)
115
+ Cnvrg::Logger.log_method(bind: binding)
116
+ raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
117
+ end
118
+ Cnvrg::Logger.log_info("Upload Success")
119
+ return files.keys.length
113
120
  rescue => e
114
- return Cnvrg::Result.new(false, "Failed to upload files")
121
+ Cnvrg::Logger.log_method(bind: binding)
122
+ Cnvrg::Logger.log_error(e)
123
+ raise e
115
124
  end
116
125
 
117
126
  end
118
127
 
119
128
  def upload_single_file(file, s3_bucket, upload_options={})
129
+ begin
120
130
  file = file.with_indifferent_access
121
- resp = s3_bucket.
122
- object(file[:path]).
123
- upload_file(file[:absolute_path], upload_options)
131
+ Cnvrg::Logger.log_info("Uploading #{file[:absolute_path]}")
132
+ obj = s3_bucket.
133
+ object(file[:path])
134
+ obj.upload_file(file[:absolute_path], upload_options)
135
+ Cnvrg::Logger.log_info("#{file[:absolute_path]} uploaded.")
136
+ rescue => e
137
+ Cnvrg::Logger.log_error_message("Error while upload single file #{file[:path]}")
138
+ Cnvrg::Logger.log_error(e)
139
+ end
124
140
  end
125
141
 
126
142
  def upload_file(absolute_path, relative_path, commit_sha1)
@@ -809,10 +825,13 @@ module Cnvrg
809
825
  Cnvrg::CLI.is_response_success(response, true)
810
826
  return response
811
827
  end
812
- def start_commit(new_branch,force=false,delete_commit=nil, chunks: 0, current_commit: nil)
828
+ def start_commit(new_branch,force=false,delete_commit=nil, chunks: 0, dataset: nil)
813
829
  begin
830
+ idx = dataset.get_idx
831
+ commit = idx[:commit]
832
+ next_commit = idx[:next_commit]
814
833
  response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch,force:force,
815
- username: @owner,current_commit:current_commit, total_chunks: chunks})
834
+ username: @owner,current_commit: commit, next_commit: next_commit, total_chunks: chunks})
816
835
  Cnvrg::CLI.is_response_success(response, true)
817
836
  return response
818
837
  rescue => e
@@ -944,11 +963,12 @@ module Cnvrg
944
963
  file_path = "#{file_path}.conflict" if conflict
945
964
  dirname = File.dirname file_path
946
965
  download_dir(project_home, dirname) unless dirname.eql? "."
947
-
966
+ Cnvrg::Logger.info("Download #{file_path}")
948
967
  File.open(project_home+"/"+file_path, 'w+') do |file|
949
968
  resp = client.get_object({bucket:bucket,
950
969
  key:file_key}, target: file)
951
970
  end
971
+ Cnvrg::Logger.log_info("Download #{file_path} success")
952
972
 
953
973
  if resp
954
974
  download_succ_count +=1
@@ -20,6 +20,20 @@ module Cnvrg
20
20
  end
21
21
  end
22
22
 
23
+
24
+ def backup_idx
25
+ Cnvrg::Logger.log_info("Backup idx")
26
+ idx = self.get_idx
27
+ File.open("#{self.local_path}/.cnvrg/idx.yml.backup", 'w') { |f| f.write idx.to_yaml }
28
+ end
29
+
30
+ def restore_idx
31
+ Cnvrg::Logger.log_info("Restore idx because an error.")
32
+ Cnvrg::Logger.log_method(bind: binding)
33
+ idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml.backup")
34
+ self.set_idx(idx)
35
+ end
36
+
23
37
  def change_url(owner: '', slug: '', title: '')
24
38
  config = {dataset_home: title, dataset_slug: slug, owner: owner}
25
39
  File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
@@ -293,9 +307,18 @@ module Cnvrg
293
307
  end
294
308
 
295
309
 
296
- def get_idx
297
- YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
310
+ def get_idx
311
+ if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
312
+ return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
298
313
  end
314
+ {commit: nil, tree: {}}
315
+
316
+ end
317
+
318
+ def set_idx(idx)
319
+ File.open("#{self.local_path}/.cnvrg/idx.yml", 'w+') { |f| f.write idx.to_yaml }
320
+ end
321
+
299
322
  def url
300
323
  url = Cnvrg::Helpers.remote_url
301
324
  "#{url}/#{self.owner}/datasets/#{self.slug}"
@@ -342,8 +365,9 @@ module Cnvrg
342
365
  self.revert_next_commit
343
366
  end
344
367
 
345
- def list_all_files
368
+ def list_all_files(with_ignore=false)
346
369
  list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~/^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~/^#{self.local_path}\/\.cnvrgignore/) }
370
+ return list if with_ignore
347
371
  list_ignore = self.get_ignore_list.map{|ignore_file| "#{self.local_path}/#{ignore_file}"}
348
372
  list.select{|file| !list_ignore.include? file}
349
373
  end
@@ -357,6 +381,12 @@ module Cnvrg
357
381
  File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
358
382
  end
359
383
 
384
+ def write_tree(tree)
385
+ idx = self.get_idx
386
+ idx[:tree] = tree
387
+ self.set_idx(idx)
388
+ end
389
+
360
390
  def generate_idx(show_progress=false)
361
391
  if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
362
392
  old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
@@ -454,10 +484,23 @@ module Cnvrg
454
484
 
455
485
  def compare_idx_download(all_files: false, desired_commit: nil)
456
486
  current_commit = self.last_local_commit
487
+ next_commit = self.get_next_commit
457
488
  ignore_list = self.send_ignore_list()
458
- return Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/download_status", 'POST', {current_commit: current_commit, ignore:ignore_list,all_files:all_files, desired_commit: desired_commit.presence})
489
+ return Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/download_status", 'POST', {current_commit: current_commit, next_commit: next_commit, ignore:ignore_list,all_files:all_files, desired_commit: desired_commit.presence})
459
490
  end
460
-
491
+
492
+ def set_partial_commit(commit_sha1)
493
+ idx = self.get_idx
494
+ idx[:partial_commit] = commit_sha1
495
+ self.set_idx(idx)
496
+ end
497
+
498
+ def get_partial_commit
499
+ idx = self.get_idx
500
+ idx.try(:fetch, :partial_commit)
501
+ end
502
+
503
+
461
504
  def current_status(new_branch)
462
505
  commit=last_local_commit
463
506
  response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status_current", 'POST', {current_commit: commit,new_branch: new_branch})
@@ -533,10 +576,8 @@ module Cnvrg
533
576
  return nil
534
577
  end
535
578
  idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
536
- idx = Hash.new()
537
- idx[:commit] = idx_hash[:commit]
538
- idx[:tree] = idx_hash[:tree]
539
- File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
579
+ idx_hash = idx_hash.except(:next_commit)
580
+ File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
540
581
  end
541
582
  def get_current_commit()
542
583
  if !File.exist? "#{self.local_path}/.cnvrg/idx.yml"