cnvrg 0.0.8 → 0.0.9

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: f68e73ce51d096ab10bae5dd5ba07b6444c4d898
4
- data.tar.gz: a50d1f1e96a4e21d9a408d7c5c5e915888480ec2
3
+ metadata.gz: 496681d6eef33bf3a628234cce27bc4ad7ad8ade
4
+ data.tar.gz: 3b63e71e5cbe3f12e8754ca8ed0c243c9a8e11df
5
5
  SHA512:
6
- metadata.gz: e1d21cda34620db5df089a2e4db43b730c0a3eb5265d423101cd91d5e5c606e56b15be8df027bc7e4b0ff35eb1de9eddcb7ea6f4e735792fb3992e7d4f5f10b8
7
- data.tar.gz: 9e8094bb8658f213aa7af9c943e3015e38fbb7d244176da607186cefb8fd64b5221d3a7d4ce27427aba45dfe8f33fed031f5e044a165e17a7625b34dcb106084
6
+ metadata.gz: ea24653cfa006ea02c2702cf24219e99669e4db48c423b116f67003b00379d1ded3f2bf0badc95d74978e270129cf64fc176b3d83b88c8c72451cf035a8fe63c
7
+ data.tar.gz: 950a88a8e3bb820ac1c4bb66f81eeeb142e231a51c504d82c19ceb540835c01b2061e56536c334f8beb87ed8ec50b271934cd8fc996924ee74a98fdaf9ef2d92
@@ -32,7 +32,9 @@ Gem::Specification.new do |spec|
32
32
  spec.add_runtime_dependency 'highline', '~> 1.7', '>= 1.7.8'
33
33
  spec.add_runtime_dependency 'thor', '~> 0.19.0','>=0.19.1'
34
34
  spec.add_runtime_dependency 'aws-sdk'
35
+ spec.add_runtime_dependency 'sucker_punch', '~> 2.0'
35
36
  spec.add_runtime_dependency 'urlcrypt', '~> 0.1.1'
37
+ spec.add_runtime_dependency 'logstash-logger'
36
38
 
37
39
  end
38
40
 
@@ -32,7 +32,6 @@ module Cnvrg
32
32
  begin
33
33
  n = Netrc.read
34
34
  rescue => e
35
- puts e.message
36
35
  end
37
36
 
38
37
  # Make sure there is an entry for the Acquia API before generating the
@@ -18,6 +18,9 @@ require 'cnvrg/project'
18
18
  require 'cnvrg/files'
19
19
  require 'cnvrg/experiment'
20
20
  require 'etc'
21
+ require 'logstash-logger'
22
+ require 'cnvrg/job'
23
+ require 'pry'
21
24
 
22
25
 
23
26
  # DEV VERSION
@@ -27,13 +30,12 @@ module Cnvrg
27
30
 
28
31
  INSTALLATION_URLS = {docker: "https://docs.docker.com/engine/installation/", jupyter: "http://jupyter.readthedocs.io/en/latest/install.html"}
29
32
 
33
+
30
34
  desc 'version', 'Prints cnvrg current version'
31
35
 
32
36
  def version
33
-
34
37
  puts Cnvrg::VERSION
35
38
  end
36
-
37
39
  map %w(-v --version) => :version
38
40
 
39
41
  desc 'set api url', 'set api url'
@@ -132,22 +134,37 @@ module Cnvrg
132
134
  if owners.empty?
133
135
  else
134
136
  owners << choose_owner
137
+ chosen = false
138
+ while !chosen
135
139
  choose_owner = ask("Choose default owner:\n"+owners.join("\n")+"\n")
140
+ owners_lower = owners.map{|o| o.downcase}
141
+ if !owners_lower.include? choose_owner.downcase
142
+ say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
143
+ else
144
+ chosen = true
145
+ end
146
+ end
147
+
136
148
 
137
149
  end
138
150
  if set_owner(choose_owner, result["username"])
139
151
  say "Setting default owner: #{choose_owner}", Thor::Shell::Color::GREEN
152
+
140
153
  else
141
154
  say "Setting default owenr has failed, logging out", Thor::Shell::Color::RED
155
+
142
156
  return logout()
143
157
  end
144
158
 
145
159
  else
146
160
  say "Failed to authenticate, wrong email/password", Thor::Shell::Color::RED
147
- exit false
161
+
162
+ exit(1)
148
163
  end
149
164
  rescue SignalException
165
+
150
166
  say "/nAborting"
167
+ logout()
151
168
  exit(1)
152
169
  end
153
170
  end
@@ -168,18 +185,25 @@ module Cnvrg
168
185
 
169
186
  end
170
187
 
188
+
171
189
  desc 'me', 'Prints the current logged in user email'
172
190
 
173
- def me
191
+ def me()
174
192
  begin
175
- verify_logged_in()
193
+
194
+ verify_logged_in(false)
195
+ log_start(__method__,args,options)
176
196
  auth = Cnvrg::Auth.new
177
197
  if (email = auth.get_email)
178
198
  say "Logged in as: #{email}", Thor::Shell::Color::GREEN
179
199
  else
180
200
  say "You're not logged in.", Thor::Shell::Color::RED
181
201
  end
202
+
203
+ log_end(0)
182
204
  rescue SignalException
205
+ log_end(-1)
206
+
183
207
  say "/nAborting"
184
208
  exit(1)
185
209
  end
@@ -193,7 +217,8 @@ module Cnvrg
193
217
 
194
218
  def new(project_name)
195
219
  begin
196
- verify_logged_in()
220
+ verify_logged_in(false)
221
+ log_start(__method__,args,options)
197
222
  clean = options["clean"]
198
223
  docker_image = options["docker_image"]
199
224
  if !docker_image.nil? and !docker_image.empty?
@@ -202,6 +227,7 @@ module Cnvrg
202
227
  say "Creating #{project_name}", Thor::Shell::Color::BLUE
203
228
  if Dir.exists? project_name or File.exists? project_name
204
229
  say "Conflict with dir/file #{project_name}", Thor::Shell::Color::RED
230
+ log_end(1,"conflict with dir/file #{project_name}")
205
231
  exit(1)
206
232
  end
207
233
 
@@ -211,13 +237,18 @@ module Cnvrg
211
237
  @project.generate_idx
212
238
  else
213
239
  say "Error creating project, please contact support.", Thor::Shell::Color::RED
240
+ log_end(1,"can't create project #{project_name}")
241
+
214
242
  exit(0)
215
243
  end
216
244
 
217
245
  say "created\t\tproject's tree", Thor::Shell::Color::GREEN
218
246
  say "created\t\tproject's config", Thor::Shell::Color::GREEN
219
247
  say "Linked directory to\t#{@project.url}", Thor::Shell::Color::GREEN
248
+
220
249
  rescue SignalException
250
+ log_end(-1)
251
+
221
252
  say "/nAborting"
222
253
  exit(1)
223
254
  end
@@ -229,13 +260,16 @@ module Cnvrg
229
260
 
230
261
  def link
231
262
  begin
232
- verify_logged_in()
263
+ verify_logged_in(false)
264
+ log_start(__method__,args,options)
265
+
233
266
  sync = options["sync"]
234
267
  project_name =File.basename(Dir.getwd)
235
268
  say "Linking #{project_name}", Thor::Shell::Color::BLUE
236
269
  if File.directory?(Dir.getwd+"/.cnvrg")
237
270
  config = YAML.load_file("#{Dir.getwd}/.cnvrg/config.yml")
238
271
  say "Directory is already linked to #{config[:project_slug]}", Thor::Shell::Color::RED
272
+
239
273
  exit(0)
240
274
  end
241
275
  if Project.link(project_name)
@@ -248,11 +282,16 @@ module Cnvrg
248
282
 
249
283
  url = @project.url
250
284
  say "#{project_name}'s location is: #{url}\n", Thor::Shell::Color::BLUE
285
+ log_end(0)
286
+
251
287
  else
288
+ log_end(1,"can't link project")
252
289
  say "Error linking project, please contact support.", Thor::Shell::Color::RED
253
290
  exit(0)
254
291
  end
255
292
  rescue SignalException
293
+ log_end(-1)
294
+
256
295
  say "/nAborting"
257
296
  exit(1)
258
297
  end
@@ -262,7 +301,8 @@ module Cnvrg
262
301
 
263
302
  def clone(project_url)
264
303
  begin
265
- verify_logged_in()
304
+ verify_logged_in(false)
305
+ log_start(__method__,args,options)
266
306
  url_parts = project_url.split("/")
267
307
  project_index = Cnvrg::Helpers.look_for_in_path(project_url, "projects")
268
308
  slug = url_parts[project_index+1]
@@ -273,6 +313,8 @@ module Cnvrg
273
313
  project_name = response["title"]
274
314
  say "Cloning #{project_name}", Thor::Shell::Color::BLUE
275
315
  if Dir.exists? project_name or File.exists? project_name
316
+ log_end(1,"conflict with dir/file #{project_name}")
317
+
276
318
  say "Error: Conflict with dir/file #{project_name}", Thor::Shell::Color::RED
277
319
  exit(1)
278
320
  end
@@ -302,11 +344,15 @@ module Cnvrg
302
344
  end
303
345
  end
304
346
  say "Done.\nDownloaded total of #{successful_changes.size} files", Thor::Shell::Color::GREEN
347
+ log_end(0)
305
348
  else
349
+ log_end(1,"can't create directory")
350
+
306
351
  say "Error: Couldn't create directory: #{project_name}", Thor::Shell::Color::RED
307
352
  exit(1)
308
353
  end
309
354
  rescue SignalException
355
+ log_end(-1)
310
356
  say "/nAborting"
311
357
  exit(1)
312
358
  end
@@ -317,6 +363,7 @@ module Cnvrg
317
363
  def status
318
364
  begin
319
365
  verify_logged_in()
366
+ log_start(__method__,args,options)
320
367
  @project = Project.new(get_project_home)
321
368
  result = @project.compare_idx["result"]
322
369
  commit = result["commit"]
@@ -324,6 +371,7 @@ module Cnvrg
324
371
  say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE
325
372
  if result["added"].empty? and result["updated_on_local"].empty? and result["updated_on_server"].empty? and result["deleted"].empty? and result["conflicts"].empty?
326
373
  say "Project is up to date", Thor::Shell::Color::GREEN
374
+ log_end(0)
327
375
  return true
328
376
  end
329
377
  if result["added"].size > 0
@@ -359,7 +407,9 @@ module Cnvrg
359
407
  say "\t\tC:\t#{a}", Thor::Shell::Color::RED
360
408
  end
361
409
  end
410
+ log_end(0)
362
411
  rescue SignalException
412
+ log_end(-1)
363
413
  say "/nAborting"
364
414
  exit(1)
365
415
  end
@@ -373,6 +423,8 @@ module Cnvrg
373
423
 
374
424
  begin
375
425
  verify_logged_in()
426
+ log_start(__method__,args,options)
427
+
376
428
  @project = Project.new(get_project_home)
377
429
 
378
430
  @files = Cnvrg::Files.new(@project.owner, @project.slug)
@@ -384,6 +436,8 @@ module Cnvrg
384
436
  commit = result["result"]["commit"]
385
437
  if !link
386
438
  if commit != @project.last_local_commit and !@project.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
439
+ log_end(0)
440
+
387
441
  say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
388
442
  exit(1)
389
443
  end
@@ -391,6 +445,7 @@ module Cnvrg
391
445
  end
392
446
  result = result["result"]["tree"]
393
447
  if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
448
+ log_end(0)
394
449
  say "Project is up to date", Thor::Shell::Color::GREEN
395
450
  return true
396
451
  end
@@ -427,6 +482,7 @@ module Cnvrg
427
482
  successful_updates<< relative_path
428
483
  else
429
484
  @files.rollback_commit(commit_sha1)
485
+ log_end(1,"can't upload, Rolling Back all changes")
430
486
  say "Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED
431
487
  exit(0)
432
488
  end
@@ -448,13 +504,15 @@ module Cnvrg
448
504
  end
449
505
  end
450
506
  end
507
+ log_end(0)
451
508
 
452
509
  rescue SignalException
510
+ log_end(-1)
453
511
  @files.rollback_commit(commit_sha1)
454
512
  say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
455
513
  exit(0)
456
514
  rescue => e
457
- puts e
515
+ log_end(1,e.message)
458
516
  @files.rollback_commit(commit_sha1)
459
517
  say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
460
518
  exit(0)
@@ -467,7 +525,8 @@ module Cnvrg
467
525
  @project.update_idx_with_files_commits!((successful_deletions+successful_updates), res["result"]["commit_time"])
468
526
 
469
527
  @project.update_idx_with_commit!(commit_sha1)
470
- rescue
528
+ rescue => e
529
+ log_end(1,e.message)
471
530
  @files.rollback_commit(commit_sha1)
472
531
  say "Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED
473
532
  exit(1)
@@ -486,14 +545,20 @@ module Cnvrg
486
545
  say del.join("\n"), Thor::Shell::Color::GREEN
487
546
  end
488
547
  say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
548
+ log_end(0)
489
549
  else
490
550
  @files.rollback_commit(commit_sha1)
551
+ log_end(1, "error. Rolling Back all changes")
491
552
  say "Error. Rolling Back all changes.", Thor::Shell::Color::RED
492
553
  end
493
554
  else
555
+ log_end(1, "error. Rolling Back all changes")
556
+
494
557
  @files.rollback_commit(commit_sha1)
495
558
  end
496
559
  rescue SignalException
560
+ log_end(-1)
561
+
497
562
  say "\nAborting",Thor::Shell::Color::BLUE
498
563
  say "\nRolling back all changes",Thor::Shell::Color::BLUE
499
564
  @files.rollback_commit(commit_sha1)
@@ -507,6 +572,7 @@ module Cnvrg
507
572
  def download
508
573
  begin
509
574
  verify_logged_in()
575
+ log_start(__method__,args,options)
510
576
  project_home = get_project_home
511
577
  @project = Project.new(project_home)
512
578
  @files = Cnvrg::Files.new(@project.owner, @project.slug)
@@ -516,6 +582,7 @@ module Cnvrg
516
582
  commit = res["commit"]
517
583
  if result["updated_on_server"].empty? and result["conflicts"] and result["deleted"].empty?
518
584
  say "Project is up to date", Thor::Shell::Color::GREEN
585
+ log_end(0)
519
586
  return true
520
587
  end
521
588
  update_count = 0
@@ -555,11 +622,14 @@ module Cnvrg
555
622
  if update_total == successful_changes.size
556
623
  # update idx with latest commit
557
624
  @project.update_idx_with_commit!(commit)
625
+
558
626
  say "Done. Downloaded:", Thor::Shell::Color::GREEN
559
627
  say successful_changes.join("\n"), Thor::Shell::Color::GREEN
560
628
  say "Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN
629
+ log_end(0)
561
630
  end
562
631
  rescue SignalException
632
+ log_end(-1)
563
633
  say "\nAborting", Thor::Shell::Color::BLUE
564
634
  if successful_changes.nil?
565
635
  exit(1)
@@ -624,11 +694,13 @@ module Cnvrg
624
694
  method_option :upload_output, :type => :string, :aliases => ["--uo", "-uo"], :default => ""
625
695
 
626
696
  def exec(*cmd)
697
+ # LogJob.perform_async(cmd,options)
698
+ #
627
699
  log = []
628
700
  cpu_average =0
629
701
  memory_average = 0
630
702
  verify_logged_in()
631
-
703
+ log_start(__method__,args,options)
632
704
  project_home = get_project_home
633
705
  @project = Project.new(project_home)
634
706
  sync_before = options["sync_before"]
@@ -638,6 +710,7 @@ module Cnvrg
638
710
  email_notification = options["email_notification"]
639
711
  upload_output = options["upload_output"]
640
712
  time_to_upload = calc_output_time(upload_output)
713
+
641
714
  begin
642
715
  if sync_before
643
716
  # Sync before run
@@ -714,23 +787,23 @@ module Cnvrg
714
787
  stderr.each do |err|
715
788
 
716
789
  log << {time: Time.now, message: err, type: "stderr"}
717
- puts err
718
790
  end
719
791
  end
720
792
 
721
793
  rescue Errno::EIO => e
722
794
  break
723
795
  rescue Errno::ENOENT
796
+ log_end(1, "command #{cmd} isn't valid")
724
797
 
725
798
  exp_success = false
726
799
 
727
800
  say "command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED
728
801
  rescue PTY::ChildExited
729
-
802
+ log_end(1, "proccess exited")
730
803
  exp_success = false
731
804
  say "The process exited!", Thor::Shell::Color::RED
732
805
  rescue => e
733
-
806
+ log_end(1,e.message)
734
807
  end
735
808
  ::Process.wait pid
736
809
  cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
@@ -749,6 +822,7 @@ module Cnvrg
749
822
  exit_status = 0
750
823
  else
751
824
  say "Experiment has failed, your'e computer is offline", Thor::Shell::Color::RED
825
+ log_end(1,"experiment has failed,computer is offline")
752
826
  exit(0)
753
827
  end
754
828
  else
@@ -756,6 +830,7 @@ module Cnvrg
756
830
  end_commit = @project.last_local_commit
757
831
  res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
758
832
  say "Experiment has failed, look at the log for more details or run cnvrg exec --log", Thor::Shell::Color::RED
833
+ log_end(1,"experiment has failed")
759
834
  exit(0)
760
835
  end
761
836
 
@@ -772,9 +847,10 @@ module Cnvrg
772
847
  res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
773
848
  check = Helpers.checkmark()
774
849
  say "#{check} Done. Experiment's result: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN
850
+ log_end(0)
775
851
  end
776
- rescue
777
-
852
+ rescue =>e
853
+ log_end(1,e.message)
778
854
  say "Couldn't run #{cmd}, check your input parameters", Thor::Shell::Color::RED
779
855
  exit(1)
780
856
  end
@@ -785,6 +861,7 @@ module Cnvrg
785
861
  end
786
862
  rescue SignalException
787
863
  exit_status = -1
864
+ log_end(-1)
788
865
  end_commit = @project.last_local_commit
789
866
 
790
867
  res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
@@ -800,7 +877,10 @@ module Cnvrg
800
877
  method_option :image_name, :type => :string, :aliases => ["-i", "--i"], :default => ""
801
878
 
802
879
  def run_notebook
880
+
803
881
  begin
882
+ verify_logged_in(false)
883
+ log_start(__method__,args,options)
804
884
  cur_path = Dir.pwd
805
885
  notebook_dir = options["notebook_dir"]
806
886
  if notebook_dir.empty?
@@ -841,6 +921,7 @@ module Cnvrg
841
921
  try_again= false
842
922
  else
843
923
  say "Couldn't start notebook server", Thor::Shell::Color::RED
924
+ log_end(1, "can't start notebook server")
844
925
  exit(1)
845
926
  end
846
927
 
@@ -852,16 +933,19 @@ module Cnvrg
852
933
  did_stop = system("#{docker_path} stop #{port_container}")
853
934
  if !did_stop
854
935
  say "Couldn't stop notebook server: #{port_container}", Thor::Shell::Color::RED
936
+ log_end(1,"can't stop notebook server")
855
937
  exit(1)
856
938
 
857
939
  end
858
940
  else
859
941
  logs = `#{docker_path} logs #{port_container}`
860
942
  url = URI.extract(logs).reject { |x| x if !x.include? "http" }.uniq![0]
861
- say "Okay, you could find the existing server in: #{url}", Thor::Shell::Color::BLUE
943
+ say "Done, your notebook server is: #{url}", Thor::Shell::Color::BLUE
944
+ log_end(0)
862
945
  exit(1)
863
946
  end
864
947
  else
948
+ log_end(1, "can;t start notebook server")
865
949
  say "Couldn't start notebook server", Thor::Shell::Color::RED
866
950
  exit(1)
867
951
 
@@ -875,13 +959,16 @@ module Cnvrg
875
959
  check = Helpers.checkmark()
876
960
 
877
961
  say "#{check} Notebook server started successfully, view notebook in url: #{url}", Thor::Shell::Color::GREEN
962
+ log_end(0)
878
963
  else
879
964
  say "Couldn't start notebook server", Thor::Shell::Color::RED
965
+ log_end(1,"can't start notebook server")
880
966
  exit(1)
881
967
  end
882
968
 
883
969
  end
884
970
  rescue SignalException
971
+ log_end(-1)
885
972
  say "Aborting"
886
973
  exit(1)
887
974
  end
@@ -894,6 +981,8 @@ module Cnvrg
894
981
 
895
982
  def install_notebook_libraries
896
983
  begin
984
+ verify_logged_in(false)
985
+ log_start(__method__,args,options)
897
986
  docker_path = verify_software_installed("docker")
898
987
  container_id = get_container_id
899
988
  say "Opening shell in notebook server\nYou can run pip install [library] to install more tools\ntype exit to finish", Thor::Shell::Color::BLUE
@@ -903,6 +992,7 @@ module Cnvrg
903
992
  return commit_notebook(commit_name)
904
993
  end
905
994
  rescue SignalException
995
+ log_End(-1)
906
996
  say "/nAborting"
907
997
  exit(1)
908
998
  end
@@ -912,6 +1002,8 @@ module Cnvrg
912
1002
  desc 'commit_notebook', 'commit notebook changes to create a new notebook image'
913
1003
 
914
1004
  def commit_notebook(notebook_image_name)
1005
+ verify_logged_in(false)
1006
+ log_start(__method__,args,options)
915
1007
  begin
916
1008
  docker_path = verify_software_installed("docker")
917
1009
 
@@ -922,12 +1014,15 @@ module Cnvrg
922
1014
  commit_res = system("#{docker_path} commit #{container_id} #{notebook_image_name}")
923
1015
  if commit_res
924
1016
  checker = Helpers.checkmark()
1017
+ log_end(0)
925
1018
  say "#{checker} Done.", Thor::Shell::Color::GREEN
926
1019
  else
1020
+ log_End(1,"can't commit new notebook image")
927
1021
  say "Couldn't commit new notebook image ", Thor::Shell::Color::RED
928
1022
 
929
1023
  end
930
1024
  rescue SignalException
1025
+ log_end(-1)
931
1026
  say "/nAborting"
932
1027
  exit(1)
933
1028
  end
@@ -936,6 +1031,8 @@ module Cnvrg
936
1031
  desc 'upload_notebook', 'commit notebook changes to create a new notebook image'
937
1032
 
938
1033
  def upload_image(image_name)
1034
+ verify_logged_in(false)
1035
+ log_start(__method__,args,options)
939
1036
  docker_path = verify_software_installed("docker")
940
1037
  owner = Cnvrg::CLI.get_owner()
941
1038
  # verify image exist
@@ -950,11 +1047,14 @@ module Cnvrg
950
1047
  if !(File.exist? path or File.exist? path+"gz")
951
1048
  saveRes = system("#{docker_path} save #{image_name}>#{path}")
952
1049
  if !saveRes
1050
+ log_End(1, "can't create tar file from image")
953
1051
  say "Couldn't create tar file from image", Thor::Shell::Color::RED
954
1052
  exit(1)
955
1053
  end
956
1054
  gzipRes = system("gzip -f #{path}")
957
1055
  if !gzipRes
1056
+ log_End(1, "can't create tar file from image")
1057
+
958
1058
  say "Couldn't create tar file from image", Thor::Shell::Color::RED
959
1059
  exit(1)
960
1060
  end
@@ -971,14 +1071,20 @@ module Cnvrg
971
1071
  File.delete(path)
972
1072
  checks = Helpers.checkmark()
973
1073
  say "#{checks} Done", Thor::Shell::Color::GREEN
1074
+ log_end(0)
974
1075
  else
975
1076
  say "Couldn't upload image", Thor::Shell::Color::RED
1077
+ log_end(1, "can't create upload imag")
1078
+
976
1079
  end
977
1080
  else
978
1081
  say "Couldn't create image file for: #{image_name}", Thor::Shell::Color::RED
1082
+ log_end(1, "can't create upload imag")
979
1083
  exit(1)
980
1084
  end
981
1085
  rescue SignalException
1086
+ log_end(-1)
1087
+
982
1088
  say "Couldn't upload image file for: #{image_name}", Thor::Shell::Color::RED
983
1089
  exit(1)
984
1090
  end
@@ -990,6 +1096,8 @@ module Cnvrg
990
1096
 
991
1097
  def download_image(image_name)
992
1098
  begin
1099
+ verify_logged_in(false)
1100
+ log_start(__method__,args,options)
993
1101
  owner = Cnvrg::CLI.get_owner()
994
1102
 
995
1103
  notebooks_res = Cnvrg::API.request("users/#{owner}/images/" + "find", 'POST', {image_name: image_name})
@@ -1030,16 +1138,20 @@ module Cnvrg
1030
1138
  if res
1031
1139
  checks = Helpers.checkmark()
1032
1140
  say "#{checks} Done", Thor::Shell::Color::GREEN
1141
+ log_end(0)
1033
1142
  return true
1034
1143
  else
1035
1144
  say "Couldn't download image #{image_name}", Thor::Shell::Color::RED
1145
+ log_end(1,"can't download image")
1036
1146
  return false
1037
1147
  end
1038
1148
  rescue Interrupt
1149
+ log_end(-1)
1039
1150
  say "The user has exited to process, aborting", Thor::Shell::Color::BLUE
1040
1151
  exit(1)
1041
1152
  end
1042
1153
  rescue SignalException
1154
+ log_end(-1)
1043
1155
  say "/nAborting"
1044
1156
  exit(1)
1045
1157
  end
@@ -1049,6 +1161,9 @@ module Cnvrg
1049
1161
  desc 'list_images', 'lists all custom images you can pull'
1050
1162
 
1051
1163
  def list_images
1164
+ begin
1165
+ verify_logged_in(false)
1166
+ log_start(__method__,args,options)
1052
1167
  owner = Cnvrg::CLI.get_owner()
1053
1168
  res = Cnvrg::API.request("users/#{owner}/images/list", 'GET')
1054
1169
  if Cnvrg::CLI.is_response_success(res)
@@ -1062,7 +1177,15 @@ module Cnvrg
1062
1177
  printf "%-20s %-20s %-30s %-20s\n", u["name"], version, update_at, created_by
1063
1178
  end
1064
1179
  end
1180
+ log_end(0)
1065
1181
  return res["result"]["images"]
1182
+ rescue SignalException
1183
+ log_end(-1)
1184
+ say "/nAborting"
1185
+ exit(1)
1186
+ end
1187
+
1188
+
1066
1189
  end
1067
1190
 
1068
1191
 
@@ -1070,19 +1193,24 @@ module Cnvrg
1070
1193
 
1071
1194
  def pull_image(image_name)
1072
1195
  begin
1196
+ verify_logged_in(false)
1197
+ log_start(__method__,args,options)
1073
1198
 
1074
1199
  if download_image(image_name)
1075
1200
  path = Dir.pwd+"/#{owner}_#{image_name}.tar.gz"
1076
1201
  loadRes = system("docker load < #{path}")
1077
1202
  if loadRes.include? "Loaded image"
1078
1203
  say loadRes, Thor::Shell::Color::GREEN
1204
+ log_end(0)
1079
1205
  else
1080
1206
  say loadRes, Thor::Shell::Color::RED
1207
+ log_end(1,loadRes)
1081
1208
  end
1082
1209
 
1083
1210
  end
1084
1211
  rescue SignalException
1085
1212
  say "\nAborting"
1213
+ log_end(-1)
1086
1214
  exit(1)
1087
1215
  end
1088
1216
 
@@ -1157,20 +1285,39 @@ module Cnvrg
1157
1285
  end
1158
1286
 
1159
1287
  end
1288
+ def log_start(command,args="", options={})
1289
+ begin
1290
+ $LOG.info "----"
1291
+ $LOG.info ruby_version: RUBY_VERSION, os: Cnvrg::Helpers.os(), cli_version:Cnvrg::VERSION
1292
+ $LOG.info command:command,args:args,options:options
1293
+ rescue
1294
+ end
1295
+ end
1296
+ def log_end(exit_status=0, error=nil)
1297
+ begin
1298
+ if exit_status==1
1299
+ $LOG.error exit_status:exit_status, error:error
1300
+ else
1301
+ $LOG.info exit_status:exit_status
1302
+ end
1303
+ rescue
1304
+ end
1305
+ end
1160
1306
 
1161
1307
  def self.is_response_success(response, should_exit=true)
1162
1308
  if response.nil?
1163
- if Cnvrg::Helpers.internet_connection?
1309
+ if !Cnvrg::Helpers.internet_connection?
1310
+ # Cnvrg::CLI.log_end(1,"no internet connection")
1164
1311
  say("<%= color('Error:You seems to be offline', RED) %>")
1165
1312
  else
1166
1313
  say("<%= color('Error', RED) %>")
1167
1314
  end
1168
1315
  elsif response["status"]!= 200
1169
1316
  error = response['message']
1317
+ # Cnvrg::CLI.log_end(1, error)
1170
1318
  if response["status"] == 500
1171
1319
  say("<%= color('Server Error', RED) %>")
1172
1320
  else
1173
- puts response
1174
1321
  say("<%= color('Error:#{error}', RED) %>")
1175
1322
  end
1176
1323
 
@@ -1221,8 +1368,39 @@ module Cnvrg
1221
1368
  return false
1222
1369
  end
1223
1370
  end
1371
+ def log_handler
1372
+ begin
1373
+ date = DateTime.now.strftime("%m_%d_%Y")
1374
+ logfile = File.expand_path('~') +"/.cnvrg/log_#{date}.log"
1375
+ if !File.exist? logfile
1376
+ FileUtils.touch([logfile])
1377
+ yesterday = get_start_day-86399
1378
+ date = yesterday.strftime("%m_%d_%Y")
1379
+
1380
+ logfile_old = File.expand_path('~') +"/.cnvrg/log_#{date}.log"
1381
+ count = 0
1382
+ while not File.exist? logfile_old and count <60
1383
+ yesterday = yesterday - 86399
1384
+ date = yesterday.strftime("%m_%d_%Y")
1385
+ logfile_old = File.expand_path('~') +"/.cnvrg/log_#{date}.log"
1386
+ count+=1
1387
+ end
1388
+ @files = Cnvrg::Files.new(Cnvrg::CLI.get_owner, "")
1389
+ @files.upload_log_file(logfile_old,"log_#{date}.log",yesterday)
1390
+ FileUtils.remove logfile_old
1391
+
1392
+
1393
+ end
1394
+ $LOG = LogStashLogger.new(type: :file, path: logfile, sync: true)
1395
+ rescue => e
1396
+ # puts e
1397
+ end
1398
+ end
1399
+
1224
1400
 
1225
- def verify_logged_in
1401
+
1402
+ def verify_logged_in(in_dir=true)
1403
+ log_handler()
1226
1404
  auth = Cnvrg::Auth.new
1227
1405
  unless auth.is_logged_in?
1228
1406
  say 'You\'re not logged in', Thor::Shell::Color::RED
@@ -1231,7 +1409,7 @@ module Cnvrg
1231
1409
  end
1232
1410
 
1233
1411
  if !Helpers.internet_connection?
1234
- wait_offline = agree "Seems like you're offline, wait until your'e back online?",limited_to: ['y', 'n'], default: 'n'
1412
+ wait_offline = agree "Seems like you're offline, wait until your' back online?",limited_to: ['y', 'n'], default: 'n'
1235
1413
  if wait_offline
1236
1414
  say "Waiting until your'e online..", Thor::Shell::Color::BLUE
1237
1415
  while !Cnvrg::Helpers.internet_connection?
@@ -1245,13 +1423,21 @@ module Cnvrg
1245
1423
  config = YAML.load_file(File.expand_path('~')+"/.cnvrg/config.yml")
1246
1424
  version_date = config.to_h[:version_last_check]
1247
1425
  next_day = get_start_day+ 86399
1248
- if (version_date..next_day).cover?(Time.now)
1249
- return true
1250
- else
1426
+ if not (version_date..next_day).cover?(Time.now)
1251
1427
  if should_update_version()
1252
1428
  say "There is a new version, run gem update cnvrg", Thor::Shell::Color::BLUE
1253
1429
  end
1254
1430
  end
1431
+ if in_dir
1432
+ current_dir = Dir.pwd
1433
+ if not Dir.exist? current_dir+"/.cnvrg"
1434
+ say "You're not in a cnvrg project directory",Thor::Shell::Color::RED
1435
+ exit(0)
1436
+
1437
+ end
1438
+ end
1439
+
1440
+
1255
1441
 
1256
1442
  end
1257
1443
 
@@ -2,7 +2,6 @@ require 'mimemagic'
2
2
  require 'aws-sdk'
3
3
  require 'URLcrypt'
4
4
 
5
-
6
5
  module Cnvrg
7
6
  class Files
8
7
 
@@ -24,13 +23,12 @@ module Cnvrg
24
23
  commit_sha1: commit_sha1, file_name: file_name,
25
24
  file_size: file_size, file_content_type: content_type})
26
25
  if Cnvrg::CLI.is_response_success(upload_resp, false)
26
+ path = upload_resp["result"]["path"]
27
27
  if file_size.to_f>= Cnvrg::Files::LARGE_FILE.to_f
28
28
  s3_res = upload_large_files_s3(upload_resp, absolute_path)
29
29
  else
30
- path = upload_resp["result"]["path"]
31
30
  s3_res = upload_small_files_s3(path, absolute_path, content_type)
32
31
  end
33
-
34
32
  if s3_res
35
33
  Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
36
34
  blob_id: upload_resp["result"]["id"]})
@@ -39,6 +37,23 @@ module Cnvrg
39
37
  end
40
38
  return false
41
39
  end
40
+ def upload_log_file(absolute_path, relative_path,log_date)
41
+ file_name = File.basename relative_path
42
+ file_size = File.size(absolute_path).to_f
43
+ content_type = "text/x-log"
44
+ upload_resp = Cnvrg::API.request("/users/#{@owner}/" + "upload_cli_log", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
45
+ file_name: file_name,log_date:log_date,
46
+ file_size: file_size, file_content_type: content_type})
47
+ if Cnvrg::CLI.is_response_success(upload_resp, false)
48
+ path = upload_resp["result"]["path"]
49
+ s3_res = upload_small_files_s3(path, absolute_path, "text/plain")
50
+ end
51
+ if s3_res
52
+ return true
53
+ end
54
+ return false
55
+
56
+ end
42
57
 
43
58
  def upload_image(absolute_path, image_name, owner)
44
59
  file_name = File.basename absolute_path
@@ -100,16 +115,16 @@ module Cnvrg
100
115
  body = response.read_body
101
116
  end
102
117
  URLcrypt::key = [body].pack('H*')
103
-
104
118
  s3 = Aws::S3::Resource.new(
105
119
  :access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
106
120
  :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
107
121
  :session_token => URLcrypt.decrypt(upload_resp["result"]["sts_st"]),
108
122
  :region => URLcrypt.decrypt(upload_resp["result"]["region"]))
109
-
110
- resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).object(File.basename(file_path)).upload_file(file_path)
123
+ resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
124
+ object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
125
+ upload_file(file_path)
111
126
  return resp
112
- rescue => e
127
+ rescue =>e
113
128
  puts e
114
129
  return false
115
130
 
@@ -118,9 +133,9 @@ module Cnvrg
118
133
 
119
134
  end
120
135
 
121
- def upload_small_files_s3(url, file, content_type)
122
- url = URI.parse(url)
123
- file = File.open(file, "rb")
136
+ def upload_small_files_s3(url_path, file_path, content_type)
137
+ url = URI.parse(url_path)
138
+ file = File.open(file_path, "rb")
124
139
  body = file.read
125
140
  begin
126
141
  Net::HTTP.start(url.host) do |http|
@@ -131,8 +146,7 @@ module Cnvrg
131
146
  return true
132
147
  rescue Interrupt
133
148
  return false
134
- rescue => e
135
- puts e
149
+ rescue
136
150
  return false
137
151
  end
138
152
  end
@@ -2,18 +2,20 @@ module Cnvrg
2
2
  module Helpers
3
3
 
4
4
  extend self
5
+
5
6
  def checkmark
6
7
  checkmark = "\u2713"
7
8
  return checkmark.encode('utf-8')
8
- end
9
+ end
9
10
 
10
- def internet_connection?
11
- begin
12
- true if open("http://www.google.com/")
13
- rescue
14
- false
15
- end
11
+ def internet_connection?
12
+ begin
13
+ true if open("http://www.google.com/")
14
+ rescue
15
+ false
16
16
  end
17
+ end
18
+
17
19
  def remote_url
18
20
  home_dir = File.expand_path('~')
19
21
  config = ""
@@ -30,7 +32,24 @@ module Cnvrg
30
32
  if !config or config.empty? or config.to_h[:api].nil?
31
33
  return "https://cnvrg.io"
32
34
  else
33
- return config.to_h[:api].gsub("/api","")
35
+ return config.to_h[:api].gsub("/api", "")
36
+ end
37
+ end
38
+
39
+ def os
40
+
41
+ if windows?
42
+ return "windows"
43
+ elsif mac?
44
+ return "mac"
45
+ elsif ubuntu?
46
+ return "ubuntu"
47
+ elsif linux?
48
+
49
+ return "linux"
50
+ else
51
+
52
+ return "N/A"
34
53
  end
35
54
  end
36
55
 
@@ -45,10 +64,12 @@ module Cnvrg
45
64
  def linux?
46
65
  not mac? and not windows?
47
66
  end
67
+
48
68
  def ubuntu?
49
69
  unix = `cat /etc/lsb-release`.downcase!
50
70
  return unix.include? "ubuntu"
51
71
  end
72
+
52
73
  def cnvrgignore_content
53
74
  %{
54
75
  # cnvrg ignore: Ignore the following directories and files
@@ -0,0 +1,42 @@
1
+ require 'mimemagic'
2
+ require 'aws-sdk'
3
+ require 'URLcrypt'
4
+ require 'pry'
5
+
6
+ require 'sucker_punch'
7
+ module Cnvrg
8
+
9
+ class LogJob
10
+ include SuckerPunch::Job
11
+
12
+ def perform(upload_resp, file_path)
13
+ begin
14
+ sts_path = upload_resp["result"]["path_sts"]
15
+ uri = URI.parse(sts_path)
16
+ http_object = Net::HTTP.new(uri.host, uri.port)
17
+ http_object.use_ssl = true if uri.scheme == 'https'
18
+ request = Net::HTTP::Get.new(sts_path)
19
+ body = ""
20
+ http_object.start do |http|
21
+ response = http.request request
22
+ body = response.read_body
23
+ end
24
+ URLcrypt::key = [body].pack('H*')
25
+ s3 = Aws::S3::Resource.new(
26
+ :access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
27
+ :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
28
+ :session_token => URLcrypt.decrypt(upload_resp["result"]["sts_st"]),
29
+ :region => URLcrypt.decrypt(upload_resp["result"]["region"]))
30
+ resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
31
+ object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
32
+ upload_file(file_path)
33
+ return resp
34
+ rescue =>e
35
+ puts e
36
+ return false
37
+
38
+ end
39
+ return true
40
+ end
41
+ end
42
+ end
@@ -134,8 +134,7 @@ module Cnvrg
134
134
  File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
135
135
  File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore }
136
136
  File.open("README.md", "w+") { |f| f.write cnvrgreadme }
137
- rescue => e
138
- puts e
137
+ rescue
139
138
  return false
140
139
  end
141
140
  return true
@@ -1,4 +1,4 @@
1
1
  module Cnvrg
2
- VERSION = '0.0.8'
2
+ VERSION = '0.0.9'
3
3
  end
4
4
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cnvrg
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.8
4
+ version: 0.0.9
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2017-01-09 00:00:00.000000000 Z
12
+ date: 2017-01-19 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: bundler
@@ -217,6 +217,20 @@ dependencies:
217
217
  - - ">="
218
218
  - !ruby/object:Gem::Version
219
219
  version: '0'
220
+ - !ruby/object:Gem::Dependency
221
+ name: sucker_punch
222
+ requirement: !ruby/object:Gem::Requirement
223
+ requirements:
224
+ - - "~>"
225
+ - !ruby/object:Gem::Version
226
+ version: '2.0'
227
+ type: :runtime
228
+ prerelease: false
229
+ version_requirements: !ruby/object:Gem::Requirement
230
+ requirements:
231
+ - - "~>"
232
+ - !ruby/object:Gem::Version
233
+ version: '2.0'
220
234
  - !ruby/object:Gem::Dependency
221
235
  name: urlcrypt
222
236
  requirement: !ruby/object:Gem::Requirement
@@ -231,6 +245,20 @@ dependencies:
231
245
  - - "~>"
232
246
  - !ruby/object:Gem::Version
233
247
  version: 0.1.1
248
+ - !ruby/object:Gem::Dependency
249
+ name: logstash-logger
250
+ requirement: !ruby/object:Gem::Requirement
251
+ requirements:
252
+ - - ">="
253
+ - !ruby/object:Gem::Version
254
+ version: '0'
255
+ type: :runtime
256
+ prerelease: false
257
+ version_requirements: !ruby/object:Gem::Requirement
258
+ requirements:
259
+ - - ">="
260
+ - !ruby/object:Gem::Version
261
+ version: '0'
234
262
  description: A CLI tool for interacting with cnvrg.io.
235
263
  email:
236
264
  - info@cnvrg.io
@@ -248,6 +276,7 @@ files:
248
276
  - lib/cnvrg/experiment.rb
249
277
  - lib/cnvrg/files.rb
250
278
  - lib/cnvrg/helpers.rb
279
+ - lib/cnvrg/job.rb
251
280
  - lib/cnvrg/project.rb
252
281
  - lib/cnvrg/runner.rb
253
282
  - lib/cnvrg/version.rb