aspera-cli 4.21.2 → 4.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +1 -1
  4. data/CHANGELOG.md +34 -16
  5. data/CONTRIBUTING.md +6 -10
  6. data/README.md +805 -574
  7. data/examples/get_proto_file.rb +1 -1
  8. data/lib/aspera/agent/base.rb +9 -5
  9. data/lib/aspera/agent/connect.rb +30 -28
  10. data/lib/aspera/agent/desktop.rb +29 -25
  11. data/lib/aspera/agent/direct.rb +137 -125
  12. data/lib/aspera/agent/httpgw.rb +22 -26
  13. data/lib/aspera/agent/node.rb +14 -11
  14. data/lib/aspera/agent/transferd.rb +6 -2
  15. data/lib/aspera/api/aoc.rb +6 -6
  16. data/lib/aspera/api/cos_node.rb +1 -1
  17. data/lib/aspera/api/httpgw.rb +7 -3
  18. data/lib/aspera/api/node.rb +6 -4
  19. data/lib/aspera/ascmd.rb +3 -3
  20. data/lib/aspera/ascp/installation.rb +15 -16
  21. data/lib/aspera/ascp/management.rb +1 -1
  22. data/lib/aspera/assert.rb +11 -2
  23. data/lib/aspera/cli/error.rb +2 -2
  24. data/lib/aspera/cli/extended_value.rb +38 -19
  25. data/lib/aspera/cli/formatter.rb +48 -48
  26. data/lib/aspera/cli/hints.rb +1 -1
  27. data/lib/aspera/cli/main.rb +190 -168
  28. data/lib/aspera/cli/manager.rb +15 -15
  29. data/lib/aspera/cli/plugin.rb +23 -20
  30. data/lib/aspera/cli/plugin_factory.rb +1 -1
  31. data/lib/aspera/cli/plugins/alee.rb +1 -1
  32. data/lib/aspera/cli/plugins/aoc.rb +144 -107
  33. data/lib/aspera/cli/plugins/ats.rb +19 -17
  34. data/lib/aspera/cli/plugins/config.rb +67 -83
  35. data/lib/aspera/cli/plugins/console.rb +5 -3
  36. data/lib/aspera/cli/plugins/faspex.rb +39 -35
  37. data/lib/aspera/cli/plugins/faspex5.rb +104 -80
  38. data/lib/aspera/cli/plugins/faspio.rb +13 -1
  39. data/lib/aspera/cli/plugins/httpgw.rb +13 -1
  40. data/lib/aspera/cli/plugins/node.rb +306 -179
  41. data/lib/aspera/cli/plugins/orchestrator.rb +34 -40
  42. data/lib/aspera/cli/plugins/preview.rb +3 -3
  43. data/lib/aspera/cli/plugins/server.rb +6 -6
  44. data/lib/aspera/cli/plugins/shares.rb +5 -5
  45. data/lib/aspera/cli/sync_actions.rb +19 -18
  46. data/lib/aspera/cli/transfer_agent.rb +5 -5
  47. data/lib/aspera/cli/transfer_progress.rb +2 -2
  48. data/lib/aspera/cli/version.rb +1 -1
  49. data/lib/aspera/command_line_builder.rb +116 -95
  50. data/lib/aspera/coverage.rb +4 -3
  51. data/lib/aspera/environment.rb +6 -6
  52. data/lib/aspera/faspex_gw.rb +14 -14
  53. data/lib/aspera/faspex_postproc.rb +7 -6
  54. data/lib/aspera/hash_ext.rb +2 -2
  55. data/lib/aspera/json_rpc.rb +1 -1
  56. data/lib/aspera/keychain/encrypted_hash.rb +47 -34
  57. data/lib/aspera/keychain/factory.rb +41 -0
  58. data/lib/aspera/keychain/hashicorp_vault.rb +71 -0
  59. data/lib/aspera/keychain/macos_security.rb +19 -11
  60. data/lib/aspera/log.rb +28 -34
  61. data/lib/aspera/nagios.rb +6 -6
  62. data/lib/aspera/node_simulator.rb +8 -8
  63. data/lib/aspera/oauth/base.rb +8 -6
  64. data/lib/aspera/oauth/factory.rb +5 -6
  65. data/lib/aspera/oauth/url_json.rb +6 -6
  66. data/lib/aspera/persistency_action_once.rb +6 -4
  67. data/lib/aspera/persistency_folder.rb +2 -2
  68. data/lib/aspera/preview/generator.rb +1 -1
  69. data/lib/aspera/preview/options.rb +16 -16
  70. data/lib/aspera/preview/terminal.rb +3 -3
  71. data/lib/aspera/preview/utils.rb +11 -13
  72. data/lib/aspera/products/connect.rb +1 -1
  73. data/lib/aspera/products/desktop.rb +1 -1
  74. data/lib/aspera/products/transferd.rb +1 -1
  75. data/lib/aspera/proxy_auto_config.rb +2 -2
  76. data/lib/aspera/rest.rb +52 -43
  77. data/lib/aspera/rest_errors_aspera.rb +1 -1
  78. data/lib/aspera/secret_hider.rb +5 -5
  79. data/lib/aspera/ssh.rb +4 -4
  80. data/lib/aspera/transfer/convert.rb +29 -0
  81. data/lib/aspera/transfer/error_info.rb +66 -66
  82. data/lib/aspera/transfer/parameters.rb +13 -68
  83. data/lib/aspera/transfer/spec.rb +5 -6
  84. data/lib/aspera/transfer/spec.schema.yaml +753 -0
  85. data/lib/aspera/transfer/spec_doc.rb +62 -0
  86. data/lib/aspera/transfer/sync.rb +23 -72
  87. data/lib/aspera/transfer/sync_instance.schema.yaml +13 -0
  88. data/lib/aspera/transfer/sync_session.schema.yaml +79 -0
  89. data/lib/aspera/transfer/uri.rb +6 -6
  90. data/lib/aspera/uri_reader.rb +1 -1
  91. data/lib/aspera/web_auth.rb +1 -1
  92. data/lib/aspera/web_server_simple.rb +53 -44
  93. data.tar.gz.sig +1 -2
  94. metadata +37 -4
  95. metadata.gz.sig +0 -0
  96. data/examples/build_package.sh +0 -28
  97. data/lib/aspera/transfer/spec.yaml +0 -718
@@ -11,6 +11,7 @@ require 'terminal-table'
11
11
  require 'tty-spinner'
12
12
  require 'yaml'
13
13
  require 'pp'
14
+ require 'word_wrap'
14
15
 
15
16
  module Aspera
16
17
  module Cli
@@ -58,12 +59,12 @@ module Aspera
58
59
  @result[name] = @formatter.special_format('empty list')
59
60
  elsif array.all?(String)
60
61
  @result[name] = array.join("\n")
61
- elsif array.all?{|i| i.is_a?(Hash) && i.keys.eql?(%w[name])}
62
+ elsif array.all?{ |i| i.is_a?(Hash) && i.keys.eql?(%w[name])}
62
63
  @result[name] = array.map(&:values).join(', ')
63
- elsif array.all?{|i| i.is_a?(Hash) && i.keys.sort.eql?(%w[name value])}
64
- flattened_hash(array.each_with_object({}){|i, h|h[i['name']] = i['value']}, name)
64
+ elsif array.all?{ |i| i.is_a?(Hash) && i.keys.sort.eql?(%w[name value])}
65
+ flattened_hash(array.each_with_object({}){ |i, h| h[i['name']] = i['value']}, name)
65
66
  else
66
- array.each_with_index { |item, index| flatten_any(item, "#{name}.#{index}")}
67
+ array.each_with_index{ |item, index| flatten_any(item, "#{name}.#{index}")}
67
68
  end
68
69
  nil
69
70
  end
@@ -101,7 +102,7 @@ module Aspera
101
102
  class << self
102
103
  def all_but(list)
103
104
  list = [list] unless list.is_a?(Array)
104
- return list.map{|i|"#{FIELDS_LESS}#{i}"}.unshift(SpecialValues::ALL)
105
+ return list.map{ |i| "#{FIELDS_LESS}#{i}"}.unshift(SpecialValues::ALL)
105
106
  end
106
107
 
107
108
  def tick(yes)
@@ -120,17 +121,6 @@ module Aspera
120
121
  return result.green if yes
121
122
  return result.red
122
123
  end
123
-
124
- def auto_type(data)
125
- result = {type: :other_struct, data: data}
126
- result[:type] = :single_object if result[:data].is_a?(Hash)
127
- if result[:data].is_a?(Array)
128
- if result[:data].all?(Hash)
129
- result[:type] = :object_list
130
- end
131
- end
132
- return result
133
- end
134
124
  end
135
125
 
136
126
  # initialize the formatter
@@ -139,10 +129,17 @@ module Aspera
139
129
  @spinner = nil
140
130
  end
141
131
 
142
- # Highlight special values
132
+ # Highlight special values on terminal
143
133
  def special_format(what)
144
134
  result = "<#{what}>"
145
- return %w[null empty].any?{|s|what.include?(s)} ? result.dim : result.reverse_color
135
+ return %w[null empty].any?{ |s| what.include?(s)} ? result.dim : result.reverse_color
136
+ end
137
+
138
+ # for transfer spec table, build line for display
139
+ def check_row(row)
140
+ row.each_key do |k|
141
+ row[k] = row[k].map{ |i| WordWrap.ww(i.to_s, 120).chomp}.join("\n") if row[k].is_a?(Array)
142
+ end
146
143
  end
147
144
 
148
145
  # call this after REST calls if several api calls are expected
@@ -201,7 +198,7 @@ module Aspera
201
198
  end
202
199
  when :image
203
200
  # get list if key arguments of method
204
- allowed_options = Preview::Terminal.method(:build).parameters.select{|i|i[0].eql?(:key)}.map{|i|i[1]}
201
+ allowed_options = Preview::Terminal.method(:build).parameters.select{ |i| i[0].eql?(:key)}.map{ |i| i[1]}
205
202
  # check that only supported options are given
206
203
  unknown_options = value.keys.map(&:to_sym) - allowed_options
207
204
  raise "Invalid parameter(s) for option image: #{unknown_options.join(', ')}, use #{allowed_options.join(', ')}" unless unknown_options.empty?
@@ -216,8 +213,8 @@ module Aspera
216
213
  # data: for requested data, not displayed if level==error
217
214
  # info: additional info, displayed if level==info
218
215
  # error: always displayed on stderr
219
- def display_message(message_level, message)
220
- message = SecretHider.hide_secrets_in_string(message) if message.is_a?(String) && hide_secrets?
216
+ def display_message(message_level, message, hide_secrets: true)
217
+ message = SecretHider.hide_secrets_in_string(message) if hide_secrets && message.is_a?(String) && hide_secrets?
221
218
  case message_level
222
219
  when :data then $stdout.puts(message) unless @options[:display].eql?(:error)
223
220
  when :info then $stdout.puts(message) if @options[:display].eql?(:info)
@@ -226,8 +223,8 @@ module Aspera
226
223
  end
227
224
  end
228
225
 
229
- def display_status(status)
230
- display_message(:info, status)
226
+ def display_status(status, **kwopt)
227
+ display_message(:info, status, **kwopt)
231
228
  end
232
229
 
233
230
  def display_item_count(number, total)
@@ -296,16 +293,15 @@ module Aspera
296
293
  obj_list = data
297
294
  if type.eql?(:single_object)
298
295
  obj_list = [obj_list]
299
- @options[:multi_single] = :yes
300
296
  end
301
297
  Aspera.assert_type(obj_list, Array)
302
298
  Aspera.assert(obj_list.all?(Hash)){"expecting Array of Hash: #{obj_list.inspect}"}
303
299
  # :object_list is an array of hash tables, where key=colum name
304
- obj_list = obj_list.map{|obj|Flattener.new(self).flatten(obj)} if @options[:flat_hash]
305
- display_table(obj_list, compute_fields(obj_list, fields))
300
+ obj_list = obj_list.map{ |obj| Flattener.new(self).flatten(obj)} if @options[:flat_hash]
301
+ display_table(obj_list, compute_fields(obj_list, fields), single: type.eql?(:single_object))
306
302
  when :value_list
307
303
  # :value_list is a simple array of values, name of column provided in the :name
308
- display_table(data.map { |i| { name => i } }, [name])
304
+ display_table(data.map{ |i| {name => i}}, [name])
309
305
  when :empty # no table
310
306
  display_message(:info, special_format('empty'))
311
307
  return
@@ -317,9 +313,6 @@ module Aspera
317
313
  when :text # no table
318
314
  # :status displays a simple message
319
315
  display_message(:data, data)
320
- when :other_struct # no table
321
- # :other_struct is any other type of structure
322
- display_message(:data, PP.pp(data, +''))
323
316
  else
324
317
  raise "unknown data type: #{type}"
325
318
  end
@@ -356,7 +349,7 @@ module Aspera
356
349
  private
357
350
 
358
351
  def all_fields(data)
359
- data.each_with_object({}){|v, m|v.each_key{|c|m[c] = true}}.keys
352
+ data.each_with_object({}){ |v, m| v.each_key{ |c| m[c] = true}}.keys
360
353
  end
361
354
 
362
355
  # @return the list of fields to display
@@ -370,8 +363,8 @@ module Aspera
370
363
  # when NilClass then [SpecialValues::DEF]
371
364
  when String then @options[:fields].split(',')
372
365
  when Array then @options[:fields]
373
- when Regexp then return all_fields(data).select{|i|i.match(@options[:fields])}
374
- when Proc then return all_fields(data).select{|i|@options[:fields].call(i)}
366
+ when Regexp then return all_fields(data).select{ |i| i.match(@options[:fields])}
367
+ when Proc then return all_fields(data).select{ |i| @options[:fields].call(i)}
375
368
  else Aspera.error_unexpected_value(@options[:fields])
376
369
  end
377
370
  result = []
@@ -387,12 +380,12 @@ module Aspera
387
380
  # get the list of all column names used in all lines, not just first one, as all lines may have different columns
388
381
  request.unshift(*all_fields(data))
389
382
  when SpecialValues::DEF
390
- default = all_fields(data).select{|i|default.call(i)} if default.is_a?(Proc)
383
+ default = all_fields(data).select{ |i| default.call(i)} if default.is_a?(Proc)
391
384
  default = all_fields(data) if default.nil?
392
385
  request.unshift(*default)
393
386
  else
394
387
  if removal
395
- result = result.reject{|i|i.eql?(item)}
388
+ result = result.reject{ |i| i.eql?(item)}
396
389
  else
397
390
  result.push(item)
398
391
  end
@@ -410,8 +403,8 @@ module Aspera
410
403
  filter_columns_on_select(data)
411
404
  return data if @options[:fields].eql?(SpecialValues::DEF)
412
405
  selected_fields = compute_fields(data, @options[:fields])
413
- return data.map{|i|i[selected_fields.first]} if selected_fields.length == 1
414
- return data.map{|i|i.slice(*selected_fields)}
406
+ return data.map{ |i| i[selected_fields.first]} if selected_fields.length == 1
407
+ return data.map{ |i| i.slice(*selected_fields)}
415
408
  end
416
409
 
417
410
  # filter the list of items on the select option
@@ -419,16 +412,16 @@ module Aspera
419
412
  def filter_columns_on_select(data)
420
413
  case @options[:select]
421
414
  when Proc
422
- data.select!{|i|@options[:select].call(i)}
415
+ data.select!{ |i| @options[:select].call(i)}
423
416
  when Hash
424
- @options[:select].each{|k, v|data.select!{|i|i[k].eql?(v)}}
417
+ @options[:select].each{ |k, v| data.select!{ |i| i[k].eql?(v)}}
425
418
  end
426
419
  end
427
420
 
428
421
  # displays a list of objects
429
422
  # @param object_array [Array] array of hash
430
423
  # @param fields [Array] list of column names
431
- def display_table(object_array, fields)
424
+ def display_table(object_array, fields, single: false)
432
425
  Aspera.assert(!fields.nil?){'missing fields parameter'}
433
426
  filter_columns_on_select(object_array)
434
427
  if object_array.empty?
@@ -438,35 +431,42 @@ module Aspera
438
431
  end
439
432
  # if table has only one element, and only one field, display the value
440
433
  if object_array.length == 1 && fields.length == 1
441
- display_message(:data, object_array.first[fields.first])
442
- return
434
+ Log.log.debug("display_table: single element, field: #{fields.first}")
435
+ data = object_array.first[fields.first]
436
+ unless data.is_a?(Array) && data.all?(Hash)
437
+ display_message(:data, data)
438
+ return
439
+ end
440
+ object_array = data
441
+ fields = all_fields(object_array)
442
+ single = false
443
443
  end
444
444
  Log.log.debug{Log.dump(:object_array, object_array)}
445
445
  # convert data to string, and keep only display fields
446
- final_table_rows = object_array.map { |r| fields.map { |c| r[c].to_s } }
446
+ final_table_rows = object_array.map{ |r| fields.map{ |c| r[c].to_s}}
447
447
  # remove empty rows
448
- final_table_rows.select!{|i| !(i.is_a?(Hash) && i.empty?)}
448
+ final_table_rows.select!{ |i| !(i.is_a?(Hash) && i.empty?)}
449
449
  # here : fields : list of column names
450
450
  case @options[:format]
451
451
  when :table
452
- if @options[:multi_single].eql?(:yes) ||
452
+ if single || @options[:multi_single].eql?(:yes) ||
453
453
  (@options[:multi_single].eql?(:single) && final_table_rows.length.eql?(1))
454
+ # display multiple objects as multiple transposed tables
454
455
  final_table_rows.each do |row|
455
- Log.log.debug{Log.dump(:row, row)}
456
456
  display_message(:data, Terminal::Table.new(
457
457
  headings: SINGLE_OBJECT_COLUMN_NAMES,
458
458
  rows: fields.zip(row),
459
459
  style: @options[:table_style]&.symbolize_keys))
460
460
  end
461
461
  else
462
- # display the table !
462
+ # display the table ! as single table
463
463
  display_message(:data, Terminal::Table.new(
464
464
  headings: fields,
465
465
  rows: final_table_rows,
466
466
  style: @options[:table_style]&.symbolize_keys))
467
467
  end
468
468
  when :csv
469
- display_message(:data, final_table_rows.map{|t| t.join(CSV_FIELD_SEPARATOR)}.join(CSV_RECORD_SEPARATOR))
469
+ display_message(:data, final_table_rows.map{ |t| t.join(CSV_FIELD_SEPARATOR)}.join(CSV_RECORD_SEPARATOR))
470
470
  else
471
471
  raise "not expected: #{@options[:format]}"
472
472
  end
@@ -89,7 +89,7 @@ module Aspera
89
89
  end
90
90
  remediation = hint[:remediation]
91
91
  remediation = [remediation] unless remediation.is_a?(Array)
92
- remediation.each{|r|formatter.display_message(:error, "#{Formatter::HINT_FLASH} #{r}")}
92
+ remediation.each{ |r| formatter.display_message(:error, "#{Formatter::HINT_FLASH} #{r}")}
93
93
  break
94
94
  end
95
95
  end