aspera-cli 4.21.1 → 4.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +1 -1
  4. data/CHANGELOG.md +52 -22
  5. data/CONTRIBUTING.md +69 -148
  6. data/README.md +929 -668
  7. data/bin/ascli +5 -14
  8. data/bin/asession +1 -3
  9. data/examples/get_proto_file.rb +4 -3
  10. data/examples/proxy.pac +20 -20
  11. data/lib/aspera/agent/base.rb +11 -5
  12. data/lib/aspera/agent/connect.rb +30 -28
  13. data/lib/aspera/agent/{alpha.rb → desktop.rb} +35 -31
  14. data/lib/aspera/agent/direct.rb +141 -121
  15. data/lib/aspera/agent/httpgw.rb +22 -26
  16. data/lib/aspera/agent/node.rb +14 -11
  17. data/lib/aspera/agent/transferd.rb +30 -19
  18. data/lib/aspera/api/alee.rb +1 -1
  19. data/lib/aspera/api/aoc.rb +6 -6
  20. data/lib/aspera/api/cos_node.rb +2 -2
  21. data/lib/aspera/api/httpgw.rb +7 -3
  22. data/lib/aspera/api/node.rb +10 -8
  23. data/lib/aspera/ascmd.rb +3 -3
  24. data/lib/aspera/ascp/installation.rb +53 -72
  25. data/lib/aspera/ascp/management.rb +1 -1
  26. data/lib/aspera/assert.rb +11 -2
  27. data/lib/aspera/cli/error.rb +2 -2
  28. data/lib/aspera/cli/extended_value.rb +46 -21
  29. data/lib/aspera/cli/formatter.rb +55 -48
  30. data/lib/aspera/cli/hints.rb +1 -1
  31. data/lib/aspera/cli/info.rb +1 -0
  32. data/lib/aspera/cli/main.rb +192 -170
  33. data/lib/aspera/cli/manager.rb +18 -18
  34. data/lib/aspera/cli/plugin.rb +23 -20
  35. data/lib/aspera/cli/plugin_factory.rb +1 -1
  36. data/lib/aspera/cli/plugins/alee.rb +1 -1
  37. data/lib/aspera/cli/plugins/aoc.rb +247 -159
  38. data/lib/aspera/cli/plugins/ats.rb +19 -17
  39. data/lib/aspera/cli/plugins/config.rb +76 -113
  40. data/lib/aspera/cli/plugins/console.rb +5 -3
  41. data/lib/aspera/cli/plugins/faspex.rb +39 -35
  42. data/lib/aspera/cli/plugins/faspex5.rb +111 -84
  43. data/lib/aspera/cli/plugins/faspio.rb +13 -1
  44. data/lib/aspera/cli/plugins/httpgw.rb +13 -1
  45. data/lib/aspera/cli/plugins/node.rb +312 -182
  46. data/lib/aspera/cli/plugins/orchestrator.rb +34 -40
  47. data/lib/aspera/cli/plugins/preview.rb +3 -3
  48. data/lib/aspera/cli/plugins/server.rb +6 -6
  49. data/lib/aspera/cli/plugins/shares.rb +5 -5
  50. data/lib/aspera/cli/sync_actions.rb +19 -18
  51. data/lib/aspera/cli/transfer_agent.rb +5 -5
  52. data/lib/aspera/cli/transfer_progress.rb +2 -2
  53. data/lib/aspera/cli/version.rb +1 -1
  54. data/lib/aspera/command_line_builder.rb +116 -95
  55. data/lib/aspera/coverage.rb +8 -5
  56. data/lib/aspera/environment.rb +26 -17
  57. data/lib/aspera/faspex_gw.rb +14 -14
  58. data/lib/aspera/faspex_postproc.rb +10 -11
  59. data/lib/aspera/hash_ext.rb +4 -14
  60. data/lib/aspera/json_rpc.rb +1 -1
  61. data/lib/aspera/keychain/encrypted_hash.rb +47 -34
  62. data/lib/aspera/keychain/factory.rb +41 -0
  63. data/lib/aspera/keychain/hashicorp_vault.rb +71 -0
  64. data/lib/aspera/keychain/macos_security.rb +19 -11
  65. data/lib/aspera/log.rb +28 -34
  66. data/lib/aspera/nagios.rb +6 -6
  67. data/lib/aspera/node_simulator.rb +8 -8
  68. data/lib/aspera/oauth/base.rb +14 -7
  69. data/lib/aspera/oauth/factory.rb +5 -6
  70. data/lib/aspera/oauth/url_json.rb +6 -6
  71. data/lib/aspera/persistency_action_once.rb +6 -4
  72. data/lib/aspera/persistency_folder.rb +2 -2
  73. data/lib/aspera/preview/generator.rb +13 -10
  74. data/lib/aspera/preview/options.rb +16 -16
  75. data/lib/aspera/preview/terminal.rb +4 -4
  76. data/lib/aspera/preview/utils.rb +15 -17
  77. data/lib/aspera/products/connect.rb +35 -1
  78. data/lib/aspera/products/{alpha.rb → desktop.rb} +3 -3
  79. data/lib/aspera/products/transferd.rb +9 -2
  80. data/lib/aspera/proxy_auto_config.rb +2 -2
  81. data/lib/aspera/rest.rb +56 -47
  82. data/lib/aspera/rest_errors_aspera.rb +1 -1
  83. data/lib/aspera/secret_hider.rb +12 -5
  84. data/lib/aspera/ssh.rb +4 -4
  85. data/lib/aspera/temp_file_manager.rb +5 -4
  86. data/lib/aspera/transfer/convert.rb +29 -0
  87. data/lib/aspera/transfer/error_info.rb +66 -66
  88. data/lib/aspera/transfer/parameters.rb +13 -68
  89. data/lib/aspera/transfer/spec.rb +5 -6
  90. data/lib/aspera/transfer/spec.schema.yaml +753 -0
  91. data/lib/aspera/transfer/spec_doc.rb +62 -0
  92. data/lib/aspera/transfer/sync.rb +23 -72
  93. data/lib/aspera/transfer/sync_instance.schema.yaml +13 -0
  94. data/lib/aspera/transfer/sync_session.schema.yaml +79 -0
  95. data/lib/aspera/transfer/uri.rb +6 -6
  96. data/lib/aspera/uri_reader.rb +18 -1
  97. data/lib/aspera/web_auth.rb +1 -1
  98. data/lib/aspera/web_server_simple.rb +53 -44
  99. data.tar.gz.sig +0 -0
  100. metadata +28 -165
  101. metadata.gz.sig +0 -0
  102. data/examples/build_exec +0 -74
  103. data/examples/build_exec_rubyc +0 -40
  104. data/examples/build_package.sh +0 -28
  105. data/lib/aspera/transfer/spec.yaml +0 -718
@@ -11,6 +11,7 @@ require 'terminal-table'
11
11
  require 'tty-spinner'
12
12
  require 'yaml'
13
13
  require 'pp'
14
+ require 'word_wrap'
14
15
 
15
16
  module Aspera
16
17
  module Cli
@@ -58,12 +59,12 @@ module Aspera
58
59
  @result[name] = @formatter.special_format('empty list')
59
60
  elsif array.all?(String)
60
61
  @result[name] = array.join("\n")
61
- elsif array.all?{|i| i.is_a?(Hash) && i.keys.eql?(%w[name])}
62
+ elsif array.all?{ |i| i.is_a?(Hash) && i.keys.eql?(%w[name])}
62
63
  @result[name] = array.map(&:values).join(', ')
63
- elsif array.all?{|i| i.is_a?(Hash) && i.keys.sort.eql?(%w[name value])}
64
- flattened_hash(array.each_with_object({}){|i, h|h[i['name']] = i['value']}, name)
64
+ elsif array.all?{ |i| i.is_a?(Hash) && i.keys.sort.eql?(%w[name value])}
65
+ flattened_hash(array.each_with_object({}){ |i, h| h[i['name']] = i['value']}, name)
65
66
  else
66
- array.each_with_index { |item, index| flatten_any(item, "#{name}.#{index}")}
67
+ array.each_with_index{ |item, index| flatten_any(item, "#{name}.#{index}")}
67
68
  end
68
69
  nil
69
70
  end
@@ -101,7 +102,7 @@ module Aspera
101
102
  class << self
102
103
  def all_but(list)
103
104
  list = [list] unless list.is_a?(Array)
104
- return list.map{|i|"#{FIELDS_LESS}#{i}"}.unshift(SpecialValues::ALL)
105
+ return list.map{ |i| "#{FIELDS_LESS}#{i}"}.unshift(SpecialValues::ALL)
105
106
  end
106
107
 
107
108
  def tick(yes)
@@ -120,17 +121,6 @@ module Aspera
120
121
  return result.green if yes
121
122
  return result.red
122
123
  end
123
-
124
- def auto_type(data)
125
- result = {type: :other_struct, data: data}
126
- result[:type] = :single_object if result[:data].is_a?(Hash)
127
- if result[:data].is_a?(Array)
128
- if result[:data].all?(Hash)
129
- result[:type] = :object_list
130
- end
131
- end
132
- return result
133
- end
134
124
  end
135
125
 
136
126
  # initialize the formatter
@@ -139,10 +129,17 @@ module Aspera
139
129
  @spinner = nil
140
130
  end
141
131
 
142
- # Highlight special values
132
+ # Highlight special values on terminal
143
133
  def special_format(what)
144
134
  result = "<#{what}>"
145
- return %w[null empty].any?{|s|what.include?(s)} ? result.dim : result.reverse_color
135
+ return %w[null empty].any?{ |s| what.include?(s)} ? result.dim : result.reverse_color
136
+ end
137
+
138
+ # for transfer spec table, build line for display
139
+ def check_row(row)
140
+ row.each_key do |k|
141
+ row[k] = row[k].map{ |i| WordWrap.ww(i.to_s, 120).chomp}.join("\n") if row[k].is_a?(Array)
142
+ end
146
143
  end
147
144
 
148
145
  # call this after REST calls if several api calls are expected
@@ -201,7 +198,7 @@ module Aspera
201
198
  end
202
199
  when :image
203
200
  # get list if key arguments of method
204
- allowed_options = Preview::Terminal.method(:build).parameters.select{|i|i[0].eql?(:key)}.map{|i|i[1]}
201
+ allowed_options = Preview::Terminal.method(:build).parameters.select{ |i| i[0].eql?(:key)}.map{ |i| i[1]}
205
202
  # check that only supported options are given
206
203
  unknown_options = value.keys.map(&:to_sym) - allowed_options
207
204
  raise "Invalid parameter(s) for option image: #{unknown_options.join(', ')}, use #{allowed_options.join(', ')}" unless unknown_options.empty?
@@ -216,7 +213,8 @@ module Aspera
216
213
  # data: for requested data, not displayed if level==error
217
214
  # info: additional info, displayed if level==info
218
215
  # error: always displayed on stderr
219
- def display_message(message_level, message)
216
+ def display_message(message_level, message, hide_secrets: true)
217
+ message = SecretHider.hide_secrets_in_string(message) if hide_secrets && message.is_a?(String) && hide_secrets?
220
218
  case message_level
221
219
  when :data then $stdout.puts(message) unless @options[:display].eql?(:error)
222
220
  when :info then $stdout.puts(message) if @options[:display].eql?(:info)
@@ -225,8 +223,8 @@ module Aspera
225
223
  end
226
224
  end
227
225
 
228
- def display_status(status)
229
- display_message(:info, status)
226
+ def display_status(status, **kwopt)
227
+ display_message(:info, status, **kwopt)
230
228
  end
231
229
 
232
230
  def display_item_count(number, total)
@@ -238,8 +236,13 @@ module Aspera
238
236
  display_status(count_msg)
239
237
  end
240
238
 
239
+ def hide_secrets?
240
+ !@options[:show_secrets] && !@options[:display].eql?(:data)
241
+ end
242
+
243
+ # hides secrets in Hash or Array
241
244
  def hide_secrets(data)
242
- SecretHider.deep_remove_secret(data) unless @options[:show_secrets] || @options[:display].eql?(:data)
245
+ SecretHider.deep_remove_secret(data) if hide_secrets?
243
246
  end
244
247
 
245
248
  # this method displays the results, especially the table format
@@ -255,6 +258,7 @@ module Aspera
255
258
  Aspera.assert(!data.nil? || %i[empty nothing].include?(type)){'result must have data'}
256
259
  display_item_count(data.length, total) unless total.nil?
257
260
  hide_secrets(data)
261
+ data = SecretHider.hide_secrets_in_string(data) if data.is_a?(String) && hide_secrets?
258
262
  case @options[:format]
259
263
  when :text
260
264
  display_message(:data, data.to_s)
@@ -289,16 +293,15 @@ module Aspera
289
293
  obj_list = data
290
294
  if type.eql?(:single_object)
291
295
  obj_list = [obj_list]
292
- @options[:multi_single] = :yes
293
296
  end
294
297
  Aspera.assert_type(obj_list, Array)
295
298
  Aspera.assert(obj_list.all?(Hash)){"expecting Array of Hash: #{obj_list.inspect}"}
296
299
  # :object_list is an array of hash tables, where key=colum name
297
- obj_list = obj_list.map{|obj|Flattener.new(self).flatten(obj)} if @options[:flat_hash]
298
- display_table(obj_list, compute_fields(obj_list, fields))
300
+ obj_list = obj_list.map{ |obj| Flattener.new(self).flatten(obj)} if @options[:flat_hash]
301
+ display_table(obj_list, compute_fields(obj_list, fields), single: type.eql?(:single_object))
299
302
  when :value_list
300
303
  # :value_list is a simple array of values, name of column provided in the :name
301
- display_table(data.map { |i| { name => i } }, [name])
304
+ display_table(data.map{ |i| {name => i}}, [name])
302
305
  when :empty # no table
303
306
  display_message(:info, special_format('empty'))
304
307
  return
@@ -310,9 +313,6 @@ module Aspera
310
313
  when :text # no table
311
314
  # :status displays a simple message
312
315
  display_message(:data, data)
313
- when :other_struct # no table
314
- # :other_struct is any other type of structure
315
- display_message(:data, PP.pp(data, +''))
316
316
  else
317
317
  raise "unknown data type: #{type}"
318
318
  end
@@ -349,7 +349,7 @@ module Aspera
349
349
  private
350
350
 
351
351
  def all_fields(data)
352
- data.each_with_object({}){|v, m|v.each_key{|c|m[c] = true}}.keys
352
+ data.each_with_object({}){ |v, m| v.each_key{ |c| m[c] = true}}.keys
353
353
  end
354
354
 
355
355
  # @return the list of fields to display
@@ -363,8 +363,8 @@ module Aspera
363
363
  # when NilClass then [SpecialValues::DEF]
364
364
  when String then @options[:fields].split(',')
365
365
  when Array then @options[:fields]
366
- when Regexp then return all_fields(data).select{|i|i.match(@options[:fields])}
367
- when Proc then return all_fields(data).select{|i|@options[:fields].call(i)}
366
+ when Regexp then return all_fields(data).select{ |i| i.match(@options[:fields])}
367
+ when Proc then return all_fields(data).select{ |i| @options[:fields].call(i)}
368
368
  else Aspera.error_unexpected_value(@options[:fields])
369
369
  end
370
370
  result = []
@@ -380,12 +380,12 @@ module Aspera
380
380
  # get the list of all column names used in all lines, not just first one, as all lines may have different columns
381
381
  request.unshift(*all_fields(data))
382
382
  when SpecialValues::DEF
383
- default = all_fields(data).select{|i|default.call(i)} if default.is_a?(Proc)
383
+ default = all_fields(data).select{ |i| default.call(i)} if default.is_a?(Proc)
384
384
  default = all_fields(data) if default.nil?
385
385
  request.unshift(*default)
386
386
  else
387
387
  if removal
388
- result = result.reject{|i|i.eql?(item)}
388
+ result = result.reject{ |i| i.eql?(item)}
389
389
  else
390
390
  result.push(item)
391
391
  end
@@ -403,8 +403,8 @@ module Aspera
403
403
  filter_columns_on_select(data)
404
404
  return data if @options[:fields].eql?(SpecialValues::DEF)
405
405
  selected_fields = compute_fields(data, @options[:fields])
406
- return data.map{|i|i[selected_fields.first]} if selected_fields.length == 1
407
- return data.map{|i|i.slice(*selected_fields)}
406
+ return data.map{ |i| i[selected_fields.first]} if selected_fields.length == 1
407
+ return data.map{ |i| i.slice(*selected_fields)}
408
408
  end
409
409
 
410
410
  # filter the list of items on the select option
@@ -412,16 +412,16 @@ module Aspera
412
412
  def filter_columns_on_select(data)
413
413
  case @options[:select]
414
414
  when Proc
415
- data.select!{|i|@options[:select].call(i)}
415
+ data.select!{ |i| @options[:select].call(i)}
416
416
  when Hash
417
- @options[:select].each{|k, v|data.select!{|i|i[k].eql?(v)}}
417
+ @options[:select].each{ |k, v| data.select!{ |i| i[k].eql?(v)}}
418
418
  end
419
419
  end
420
420
 
421
421
  # displays a list of objects
422
422
  # @param object_array [Array] array of hash
423
423
  # @param fields [Array] list of column names
424
- def display_table(object_array, fields)
424
+ def display_table(object_array, fields, single: false)
425
425
  Aspera.assert(!fields.nil?){'missing fields parameter'}
426
426
  filter_columns_on_select(object_array)
427
427
  if object_array.empty?
@@ -431,35 +431,42 @@ module Aspera
431
431
  end
432
432
  # if table has only one element, and only one field, display the value
433
433
  if object_array.length == 1 && fields.length == 1
434
- display_message(:data, object_array.first[fields.first])
435
- return
434
+ Log.log.debug("display_table: single element, field: #{fields.first}")
435
+ data = object_array.first[fields.first]
436
+ unless data.is_a?(Array) && data.all?(Hash)
437
+ display_message(:data, data)
438
+ return
439
+ end
440
+ object_array = data
441
+ fields = all_fields(object_array)
442
+ single = false
436
443
  end
437
444
  Log.log.debug{Log.dump(:object_array, object_array)}
438
445
  # convert data to string, and keep only display fields
439
- final_table_rows = object_array.map { |r| fields.map { |c| r[c].to_s } }
446
+ final_table_rows = object_array.map{ |r| fields.map{ |c| r[c].to_s}}
440
447
  # remove empty rows
441
- final_table_rows.select!{|i| !(i.is_a?(Hash) && i.empty?)}
448
+ final_table_rows.select!{ |i| !(i.is_a?(Hash) && i.empty?)}
442
449
  # here : fields : list of column names
443
450
  case @options[:format]
444
451
  when :table
445
- if @options[:multi_single].eql?(:yes) ||
452
+ if single || @options[:multi_single].eql?(:yes) ||
446
453
  (@options[:multi_single].eql?(:single) && final_table_rows.length.eql?(1))
454
+ # display multiple objects as multiple transposed tables
447
455
  final_table_rows.each do |row|
448
- Log.log.debug{Log.dump(:row, row)}
449
456
  display_message(:data, Terminal::Table.new(
450
457
  headings: SINGLE_OBJECT_COLUMN_NAMES,
451
458
  rows: fields.zip(row),
452
459
  style: @options[:table_style]&.symbolize_keys))
453
460
  end
454
461
  else
455
- # display the table !
462
+ # display the table ! as single table
456
463
  display_message(:data, Terminal::Table.new(
457
464
  headings: fields,
458
465
  rows: final_table_rows,
459
466
  style: @options[:table_style]&.symbolize_keys))
460
467
  end
461
468
  when :csv
462
- display_message(:data, final_table_rows.map{|t| t.join(CSV_FIELD_SEPARATOR)}.join(CSV_RECORD_SEPARATOR))
469
+ display_message(:data, final_table_rows.map{ |t| t.join(CSV_FIELD_SEPARATOR)}.join(CSV_RECORD_SEPARATOR))
463
470
  else
464
471
  raise "not expected: #{@options[:format]}"
465
472
  end
@@ -89,7 +89,7 @@ module Aspera
89
89
  end
90
90
  remediation = hint[:remediation]
91
91
  remediation = [remediation] unless remediation.is_a?(Array)
92
- remediation.each{|r|formatter.display_message(:error, "#{Formatter::HINT_FLASH} #{r}")}
92
+ remediation.each{ |r| formatter.display_message(:error, "#{Formatter::HINT_FLASH} #{r}")}
93
93
  break
94
94
  end
95
95
  end
@@ -10,6 +10,7 @@ module Aspera
10
10
  DOC_URL = "https://www.rubydoc.info/gems/#{GEM_NAME}"
11
11
  GEM_URL = "https://rubygems.org/gems/#{GEM_NAME}"
12
12
  SRC_URL = 'https://github.com/IBM/aspera-cli'
13
+ CONTAINER = 'docker.io/martinlaurent/ascli'
13
14
  # set this to warn in advance when minimum required ruby version will increase
14
15
  # see also required_ruby_version in gemspec file
15
16
  RUBY_FUTURE_MINIMUM_VERSION = '3.2'