aspera-cli 4.14.0 → 4.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +29 -3
  4. data/CHANGELOG.md +300 -185
  5. data/CONTRIBUTING.md +74 -23
  6. data/README.md +2346 -1619
  7. data/bin/ascli +16 -25
  8. data/bin/asession +15 -15
  9. data/examples/dascli +2 -2
  10. data/examples/proxy.pac +1 -1
  11. data/lib/aspera/aoc.rb +216 -150
  12. data/lib/aspera/ascmd.rb +25 -18
  13. data/lib/aspera/assert.rb +45 -0
  14. data/lib/aspera/cli/basic_auth_plugin.rb +9 -6
  15. data/lib/aspera/cli/error.rb +17 -0
  16. data/lib/aspera/cli/extended_value.rb +51 -16
  17. data/lib/aspera/cli/formatter.rb +276 -174
  18. data/lib/aspera/cli/hints.rb +81 -0
  19. data/lib/aspera/cli/main.rb +114 -147
  20. data/lib/aspera/cli/manager.rb +181 -136
  21. data/lib/aspera/cli/plugin.rb +82 -64
  22. data/lib/aspera/cli/plugins/alee.rb +0 -1
  23. data/lib/aspera/cli/plugins/aoc.rb +327 -331
  24. data/lib/aspera/cli/plugins/ats.rb +12 -8
  25. data/lib/aspera/cli/plugins/bss.rb +2 -2
  26. data/lib/aspera/cli/plugins/config.rb +575 -439
  27. data/lib/aspera/cli/plugins/console.rb +40 -0
  28. data/lib/aspera/cli/plugins/cos.rb +4 -5
  29. data/lib/aspera/cli/plugins/faspex.rb +111 -92
  30. data/lib/aspera/cli/plugins/faspex5.rb +245 -182
  31. data/lib/aspera/cli/plugins/node.rb +239 -160
  32. data/lib/aspera/cli/plugins/orchestrator.rb +56 -19
  33. data/lib/aspera/cli/plugins/preview.rb +54 -38
  34. data/lib/aspera/cli/plugins/server.rb +63 -20
  35. data/lib/aspera/cli/plugins/shares.rb +64 -38
  36. data/lib/aspera/cli/sync_actions.rb +68 -0
  37. data/lib/aspera/cli/transfer_agent.rb +64 -67
  38. data/lib/aspera/cli/transfer_progress.rb +73 -0
  39. data/lib/aspera/cli/version.rb +1 -1
  40. data/lib/aspera/colors.rb +3 -1
  41. data/lib/aspera/command_line_builder.rb +27 -22
  42. data/lib/aspera/cos_node.rb +6 -4
  43. data/lib/aspera/coverage.rb +22 -0
  44. data/lib/aspera/data_repository.rb +33 -2
  45. data/lib/aspera/environment.rb +21 -8
  46. data/lib/aspera/fasp/agent_alpha.rb +116 -0
  47. data/lib/aspera/fasp/agent_base.rb +40 -76
  48. data/lib/aspera/fasp/agent_connect.rb +21 -22
  49. data/lib/aspera/fasp/agent_direct.rb +169 -179
  50. data/lib/aspera/fasp/agent_httpgw.rb +200 -195
  51. data/lib/aspera/fasp/agent_node.rb +43 -35
  52. data/lib/aspera/fasp/agent_trsdk.rb +124 -41
  53. data/lib/aspera/fasp/error_info.rb +2 -2
  54. data/lib/aspera/fasp/faux_file.rb +52 -0
  55. data/lib/aspera/fasp/installation.rb +89 -191
  56. data/lib/aspera/fasp/management.rb +249 -0
  57. data/lib/aspera/fasp/parameters.rb +86 -47
  58. data/lib/aspera/fasp/parameters.yaml +75 -8
  59. data/lib/aspera/fasp/products.rb +162 -0
  60. data/lib/aspera/fasp/resume_policy.rb +7 -5
  61. data/lib/aspera/fasp/sync.rb +273 -0
  62. data/lib/aspera/fasp/transfer_spec.rb +10 -8
  63. data/lib/aspera/fasp/uri.rb +6 -6
  64. data/lib/aspera/faspex_gw.rb +11 -8
  65. data/lib/aspera/faspex_postproc.rb +8 -7
  66. data/lib/aspera/hash_ext.rb +2 -2
  67. data/lib/aspera/id_generator.rb +3 -1
  68. data/lib/aspera/json_rpc.rb +51 -0
  69. data/lib/aspera/keychain/encrypted_hash.rb +46 -11
  70. data/lib/aspera/keychain/macos_security.rb +15 -13
  71. data/lib/aspera/line_logger.rb +23 -0
  72. data/lib/aspera/log.rb +61 -19
  73. data/lib/aspera/nagios.rb +7 -2
  74. data/lib/aspera/node.rb +105 -21
  75. data/lib/aspera/node_simulator.rb +214 -0
  76. data/lib/aspera/oauth.rb +57 -36
  77. data/lib/aspera/open_application.rb +4 -4
  78. data/lib/aspera/persistency_action_once.rb +13 -14
  79. data/lib/aspera/persistency_folder.rb +5 -4
  80. data/lib/aspera/preview/file_types.rb +56 -268
  81. data/lib/aspera/preview/generator.rb +28 -39
  82. data/lib/aspera/preview/options.rb +2 -0
  83. data/lib/aspera/preview/terminal.rb +36 -16
  84. data/lib/aspera/preview/utils.rb +23 -29
  85. data/lib/aspera/proxy_auto_config.rb +6 -3
  86. data/lib/aspera/rest.rb +127 -80
  87. data/lib/aspera/rest_call_error.rb +1 -1
  88. data/lib/aspera/rest_error_analyzer.rb +16 -14
  89. data/lib/aspera/rest_errors_aspera.rb +39 -34
  90. data/lib/aspera/secret_hider.rb +18 -17
  91. data/lib/aspera/ssh.rb +10 -5
  92. data/lib/aspera/temp_file_manager.rb +11 -4
  93. data/lib/aspera/web_auth.rb +10 -7
  94. data/lib/aspera/web_server_simple.rb +11 -5
  95. data.tar.gz.sig +0 -0
  96. metadata +108 -39
  97. metadata.gz.sig +0 -0
  98. data/lib/aspera/cli/listener/line_dump.rb +0 -19
  99. data/lib/aspera/cli/listener/logger.rb +0 -22
  100. data/lib/aspera/cli/listener/progress.rb +0 -50
  101. data/lib/aspera/cli/listener/progress_multi.rb +0 -84
  102. data/lib/aspera/cli/plugins/sync.rb +0 -44
  103. data/lib/aspera/fasp/listener.rb +0 -13
  104. data/lib/aspera/sync.rb +0 -213
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'aspera/cli/extended_value'
4
+ require 'aspera/assert'
4
5
 
5
6
  module Aspera
6
7
  module Cli
@@ -16,67 +17,77 @@ module Aspera
16
17
  MAX_ITEMS = 'max'
17
18
  # special query parameter: max number of pages for list command
18
19
  MAX_PAGES = 'pmax'
19
- # used when all resources are selected
20
- VAL_ALL = 'ALL'
21
20
  # special identifier format: look for this name to find where supported
22
21
  REGEX_LOOKUP_ID_BY_FIELD = /^%([^:]+):(.*)$/.freeze
23
22
 
24
- # global for inherited classes
25
- @@options_created = false # rubocop:disable Style/ClassVars
23
+ class << self
24
+ def declare_generic_options(options)
25
+ options.declare(:query, 'Additional filter for for some commands (list/delete)', types: Hash)
26
+ options.declare(
27
+ :value, 'Value for create, update, list filter', types: Hash,
28
+ deprecation: '(4.14) Use positional value for create/modify or option: query for list/delete')
29
+ options.declare(:property, 'Name of property to set (modify operation)')
30
+ options.declare(:id, 'Resource identifier', deprecation: "(4.14) Use positional identifier after verb (#{INSTANCE_OPS.join(',')})")
31
+ options.declare(:bulk, 'Bulk operation (only some)', values: :bool, default: :no)
32
+ options.declare(:bfail, 'Bulk operation error handling', values: :bool, default: :yes)
33
+ end
34
+ end
26
35
 
27
36
  def initialize(env)
28
- raise 'must be Hash' unless env.is_a?(Hash)
29
- # env.each_key {|k| raise "wrong agent key #{k}" unless AGENTS.include?(k)}
37
+ assert_type(env, Hash)
30
38
  @agents = env
31
39
  # check presence in descendant of mandatory method and constant
32
- raise StandardError, "Missing method 'execute_action' in #{self.class}" unless respond_to?(:execute_action)
33
- raise StandardError, 'ACTIONS shall be redefined by subclass' unless self.class.constants.include?(:ACTIONS)
40
+ assert(respond_to?(:execute_action)){"Missing method 'execute_action' in #{self.class}"}
41
+ assert(self.class.constants.include?(:ACTIONS)){'ACTIONS shall be redefined by subclass'}
42
+ # manual header for all plugins
34
43
  options.parser.separator('')
35
44
  options.parser.separator("COMMAND: #{self.class.name.split('::').last.downcase}")
36
45
  options.parser.separator("SUBCOMMANDS: #{self.class.const_get(:ACTIONS).map(&:to_s).sort.join(' ')}")
37
46
  options.parser.separator('OPTIONS:')
38
- return if @@options_created
39
- options.declare(:query, 'Additional filter for for some commands (list/delete)', types: Hash)
40
- options.declare(
41
- :value, 'Value for create, update, list filter', types: Hash,
42
- deprecation: 'Use positional value for create/modify or option: query for list/delete')
43
- options.declare(:property, 'Name of property to set (modify operation)')
44
- options.declare(:id, 'Resource identifier', deprecation: "Use identifier after verb (#{INSTANCE_OPS.join(',')})")
45
- options.declare(:bulk, 'Bulk operation (only some)', values: :bool, default: :no)
46
- options.declare(:bfail, 'Bulk operation error handling', values: :bool, default: :yes)
47
- options.parse_options!
48
- @@options_created = true # rubocop:disable Style/ClassVars
49
47
  end
50
48
 
51
49
  # must be called AFTER the instance action, ... folder browse <call instance_identifier>
52
50
  # @param description [String] description of the identifier
51
+ # @param as_option [Symbol] option name to use if identifier is an option
53
52
  # @param block [Proc] block to search for identifier based on attribute value
54
53
  # @return [String] identifier
55
- def instance_identifier(description: 'identifier', &block)
56
- res_id = options.get_option(:id)
57
- res_id = options.get_next_argument(description) if res_id.nil?
54
+ def instance_identifier(description: 'identifier', as_option: nil, &block)
55
+ if as_option.nil?
56
+ res_id = options.get_option(:id)
57
+ res_id = options.get_next_argument(description) if res_id.nil?
58
+ else
59
+ res_id = options.get_option(as_option)
60
+ end
58
61
  # cab be an Array
59
62
  if res_id.is_a?(String) && (m = res_id.match(REGEX_LOOKUP_ID_BY_FIELD))
60
63
  if block
61
64
  res_id = yield(m[1], ExtendedValue.instance.evaluate(m[2]))
62
65
  else
63
- raise CliBadArgument, "Percent syntax for #{description} not supported in this context"
66
+ raise Cli::BadArgument, "Percent syntax for #{description} not supported in this context"
64
67
  end
65
68
  end
66
69
  return res_id
67
70
  end
68
71
 
69
72
  # For create and delete operations: execute one actin or multiple if bulk is yes
70
- # @param single_or_array [Object] single hash, or array of hash for bulk
71
- # @param success_msg deleted or created
73
+ # @param command [Symbol] operation: :create, :delete, ...
74
+ # @param descr [String] description of the value
75
+ # @param values [Object] the value(s), or the type of value to get from user
72
76
  # @param id_result [String] key in result hash to use as identifier
73
77
  # @param fields [Array] fields to display
74
- def do_bulk_operation(single_or_array, success_msg, id_result: 'id', fields: :default)
75
- raise 'programming error: missing block' unless block_given?
76
- params = options.get_option(:bulk) ? single_or_array : [single_or_array]
77
- raise 'expecting Array for bulk operation' unless params.is_a?(Array)
78
+ def do_bulk_operation(command:, descr:, values: Hash, id_result: 'id', fields: :default)
79
+ assert(block_given?){'missing block'}
80
+ is_bulk = options.get_option(:bulk)
81
+ case values
82
+ when :identifier
83
+ values = instance_identifier
84
+ when Class
85
+ values = value_create_modify(command: command, type: values, bulk: is_bulk)
86
+ end
87
+ # if not bulk, there is a single value
88
+ params = is_bulk ? values : [values]
78
89
  Log.log.warn('Empty list given for bulk operation') if params.empty?
79
- Log.dump(:bulk_operation, params)
90
+ Log.log.debug{Log.dump(:bulk_operation, params)}
80
91
  result_list = []
81
92
  params.each do |param|
82
93
  # init for delete
@@ -86,7 +97,8 @@ module Aspera
86
97
  res = yield(param)
87
98
  # if block returns a hash, let's use this (create)
88
99
  result = res if param.is_a?(Hash)
89
- result['status'] = success_msg
100
+ # create -> created
101
+ result['status'] = "#{command}#{'e' unless command.to_s.end_with?('e')}d".gsub(/yed$/, 'ied')
90
102
  rescue StandardError => e
91
103
  raise e if options.get_option(:bfail)
92
104
  result['status'] = e.to_s
@@ -94,7 +106,7 @@ module Aspera
94
106
  result_list.push(result)
95
107
  end
96
108
  display_fields = [id_result, 'status']
97
- if options.get_option(:bulk)
109
+ if is_bulk
98
110
  return {type: :object_list, data: result_list, fields: display_fields}
99
111
  else
100
112
  display_fields = fields unless fields.eql?(:default)
@@ -106,21 +118,16 @@ module Aspera
106
118
  # @param rest_api [Rest] api to use
107
119
  # @param res_class_path [String] sub path in URL to resource relative to base url
108
120
  # @param display_fields [Array] fields to display by default
109
- # @param id_default [String] default identifier to use for existing entity commands (show, modify)
110
121
  # @param item_list_key [String] result is in a sub key of the json
111
122
  # @param id_as_arg [String] if set, the id is provided as url argument ?<id_as_arg>=<id>
112
123
  # @param is_singleton [Boolean] if true, res_class_path is the full path to the resource
124
+ # @param block [Proc] block to search for identifier based on attribute value
113
125
  # @return result suitable for CLI result
114
- def entity_command(command, rest_api, res_class_path, display_fields: nil, id_default: nil, item_list_key: false, id_as_arg: false, is_singleton: false, &block)
126
+ def entity_command(command, rest_api, res_class_path, display_fields: nil, item_list_key: false, id_as_arg: false, is_singleton: false, &block)
115
127
  if is_singleton
116
128
  one_res_path = res_class_path
117
129
  elsif INSTANCE_OPS.include?(command)
118
- begin
119
- one_res_id = instance_identifier(&block)
120
- rescue StandardError => e
121
- raise e if id_default.nil?
122
- one_res_id = id_default
123
- end
130
+ one_res_id = instance_identifier(&block)
124
131
  one_res_path = "#{res_class_path}/#{one_res_id}"
125
132
  one_res_path = "#{res_class_path}?#{id_as_arg}=#{one_res_id}" if id_as_arg
126
133
  end
@@ -128,13 +135,12 @@ module Aspera
128
135
  case command
129
136
  when :create
130
137
  raise 'cannot create singleton' if is_singleton
131
- return do_bulk_operation(value_create_modify(command: command, type: :bulk_hash), 'created', fields: display_fields) do |params|
132
- raise 'expecting Hash' unless params.is_a?(Hash)
138
+ return do_bulk_operation(command: command, descr: 'data', fields: display_fields) do |params|
133
139
  rest_api.create(res_class_path, params)[:data]
134
140
  end
135
141
  when :delete
136
142
  raise 'cannot delete singleton' if is_singleton
137
- return do_bulk_operation(one_res_id, 'deleted') do |one_id|
143
+ return do_bulk_operation(command: command, descr: 'identifier', values: one_res_id) do |one_id|
138
144
  rest_api.delete("#{res_class_path}/#{one_id}", old_query_read_delete)
139
145
  {'id' => one_id}
140
146
  end
@@ -142,6 +148,7 @@ module Aspera
142
148
  return {type: :single_object, data: rest_api.read(one_res_path)[:data], fields: display_fields}
143
149
  when :list
144
150
  resp = rest_api.read(res_class_path, old_query_read_delete)
151
+ return Main.result_empty if resp[:http].code == '204'
145
152
  data = resp[:data]
146
153
  # TODO: not generic : which application is this for ?
147
154
  if resp[:http]['Content-Type'].start_with?('application/vnd.api+json')
@@ -164,7 +171,7 @@ module Aspera
164
171
  raise "An error occurred: unexpected result type for list: #{data.class}"
165
172
  end
166
173
  when :modify
167
- parameters = value_create_modify(command: command, type: Hash)
174
+ parameters = value_create_modify(command: command)
168
175
  property = options.get_option(:property)
169
176
  parameters = {property => parameters} unless property.nil?
170
177
  rest_api.update(one_res_path, parameters)
@@ -175,10 +182,10 @@ module Aspera
175
182
  end
176
183
 
177
184
  # implement generic rest operations on given resource path
178
- def entity_action(rest_api, res_class_path, **opts)
185
+ def entity_action(rest_api, res_class_path, **opts, &block)
179
186
  # res_name=res_class_path.gsub(%r{^.*/},'').gsub(%r{s$},'').gsub('_',' ')
180
187
  command = options.get_next_command(ALL_OPS)
181
- return entity_command(command, rest_api, res_class_path, **opts)
188
+ return entity_command(command, rest_api, res_class_path, **opts, &block)
182
189
  end
183
190
 
184
191
  # query parameters in URL suitable for REST list/GET and delete/DELETE
@@ -191,12 +198,13 @@ module Aspera
191
198
  # check it is suitable
192
199
  URI.encode_www_form(query) unless query.nil?
193
200
  rescue StandardError => e
194
- raise CliBadArgument, "Query must be an extended value which can be encoded with URI.encode_www_form. Refer to manual. (#{e.message})"
201
+ raise Cli::BadArgument, "Query must be an extended value which can be encoded with URI.encode_www_form. Refer to manual. (#{e.message})"
195
202
  end
196
203
  return query
197
204
  end
198
205
 
199
206
  # TODO: when deprecation of `value` is completed: remove this method, replace with query_read_delete
207
+ # deprecation: 4.14
200
208
  def old_query_read_delete
201
209
  query = options.get_option(:value) # legacy, deprecated, remove, one day...
202
210
  query = query_read_delete if query.nil?
@@ -204,31 +212,41 @@ module Aspera
204
212
  end
205
213
 
206
214
  # TODO: when deprecation of `value` is completed: remove this method, replace with options.get_option(:query)
207
- def value_or_query(mandatory: false, allowed_types: nil)
208
- value = options.get_option(:value, mandatory: false, allowed_types: allowed_types)
209
- value = options.get_option(:query, mandatory: mandatory, allowed_types: allowed_types) if value.nil?
215
+ # deprecation: 4.14
216
+ def query_option(mandatory: false, default: nil)
217
+ option = :value
218
+ value = options.get_option(option, mandatory: false)
219
+ if value.nil?
220
+ option = :query
221
+ value = options.get_option(option, mandatory: mandatory, default: default)
222
+ end
210
223
  return value
211
224
  end
212
225
 
213
226
  # Retrieves an extended value from command line, used for creation or modification of entities
227
+ # @param command [Symbol] command name for error message
228
+ # @param type [Class] expected type of value, either a Class, an Array of Class
229
+ # @param bulk [Boolean] if true, value must be an Array of <type>
214
230
  # @param default [Object] default value if not provided
215
- # @param command [String] command name for error message
216
- # @param type [Class] expected type of value
217
231
  # TODO: when deprecation of `value` is completed: remove line with :value
218
- def value_create_modify(default: nil, command: 'command', type: nil)
232
+ def value_create_modify(command:, description: nil, type: Hash, bulk: false, default: nil)
219
233
  value = options.get_option(:value)
220
- value = options.get_next_argument("parameters for #{command}", mandatory: default.nil?) if value.nil?
234
+ Log.log.warn("option `value` is deprecated. Use positional parameter for #{command}") unless value.nil?
235
+ value = options.get_next_argument(
236
+ "parameters for #{command}#{description.nil? ? '' : " (#{description})"}", mandatory: default.nil?,
237
+ type: bulk ? Array : type) if value.nil?
221
238
  value = default if value.nil?
222
- if type.nil?
223
- # nothing to do
224
- elsif type.is_a?(Class)
225
- raise CliBadArgument, "Value must be a #{type}" unless value.is_a?(type)
226
- elsif type.is_a?(Array)
227
- raise CliBadArgument, "Value must be one of #{type.join(', ')}" unless type.any?{|t| value.is_a?(t)}
228
- elsif type.eql?(:bulk_hash)
229
- raise CliBadArgument, 'Value must be a Hash or Array of Hash' unless value.is_a?(Hash) || (value.is_a?(Array) && value.all?(Hash))
230
- else
231
- raise "Internal error: #{type}"
239
+ unless type.nil?
240
+ type = [type] unless type.is_a?(Array)
241
+ assert(type.all?(Class)){"check types must be a Class, not #{type.map(&:class).join(',')}"}
242
+ if bulk
243
+ assert_type(value, Array, exception_class: Cli::BadArgument)
244
+ value.each do |v|
245
+ assert_values(v.class, type, exception_class: Cli::BadArgument)
246
+ end
247
+ else
248
+ assert_values(value.class, type, exception_class: Cli::BadArgument)
249
+ end
232
250
  end
233
251
  return value
234
252
  end
@@ -1,6 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/rest'
4
3
  require 'aspera/aoc'
5
4
 
6
5
  module Aspera