aspera-cli 4.13.0 → 4.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +81 -7
  4. data/CONTRIBUTING.md +22 -6
  5. data/README.md +2038 -1080
  6. data/bin/ascli +18 -9
  7. data/bin/asession +12 -14
  8. data/examples/dascli +1 -1
  9. data/examples/proxy.pac +1 -1
  10. data/examples/rubyc +24 -0
  11. data/lib/aspera/aoc.rb +219 -159
  12. data/lib/aspera/ascmd.rb +25 -14
  13. data/lib/aspera/cli/basic_auth_plugin.rb +12 -9
  14. data/lib/aspera/cli/error.rb +17 -0
  15. data/lib/aspera/cli/extended_value.rb +47 -12
  16. data/lib/aspera/cli/formatter.rb +260 -179
  17. data/lib/aspera/cli/hints.rb +80 -0
  18. data/lib/aspera/cli/main.rb +104 -156
  19. data/lib/aspera/cli/manager.rb +259 -209
  20. data/lib/aspera/cli/plugin.rb +123 -63
  21. data/lib/aspera/cli/plugins/alee.rb +2 -3
  22. data/lib/aspera/cli/plugins/aoc.rb +341 -261
  23. data/lib/aspera/cli/plugins/ats.rb +22 -21
  24. data/lib/aspera/cli/plugins/bss.rb +5 -5
  25. data/lib/aspera/cli/plugins/config.rb +578 -627
  26. data/lib/aspera/cli/plugins/console.rb +44 -6
  27. data/lib/aspera/cli/plugins/cos.rb +15 -17
  28. data/lib/aspera/cli/plugins/faspex.rb +114 -100
  29. data/lib/aspera/cli/plugins/faspex5.rb +411 -264
  30. data/lib/aspera/cli/plugins/node.rb +354 -259
  31. data/lib/aspera/cli/plugins/orchestrator.rb +61 -29
  32. data/lib/aspera/cli/plugins/preview.rb +82 -90
  33. data/lib/aspera/cli/plugins/server.rb +79 -32
  34. data/lib/aspera/cli/plugins/shares.rb +55 -42
  35. data/lib/aspera/cli/sync_actions.rb +68 -0
  36. data/lib/aspera/cli/transfer_agent.rb +66 -73
  37. data/lib/aspera/cli/transfer_progress.rb +74 -0
  38. data/lib/aspera/cli/version.rb +1 -1
  39. data/lib/aspera/colors.rb +12 -8
  40. data/lib/aspera/command_line_builder.rb +14 -11
  41. data/lib/aspera/cos_node.rb +3 -2
  42. data/lib/aspera/data/6 +0 -0
  43. data/lib/aspera/environment.rb +24 -9
  44. data/lib/aspera/fasp/agent_aspera.rb +126 -0
  45. data/lib/aspera/fasp/agent_base.rb +31 -77
  46. data/lib/aspera/fasp/agent_connect.rb +25 -21
  47. data/lib/aspera/fasp/agent_direct.rb +89 -103
  48. data/lib/aspera/fasp/agent_httpgw.rb +231 -149
  49. data/lib/aspera/fasp/agent_node.rb +41 -34
  50. data/lib/aspera/fasp/agent_trsdk.rb +75 -32
  51. data/lib/aspera/fasp/error_info.rb +4 -2
  52. data/lib/aspera/fasp/faux_file.rb +52 -0
  53. data/lib/aspera/fasp/installation.rb +53 -195
  54. data/lib/aspera/fasp/management.rb +244 -0
  55. data/lib/aspera/fasp/parameters.rb +71 -37
  56. data/lib/aspera/fasp/parameters.yaml +76 -8
  57. data/lib/aspera/fasp/products.rb +162 -0
  58. data/lib/aspera/fasp/resume_policy.rb +3 -3
  59. data/lib/aspera/fasp/transfer_spec.rb +7 -6
  60. data/lib/aspera/fasp/uri.rb +26 -24
  61. data/lib/aspera/faspex_gw.rb +2 -2
  62. data/lib/aspera/faspex_postproc.rb +2 -2
  63. data/lib/aspera/hash_ext.rb +14 -4
  64. data/lib/aspera/json_rpc.rb +49 -0
  65. data/lib/aspera/keychain/macos_security.rb +13 -13
  66. data/lib/aspera/line_logger.rb +23 -0
  67. data/lib/aspera/log.rb +58 -16
  68. data/lib/aspera/node.rb +157 -92
  69. data/lib/aspera/oauth.rb +37 -19
  70. data/lib/aspera/open_application.rb +4 -4
  71. data/lib/aspera/persistency_action_once.rb +1 -1
  72. data/lib/aspera/persistency_folder.rb +2 -2
  73. data/lib/aspera/preview/file_types.rb +4 -2
  74. data/lib/aspera/preview/generator.rb +22 -35
  75. data/lib/aspera/preview/options.rb +2 -0
  76. data/lib/aspera/preview/terminal.rb +73 -16
  77. data/lib/aspera/preview/utils.rb +21 -28
  78. data/lib/aspera/proxy_auto_config.js +2 -2
  79. data/lib/aspera/rest.rb +136 -68
  80. data/lib/aspera/rest_call_error.rb +1 -1
  81. data/lib/aspera/rest_error_analyzer.rb +15 -14
  82. data/lib/aspera/rest_errors_aspera.rb +37 -34
  83. data/lib/aspera/secret_hider.rb +18 -15
  84. data/lib/aspera/ssh.rb +5 -2
  85. data/lib/aspera/sync.rb +127 -119
  86. data/lib/aspera/temp_file_manager.rb +10 -3
  87. data/lib/aspera/web_auth.rb +10 -7
  88. data/lib/aspera/web_server_simple.rb +9 -4
  89. data.tar.gz.sig +0 -0
  90. metadata +34 -17
  91. metadata.gz.sig +0 -0
  92. data/docs/test_env.conf +0 -186
  93. data/lib/aspera/cli/listener/line_dump.rb +0 -19
  94. data/lib/aspera/cli/listener/logger.rb +0 -22
  95. data/lib/aspera/cli/listener/progress.rb +0 -50
  96. data/lib/aspera/cli/listener/progress_multi.rb +0 -84
  97. data/lib/aspera/cli/plugins/sync.rb +0 -44
  98. data/lib/aspera/data/7 +0 -0
  99. data/lib/aspera/fasp/listener.rb +0 -13
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'aspera/cli/extended_value'
4
+
3
5
  module Aspera
4
6
  module Cli
5
7
  # base class for plugins modules
@@ -8,62 +10,83 @@ module Aspera
8
10
  GLOBAL_OPS = %i[create list].freeze
9
11
  # operations with id
10
12
  INSTANCE_OPS = %i[modify delete show].freeze
13
+ # all standard operations
11
14
  ALL_OPS = [GLOBAL_OPS, INSTANCE_OPS].flatten.freeze
12
- # max number of items for list command
15
+ # special query parameter: max number of items for list command
13
16
  MAX_ITEMS = 'max'
14
- # max number of pages for list command
17
+ # special query parameter: max number of pages for list command
15
18
  MAX_PAGES = 'pmax'
16
- # used when all resources are selected
17
- VAL_ALL = 'ALL'
19
+ # special identifier format: look for this name to find where supported
20
+ REGEX_LOOKUP_ID_BY_FIELD = /^%([^:]+):(.*)$/.freeze
18
21
 
19
- # global for inherited classes
20
- @@options_created = false # rubocop:disable Style/ClassVars
22
+ class << self
23
+ def declare_generic_options(options)
24
+ options.declare(:query, 'Additional filter for for some commands (list/delete)', types: Hash)
25
+ options.declare(
26
+ :value, 'Value for create, update, list filter', types: Hash,
27
+ deprecation: '(4.14) Use positional value for create/modify or option: query for list/delete')
28
+ options.declare(:property, 'Name of property to set (modify operation)')
29
+ options.declare(:id, 'Resource identifier', deprecation: "(4.14) Use positional identifier after verb (#{INSTANCE_OPS.join(',')})")
30
+ options.declare(:bulk, 'Bulk operation (only some)', values: :bool, default: :no)
31
+ options.declare(:bfail, 'Bulk operation error handling', values: :bool, default: :yes)
32
+ end
33
+ end
21
34
 
22
35
  def initialize(env)
23
- raise 'must be Hash' unless env.is_a?(Hash)
24
- # env.each_key {|k| raise "wrong agent key #{k}" unless AGENTS.include?(k)}
36
+ raise 'env must be Hash' unless env.is_a?(Hash)
25
37
  @agents = env
26
38
  # check presence in descendant of mandatory method and constant
27
- raise StandardError, "missing method 'execute_action' in #{self.class}" unless respond_to?(:execute_action)
39
+ raise StandardError, "Missing method 'execute_action' in #{self.class}" unless respond_to?(:execute_action)
28
40
  raise StandardError, 'ACTIONS shall be redefined by subclass' unless self.class.constants.include?(:ACTIONS)
41
+ # manual header for all plugins
29
42
  options.parser.separator('')
30
43
  options.parser.separator("COMMAND: #{self.class.name.split('::').last.downcase}")
31
44
  options.parser.separator("SUBCOMMANDS: #{self.class.const_get(:ACTIONS).map(&:to_s).sort.join(' ')}")
32
45
  options.parser.separator('OPTIONS:')
33
- return if @@options_created
34
- options.add_opt_simple(:query, 'additional filter for API calls (extended value) (some commands)')
35
- options.add_opt_simple(:value, 'extended value for create, update, list filter')
36
- options.add_opt_simple(:property, 'name of property to set')
37
- options.add_opt_simple(:id, "resource identifier (#{INSTANCE_OPS.join(',')})")
38
- options.add_opt_boolean(:bulk, 'Bulk operation (only some)')
39
- options.add_opt_boolean(:bfail, 'Bulk operation error handling')
40
- options.set_option(:bulk, :no)
41
- options.set_option(:bfail, :yes)
42
- options.parse_options!
43
- @@options_created = true # rubocop:disable Style/ClassVars
44
46
  end
45
47
 
46
- # must be called AFTER the instance action
47
- def instance_identifier
48
- res_id = options.get_option(:id)
49
- res_id = options.get_next_argument('identifier') if res_id.nil?
48
+ # must be called AFTER the instance action, ... folder browse <call instance_identifier>
49
+ # @param description [String] description of the identifier
50
+ # @param as_option [Symbol] option name to use if identifier is an option
51
+ # @param block [Proc] block to search for identifier based on attribute value
52
+ # @return [String] identifier
53
+ def instance_identifier(description: 'identifier', as_option: nil, &block)
54
+ if as_option.nil?
55
+ res_id = options.get_option(:id)
56
+ res_id = options.get_next_argument(description) if res_id.nil?
57
+ else
58
+ res_id = options.get_option(as_option)
59
+ end
60
+ # cab be an Array
61
+ if res_id.is_a?(String) && (m = res_id.match(REGEX_LOOKUP_ID_BY_FIELD))
62
+ if block
63
+ res_id = yield(m[1], ExtendedValue.instance.evaluate(m[2]))
64
+ else
65
+ raise Cli::BadArgument, "Percent syntax for #{description} not supported in this context"
66
+ end
67
+ end
50
68
  return res_id
51
69
  end
52
70
 
53
- # TODO
54
- # def get_next_id_command(instance_ops: INSTANCE_OPS,global_ops: GLOBAL_OPS)
55
- # return get_next_argument('command',expected: command_list)
56
- # end
57
-
58
71
  # For create and delete operations: execute one actin or multiple if bulk is yes
59
- # @param params either single id or hash, or array for bulk
60
- # @param success_msg deleted or created
61
- def do_bulk_operation(single_or_array, success_msg, id_result: 'id', fields: :default)
62
- raise 'programming error: missing block' unless block_given?
63
- params = options.get_option(:bulk) ? single_or_array : [single_or_array]
64
- raise 'expecting Array for bulk operation' unless params.is_a?(Array)
72
+ # @param command [Symbol] operation: :create, :delete, ...
73
+ # @param descr [String] description of the value
74
+ # @param values [Object] the value(s), or the type of value to get from user
75
+ # @param id_result [String] key in result hash to use as identifier
76
+ # @param fields [Array] fields to display
77
+ def do_bulk_operation(command:, descr:, values: Hash, id_result: 'id', fields: :default)
78
+ is_bulk = options.get_option(:bulk)
79
+ case values
80
+ when :identifier
81
+ values = instance_identifier
82
+ when Class
83
+ values = value_create_modify(command: command, type: values, bulk: is_bulk)
84
+ end
85
+ raise 'Internal error: missing block' unless block_given?
86
+ # if not bulk, there is a single value
87
+ params = is_bulk ? values : [values]
65
88
  Log.log.warn('Empty list given for bulk operation') if params.empty?
66
- Log.dump(:bulk_create, params)
89
+ Log.log.debug{Log.dump(:bulk_operation, params)}
67
90
  result_list = []
68
91
  params.each do |param|
69
92
  # init for delete
@@ -73,7 +96,8 @@ module Aspera
73
96
  res = yield(param)
74
97
  # if block returns a hash, let's use this (create)
75
98
  result = res if param.is_a?(Hash)
76
- result['status'] = success_msg
99
+ # create -> created
100
+ result['status'] = "#{command}#{'e' unless command.to_s.end_with?('e')}d".gsub(/yed$/, 'ied')
77
101
  rescue StandardError => e
78
102
  raise e if options.get_option(:bfail)
79
103
  result['status'] = e.to_s
@@ -81,7 +105,7 @@ module Aspera
81
105
  result_list.push(result)
82
106
  end
83
107
  display_fields = [id_result, 'status']
84
- if options.get_option(:bulk)
108
+ if is_bulk
85
109
  return {type: :object_list, data: result_list, fields: display_fields}
86
110
  else
87
111
  display_fields = fields unless fields.eql?(:default)
@@ -93,44 +117,36 @@ module Aspera
93
117
  # @param rest_api [Rest] api to use
94
118
  # @param res_class_path [String] sub path in URL to resource relative to base url
95
119
  # @param display_fields [Array] fields to display by default
96
- # @param id_default [String] default identifier to use for existing entity commands (show, modify)
97
120
  # @param item_list_key [String] result is in a sub key of the json
98
121
  # @param id_as_arg [String] if set, the id is provided as url argument ?<id_as_arg>=<id>
122
+ # @param is_singleton [Boolean] if true, res_class_path is the full path to the resource
123
+ # @param block [Proc] block to search for identifier based on attribute value
99
124
  # @return result suitable for CLI result
100
- def entity_command(command, rest_api, res_class_path, display_fields: nil, id_default: nil, item_list_key: false, id_as_arg: false)
101
- if INSTANCE_OPS.include?(command)
102
- begin
103
- one_res_id = instance_identifier
104
- rescue StandardError => e
105
- raise e if id_default.nil?
106
- one_res_id = id_default
107
- end
125
+ def entity_command(command, rest_api, res_class_path, display_fields: nil, item_list_key: false, id_as_arg: false, is_singleton: false, &block)
126
+ if is_singleton
127
+ one_res_path = res_class_path
128
+ elsif INSTANCE_OPS.include?(command)
129
+ one_res_id = instance_identifier(&block)
108
130
  one_res_path = "#{res_class_path}/#{one_res_id}"
109
131
  one_res_path = "#{res_class_path}?#{id_as_arg}=#{one_res_id}" if id_as_arg
110
132
  end
111
- # parameters mandatory for create/modify
112
- if %i[create modify].include?(command)
113
- parameters = options.get_option(:value, is_type: :mandatory)
114
- end
115
- # parameters optional for list
116
- if %i[list delete].include?(command)
117
- parameters = options.get_option(:value)
118
- end
133
+
119
134
  case command
120
135
  when :create
121
- return do_bulk_operation(parameters, 'created', fields: display_fields) do |params|
122
- raise 'expecting Hash' unless params.is_a?(Hash)
136
+ raise 'cannot create singleton' if is_singleton
137
+ return do_bulk_operation(command: command, descr: 'data', fields: display_fields) do |params|
123
138
  rest_api.create(res_class_path, params)[:data]
124
139
  end
125
140
  when :delete
126
- return do_bulk_operation(one_res_id, 'deleted') do |one_id|
127
- rest_api.delete("#{res_class_path}/#{one_id}", parameters)
141
+ raise 'cannot delete singleton' if is_singleton
142
+ return do_bulk_operation(command: command, descr: 'identifier', values: one_res_id) do |one_id|
143
+ rest_api.delete("#{res_class_path}/#{one_id}", old_query_read_delete)
128
144
  {'id' => one_id}
129
145
  end
130
146
  when :show
131
147
  return {type: :single_object, data: rest_api.read(one_res_path)[:data], fields: display_fields}
132
148
  when :list
133
- resp = rest_api.read(res_class_path, parameters)
149
+ resp = rest_api.read(res_class_path, old_query_read_delete)
134
150
  data = resp[:data]
135
151
  # TODO: not generic : which application is this for ?
136
152
  if resp[:http]['Content-Type'].start_with?('application/vnd.api+json')
@@ -153,6 +169,7 @@ module Aspera
153
169
  raise "An error occurred: unexpected result type for list: #{data.class}"
154
170
  end
155
171
  when :modify
172
+ parameters = value_create_modify(command: command)
156
173
  property = options.get_option(:property)
157
174
  parameters = {property => parameters} unless property.nil?
158
175
  rest_api.update(one_res_path, parameters)
@@ -169,8 +186,8 @@ module Aspera
169
186
  return entity_command(command, rest_api, res_class_path, **opts)
170
187
  end
171
188
 
172
- # query for list operation
173
- def option_url_query(default)
189
+ # query parameters in URL suitable for REST list/GET and delete/DELETE
190
+ def query_read_delete(default: nil)
174
191
  query = options.get_option(:query)
175
192
  # dup default, as it could be frozen
176
193
  query = default.dup if query.nil?
@@ -179,11 +196,54 @@ module Aspera
179
196
  # check it is suitable
180
197
  URI.encode_www_form(query) unless query.nil?
181
198
  rescue StandardError => e
182
- raise CliBadArgument, "query must be an extended value which can be encoded with URI.encode_www_form. Refer to manual. (#{e.message})"
199
+ raise Cli::BadArgument, "Query must be an extended value which can be encoded with URI.encode_www_form. Refer to manual. (#{e.message})"
183
200
  end
184
201
  return query
185
202
  end
186
203
 
204
+ # TODO: when deprecation of `value` is completed: remove this method, replace with query_read_delete
205
+ # deprecation: 4.14
206
+ def old_query_read_delete
207
+ query = options.get_option(:value) # legacy, deprecated, remove, one day...
208
+ query = query_read_delete if query.nil?
209
+ return query
210
+ end
211
+
212
+ # TODO: when deprecation of `value` is completed: remove this method, replace with options.get_option(:query)
213
+ # deprecation: 4.14
214
+ def query_option(mandatory: false, default: nil)
215
+ option = :value
216
+ value = options.get_option(option, mandatory: false)
217
+ if value.nil?
218
+ option = :query
219
+ value = options.get_option(option, mandatory: mandatory, default: default)
220
+ end
221
+ return value
222
+ end
223
+
224
+ # Retrieves an extended value from command line, used for creation or modification of entities
225
+ # @param command [Symbol] command name for error message
226
+ # @param type [Class] expected type of value, either a Class, an Array of Class, or :bulk_hash
227
+ # @param default [Object] default value if not provided
228
+ # TODO: when deprecation of `value` is completed: remove line with :value
229
+ def value_create_modify(command:, type: Hash, bulk: false, default: nil)
230
+ value = options.get_option(:value)
231
+ Log.log.warn("option `value` is deprecated. Use positional parameter for #{command}") unless value.nil?
232
+ value = options.get_next_argument("parameters for #{command}", mandatory: default.nil?) if value.nil?
233
+ value = default if value.nil?
234
+ unless type.nil?
235
+ type = [type] unless type.is_a?(Array)
236
+ raise "Internal error, check types must be a Class, not #{type.map(&:class).join(',')}" unless type.all?(Class)
237
+ if bulk
238
+ raise Cli::BadArgument, "Value must be an Array of #{type.join(',')}" unless value.is_a?(Array)
239
+ raise Cli::BadArgument, "Value must be a #{type.join(',')}, not #{value.map{|i| i.class.name}.uniq.join(',')}" unless value.all?{|v|type.include?(v.class)}
240
+ else
241
+ raise Cli::BadArgument, "Value must be a #{type.join(',')}, not #{value.class.name}" unless type.include?(value.class)
242
+ end
243
+ end
244
+ return value
245
+ end
246
+
187
247
  # shortcuts helpers for plugin environment
188
248
  %i[options transfer config formatter persistency].each do |name|
189
249
  define_method(name){@agents[name]}
@@ -1,6 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/rest'
4
3
  require 'aspera/aoc'
5
4
 
6
5
  module Aspera
@@ -13,8 +12,8 @@ module Aspera
13
12
  command = options.get_next_command(ACTIONS)
14
13
  case command
15
14
  when :entitlement
16
- entitlement_id = options.get_option(:username, is_type: :mandatory)
17
- customer_id = options.get_option(:password, is_type: :mandatory)
15
+ entitlement_id = options.get_option(:username, mandatory: true)
16
+ customer_id = options.get_option(:password, mandatory: true)
18
17
  api_metering = AoC.metering_api(entitlement_id, customer_id)
19
18
  return {type: :single_object, data: api_metering.read('entitlement')[:data]}
20
19
  end