aspera-cli 4.24.1 → 4.25.0.pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +1064 -745
  4. data/CONTRIBUTING.md +43 -100
  5. data/README.md +1281 -720
  6. data/bin/ascli +20 -1
  7. data/bin/asession +23 -27
  8. data/lib/aspera/agent/base.rb +10 -21
  9. data/lib/aspera/agent/connect.rb +2 -3
  10. data/lib/aspera/agent/desktop.rb +2 -2
  11. data/lib/aspera/agent/direct.rb +49 -32
  12. data/lib/aspera/agent/factory.rb +31 -0
  13. data/lib/aspera/api/aoc.rb +134 -76
  14. data/lib/aspera/api/cos_node.rb +3 -2
  15. data/lib/aspera/api/faspex.rb +213 -0
  16. data/lib/aspera/api/node.rb +107 -94
  17. data/lib/aspera/ascmd.rb +1 -2
  18. data/lib/aspera/ascp/installation.rb +73 -58
  19. data/lib/aspera/ascp/management.rb +119 -23
  20. data/lib/aspera/assert.rb +39 -11
  21. data/lib/aspera/cli/error.rb +4 -2
  22. data/lib/aspera/cli/extended_value.rb +91 -67
  23. data/lib/aspera/cli/formatter.rb +62 -27
  24. data/lib/aspera/cli/hints.rb +8 -0
  25. data/lib/aspera/cli/info.rb +4 -4
  26. data/lib/aspera/cli/main.rb +76 -84
  27. data/lib/aspera/cli/manager.rb +352 -248
  28. data/lib/aspera/cli/plugins/alee.rb +5 -4
  29. data/lib/aspera/cli/plugins/aoc.rb +175 -195
  30. data/lib/aspera/cli/plugins/ats.rb +4 -4
  31. data/lib/aspera/cli/plugins/base.rb +343 -0
  32. data/lib/aspera/cli/plugins/basic_auth.rb +45 -0
  33. data/lib/aspera/cli/plugins/config.rb +283 -269
  34. data/lib/aspera/cli/plugins/console.rb +27 -22
  35. data/lib/aspera/cli/plugins/cos.rb +3 -3
  36. data/lib/aspera/cli/plugins/factory.rb +78 -0
  37. data/lib/aspera/cli/plugins/faspex.rb +49 -46
  38. data/lib/aspera/cli/plugins/faspex5.rb +113 -225
  39. data/lib/aspera/cli/plugins/faspio.rb +19 -18
  40. data/lib/aspera/cli/plugins/httpgw.rb +14 -13
  41. data/lib/aspera/cli/plugins/node.rb +162 -149
  42. data/lib/aspera/cli/plugins/oauth.rb +48 -0
  43. data/lib/aspera/cli/plugins/orchestrator.rb +129 -45
  44. data/lib/aspera/cli/plugins/preview.rb +30 -50
  45. data/lib/aspera/cli/plugins/server.rb +21 -21
  46. data/lib/aspera/cli/plugins/shares.rb +45 -47
  47. data/lib/aspera/cli/sync_actions.rb +50 -39
  48. data/lib/aspera/cli/transfer_agent.rb +35 -49
  49. data/lib/aspera/cli/transfer_progress.rb +6 -6
  50. data/lib/aspera/cli/version.rb +3 -3
  51. data/lib/aspera/cli/wizard.rb +70 -55
  52. data/lib/aspera/colors.rb +6 -0
  53. data/lib/aspera/command_line_builder.rb +59 -61
  54. data/lib/aspera/command_line_converter.rb +2 -1
  55. data/lib/aspera/coverage.rb +2 -2
  56. data/lib/aspera/data_repository.rb +1 -1
  57. data/lib/aspera/environment.rb +51 -41
  58. data/lib/aspera/faspex_gw.rb +7 -5
  59. data/lib/aspera/faspex_postproc.rb +1 -1
  60. data/lib/aspera/keychain/factory.rb +1 -2
  61. data/lib/aspera/keychain/macos_security.rb +1 -1
  62. data/lib/aspera/log.rb +37 -9
  63. data/lib/aspera/markdown.rb +31 -0
  64. data/lib/aspera/nagios.rb +7 -6
  65. data/lib/aspera/oauth/base.rb +25 -28
  66. data/lib/aspera/oauth/factory.rb +9 -9
  67. data/lib/aspera/oauth/url_json.rb +2 -1
  68. data/lib/aspera/oauth/web.rb +2 -2
  69. data/lib/aspera/preview/file_types.rb +23 -37
  70. data/lib/aspera/products/connect.rb +7 -6
  71. data/lib/aspera/products/desktop.rb +1 -4
  72. data/lib/aspera/products/other.rb +9 -1
  73. data/lib/aspera/products/transferd.rb +0 -1
  74. data/lib/aspera/rest.rb +168 -113
  75. data/lib/aspera/rest_error_analyzer.rb +4 -4
  76. data/lib/aspera/ssh.rb +7 -4
  77. data/lib/aspera/ssl.rb +41 -0
  78. data/lib/aspera/sync/args.schema.yaml +46 -3
  79. data/lib/aspera/sync/conf.schema.yaml +307 -123
  80. data/lib/aspera/sync/database.rb +2 -1
  81. data/lib/aspera/sync/operations.rb +135 -79
  82. data/lib/aspera/temp_file_manager.rb +17 -5
  83. data/lib/aspera/transfer/error.rb +16 -7
  84. data/lib/aspera/transfer/parameters.rb +35 -22
  85. data/lib/aspera/transfer/resumer.rb +74 -0
  86. data/lib/aspera/transfer/spec.rb +5 -5
  87. data/lib/aspera/transfer/spec.schema.yaml +170 -59
  88. data/lib/aspera/transfer/spec_doc.rb +49 -43
  89. data/lib/aspera/uri_reader.rb +2 -2
  90. data/lib/aspera/web_auth.rb +6 -6
  91. data/lib/transferd_pb.rb +2 -2
  92. data.tar.gz.sig +0 -0
  93. metadata +26 -11
  94. metadata.gz.sig +0 -0
  95. data/lib/aspera/cli/basic_auth_plugin.rb +0 -43
  96. data/lib/aspera/cli/plugin.rb +0 -333
  97. data/lib/aspera/cli/plugin_factory.rb +0 -81
  98. data/lib/aspera/resumer.rb +0 -77
  99. data/lib/aspera/transfer/error_info.rb +0 -91
@@ -1,333 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'aspera/cli/extended_value'
4
- require 'aspera/assert'
5
-
6
- module Aspera
7
- module Cli
8
- # Base class for plugins
9
- class Plugin
10
- # operations without id
11
- GLOBAL_OPS = %i[create list].freeze
12
- # operations with id
13
- INSTANCE_OPS = %i[modify delete show].freeze
14
- # all standard operations
15
- ALL_OPS = (GLOBAL_OPS + INSTANCE_OPS).freeze
16
- # special query parameter: max number of items for list command
17
- MAX_ITEMS = 'max'
18
- # special query parameter: max number of pages for list command
19
- MAX_PAGES = 'pmax'
20
- # special identifier format: look for this name to find where supported
21
- REGEX_LOOKUP_ID_BY_FIELD = /^%([^:]+):(.*)$/
22
- PER_PAGE_DEFAULT = 1000
23
- private_constant :PER_PAGE_DEFAULT
24
-
25
- class << self
26
- def declare_generic_options(options)
27
- options.declare(:query, 'Additional filter for for some commands (list/delete)', types: [Hash, Array])
28
- options.declare(:property, 'Name of property to set (modify operation)')
29
- options.declare(:bulk, 'Bulk operation (only some)', values: :bool, default: :no)
30
- options.declare(:bfail, 'Bulk operation error handling', values: :bool, default: :yes)
31
- end
32
- end
33
-
34
- def initialize(context:)
35
- # check presence in descendant of mandatory method and constant
36
- Aspera.assert(respond_to?(:execute_action), type: InternalError){"Missing method 'execute_action' in #{self.class}"}
37
- Aspera.assert(self.class.constants.include?(:ACTIONS), type: InternalError){"Missing constant 'ACTIONS' in #{self.class}"}
38
- @context = context
39
- add_manual_header if @context.man_header
40
- end
41
-
42
- # Global objects
43
- attr_reader :context
44
-
45
- def options; @context.options; end
46
- def transfer; @context.transfer; end
47
- def config; @context.config; end
48
- def formatter; @context.formatter; end
49
- def persistency; @context.persistency; end
50
-
51
- def add_manual_header(has_options = true)
52
- # manual header for all plugins
53
- options.parser.separator('')
54
- options.parser.separator("COMMAND: #{self.class.name.split('::').last.downcase}")
55
- options.parser.separator("SUBCOMMANDS: #{self.class.const_get(:ACTIONS).map(&:to_s).sort.join(' ')}")
56
- options.parser.separator('OPTIONS:') if has_options
57
- end
58
-
59
- # Must be called AFTER the instance action:
60
- # ... folder browse _call_instance_identifier
61
- #
62
- # @param description [String] description of the identifier
63
- # @param as_option [Symbol] option name to use if identifier is an option
64
- # @param block [Proc] block to search for identifier based on attribute value
65
- # @return [String, Array] identifier or list of ids
66
- def instance_identifier(description: 'identifier', as_option: nil, &block)
67
- if as_option.nil?
68
- res_id = options.get_next_argument(description, multiple: options.get_option(:bulk)) if res_id.nil?
69
- else
70
- res_id = options.get_option(as_option)
71
- end
72
- # can be an Array
73
- if res_id.is_a?(String) && (m = res_id.match(REGEX_LOOKUP_ID_BY_FIELD))
74
- if block
75
- res_id = yield(m[1], ExtendedValue.instance.evaluate(m[2]))
76
- else
77
- raise Cli::BadArgument, "Percent syntax for #{description} not supported in this context"
78
- end
79
- end
80
- return res_id
81
- end
82
-
83
- # For create and delete operations: execute one actin or multiple if bulk is yes
84
- # @param command [Symbol] operation: :create, :delete, ...
85
- # @param descr [String] description of the value
86
- # @param values [Object] the value(s), or the type of value to get from user
87
- # @param id_result [String] key in result hash to use as identifier
88
- # @param fields [Array] fields to display
89
- # @param &block [Proc] block to execute for each value
90
- def do_bulk_operation(command:, descr: nil, values: Hash, id_result: 'id', fields: :default)
91
- Aspera.assert(block_given?){'missing block'}
92
- is_bulk = options.get_option(:bulk)
93
- case values
94
- when :identifier
95
- values = instance_identifier(description: descr)
96
- when Class
97
- values = value_create_modify(command: command, description: descr, type: values, bulk: is_bulk)
98
- end
99
- # if not bulk, there is a single value
100
- params = is_bulk ? values : [values]
101
- Log.log.warn('Empty list given for bulk operation') if params.empty?
102
- Log.dump(:bulk_operation, params)
103
- result_list = []
104
- params.each do |param|
105
- # init for delete
106
- result = {id_result => param}
107
- begin
108
- # execute custom code
109
- res = yield(param)
110
- # if block returns a hash, let's use this (create)
111
- result = res if res.is_a?(Hash)
112
- # TODO: remove when faspio gw api fixes this
113
- result = res.first if res.is_a?(Array) && res.first.is_a?(Hash)
114
- # create -> created
115
- result['status'] = "#{command}#{'e' unless command.to_s.end_with?('e')}d".gsub(/yed$/, 'ied')
116
- rescue StandardError => e
117
- raise e if options.get_option(:bfail)
118
- result['status'] = e.to_s
119
- end
120
- result_list.push(result)
121
- end
122
- display_fields = [id_result, 'status']
123
- if is_bulk
124
- return Main.result_object_list(result_list, fields: display_fields)
125
- else
126
- display_fields = fields unless fields.eql?(:default)
127
- return Main.result_single_object(result_list.first, fields: display_fields)
128
- end
129
- end
130
-
131
- # Operations: Create, Delete, Show, List, Modify
132
- # @param api [Rest] api to use
133
- # @param entity [String] sub path in URL to resource relative to base url
134
- # @param command [Symbol] command to execute: create show list modify delete
135
- # @param display_fields [Array] fields to display by default
136
- # @param items_key [String] result is in a sub key of the json
137
- # @param delete_style [String] if set, the delete operation by array in payload
138
- # @param id_as_arg [String] if set, the id is provided as url argument ?<id_as_arg>=<id>
139
- # @param is_singleton [Boolean] if true, entity is the full path to the resource
140
- # @param tclo [Bool] if set, :list use paging with total_count, limit, offset
141
- # @param block [Proc] block to search for identifier based on attribute value
142
- # @return result suitable for CLI result
143
- def entity_execute(
144
- api:,
145
- entity:,
146
- command: nil,
147
- display_fields: nil,
148
- items_key: nil,
149
- delete_style: nil,
150
- id_as_arg: false,
151
- is_singleton: false,
152
- list_query: nil,
153
- tclo: false,
154
- &block
155
- )
156
- command = options.get_next_command(ALL_OPS) if command.nil?
157
- if is_singleton
158
- one_res_path = entity
159
- elsif INSTANCE_OPS.include?(command)
160
- one_res_id = instance_identifier(&block)
161
- one_res_path = "#{entity}/#{one_res_id}"
162
- one_res_path = "#{entity}?#{id_as_arg}=#{one_res_id}" if id_as_arg
163
- end
164
-
165
- case command
166
- when :create
167
- raise BadArgument, 'cannot create singleton' if is_singleton
168
- return do_bulk_operation(command: command, descr: 'data', fields: display_fields) do |params|
169
- api.create(entity, params)
170
- end
171
- when :delete
172
- raise BadArgument, 'cannot delete singleton' if is_singleton
173
- if !delete_style.nil?
174
- one_res_id = [one_res_id] unless one_res_id.is_a?(Array)
175
- Aspera.assert_type(one_res_id, Array, type: Cli::BadArgument)
176
- api.call(
177
- operation: 'DELETE',
178
- subpath: entity,
179
- content_type: Rest::MIME_JSON,
180
- body: {delete_style => one_res_id},
181
- headers: {'Accept' => Rest::MIME_JSON}
182
- )
183
- return Main.result_status('deleted')
184
- end
185
- return do_bulk_operation(command: command, values: one_res_id) do |one_id|
186
- api.delete("#{entity}/#{one_id}", query_read_delete)
187
- {'id' => one_id}
188
- end
189
- when :show
190
- return Main.result_single_object(api.read(one_res_path), fields: display_fields)
191
- when :list
192
- if tclo
193
- data, total = list_entities_limit_offset_total_count(api: api, entity:, items_key: items_key, query: list_query)
194
- return Main.result_object_list(data, total: total, fields: display_fields)
195
- end
196
- resp = api.call(operation: 'GET', subpath: entity, headers: {'Accept' => Rest::MIME_JSON}, query: query_read_delete)
197
- return Main.result_empty if resp[:http].code == '204'
198
- data = resp[:data]
199
- # TODO: not generic : which application is this for ?
200
- if resp[:http]['Content-Type'].start_with?('application/vnd.api+json')
201
- Log.log.debug('is vnd.api')
202
- data = data[entity]
203
- end
204
- data = data[items_key] if items_key
205
- case data
206
- when Hash
207
- return Main.result_single_object(data, fields: display_fields)
208
- when Array
209
- return Main.result_object_list(data, fields: display_fields) if data.empty? || data.first.is_a?(Hash)
210
- return Main.result_value_list(data)
211
- else
212
- raise "An error occurred: unexpected result type for list: #{data.class}"
213
- end
214
- when :modify
215
- parameters = value_create_modify(command: command)
216
- property = options.get_option(:property)
217
- parameters = {property => parameters} unless property.nil?
218
- api.update(one_res_path, parameters)
219
- return Main.result_status('modified')
220
- else
221
- raise "unknown action: #{command}"
222
- end
223
- end
224
-
225
- # Query parameters in URL suitable for REST: list/GET and delete/DELETE
226
- def query_read_delete(default: nil)
227
- query = options.get_option(:query)
228
- # dup default, as it could be frozen
229
- query = default.dup if query.nil?
230
- Log.log.debug{"query_read_delete=#{query}".bg_red}
231
- begin
232
- # check it is suitable
233
- URI.encode_www_form(query) unless query.nil?
234
- rescue StandardError => e
235
- raise Cli::BadArgument, "Query must be an extended value (Hash, Array) which can be encoded with URI.encode_www_form. Refer to manual. (#{e.message})"
236
- end
237
- return query
238
- end
239
-
240
- # Retrieves an extended value from command line, used for creation or modification of entities
241
- # @param command [Symbol] command name for error message
242
- # @param type [Class] expected type of value, either a Class, an Array of Class
243
- # @param bulk [Boolean] if true, value must be an Array of <type>
244
- # @param default [Object] default value if not provided
245
- def value_create_modify(command:, description: nil, type: Hash, bulk: false, default: nil)
246
- value = options.get_next_argument(
247
- "parameters for #{command}#{" (#{description})" unless description.nil?}", mandatory: default.nil?,
248
- validation: bulk ? Array : type
249
- )
250
- value = default if value.nil?
251
- unless type.nil?
252
- type = [type] unless type.is_a?(Array)
253
- Aspera.assert(type.all?(Class)){"check types must be a Class, not #{type.map(&:class).join(',')}"}
254
- if bulk
255
- Aspera.assert_type(value, Array, type: Cli::BadArgument)
256
- value.each do |v|
257
- Aspera.assert_values(v.class, type, type: Cli::BadArgument)
258
- end
259
- else
260
- Aspera.assert_values(value.class, type, type: Cli::BadArgument)
261
- end
262
- end
263
- return value
264
- end
265
-
266
- # Get a (full or partial) list of all entities of a given type with query: offset/limit
267
- # @param `api` [Rest] the API object
268
- # @param `entity` [String,Symbol] the API endpoint of entity to list
269
- # @param `items_key` [String] key in the result to get the list of items
270
- # @param `query` [Hash,nil] additional query parameters
271
- # @return [Array] items, total_count
272
- def list_entities_limit_offset_total_count(
273
- api:,
274
- entity:,
275
- items_key: nil,
276
- query: nil
277
- )
278
- entity = entity.to_s if entity.is_a?(Symbol)
279
- items_key = entity.split('/').last if items_key.nil?
280
- query = {} if query.nil?
281
- Aspera.assert_type(entity, String)
282
- Aspera.assert_type(items_key, String)
283
- Aspera.assert_type(query, Hash)
284
- Log.log.debug{"list_entities t=#{entity} k=#{items_key} q=#{query}"}
285
- result = []
286
- offset = 0
287
- max_items = query.delete(MAX_ITEMS)
288
- remain_pages = query.delete(MAX_PAGES)
289
- # merge default parameters, by default 100 per page
290
- query = {'limit'=> PER_PAGE_DEFAULT}.merge(query)
291
- total_count = nil
292
- loop do
293
- query['offset'] = offset
294
- page_result = api.read(entity, query)
295
- Aspera.assert_type(page_result[items_key], Array)
296
- result.concat(page_result[items_key])
297
- # reach the limit set by user ?
298
- if !max_items.nil? && (result.length >= max_items)
299
- result = result.slice(0, max_items)
300
- break
301
- end
302
- total_count ||= page_result['total_count']
303
- break if result.length >= total_count
304
- remain_pages -= 1 unless remain_pages.nil?
305
- break if remain_pages == 0
306
- offset += page_result[items_key].length
307
- formatter.long_operation_running
308
- end
309
- formatter.long_operation_terminated
310
- return result, total_count
311
- end
312
-
313
- # Lookup an entity id from its name
314
- # @param entity [String] the type of entity to lookup, by default it is the path, and it is also the field name in result
315
- # @param value [String] the value to lookup
316
- # @param field [String] the field to match, by default it is 'name'
317
- # @param items_key [String] key in the result to get the list of items (override entity)
318
- # @param query [Hash] additional query parameters
319
- def lookup_entity_by_field(api:, entity:, value:, field: 'name', items_key: nil, query: :default)
320
- if query.eql?(:default)
321
- Aspera.assert(field.eql?('name')){'Default query is on name only'}
322
- query = {'q'=> value}
323
- end
324
- found = list_entities_limit_offset_total_count(api: api, entity: entity, items_key: items_key, query: query).first.select{ |i| i[field].eql?(value)}
325
- case found.length
326
- when 0 then raise "No #{entity} with #{field} = #{value}"
327
- when 1 then return found.first
328
- else raise "Found #{found.length} #{entity} with #{field} = #{value}"
329
- end
330
- end
331
- end
332
- end
333
- end
@@ -1,81 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'singleton'
4
- module Aspera
5
- module Cli
6
- # Instantiate plugin from well-known locations
7
- class PluginFactory
8
- include Singleton
9
-
10
- RUBY_FILE_EXT = '.rb'
11
- PLUGINS_MODULE = 'Plugins'
12
- private_constant :RUBY_FILE_EXT, :PLUGINS_MODULE
13
-
14
- attr_reader :lookup_folders
15
-
16
- def initialize
17
- @lookup_folders = []
18
- # information on plugins
19
- @plugins = {}
20
- end
21
-
22
- # @return list of registered plugins
23
- def plugin_list
24
- @plugins.keys
25
- end
26
-
27
- # @return path to source file of plugin
28
- def plugin_source(plugin_name_sym)
29
- @plugins[plugin_name_sym][:source]
30
- end
31
-
32
- # add a folder to the list of folders to look for plugins
33
- def add_lookup_folder(folder)
34
- @lookup_folders.unshift(folder)
35
- end
36
-
37
- # find plugins in defined paths
38
- def add_plugins_from_lookup_folders
39
- @lookup_folders.each do |folder|
40
- next unless File.directory?(folder)
41
- # TODO: add gem root to load path ? and require short folder ?
42
- # $LOAD_PATH.push(folder) if i[:add_path]
43
- Dir.entries(folder).select{ |file| file.end_with?(RUBY_FILE_EXT)}.each do |source|
44
- add_plugin_info(File.join(folder, source))
45
- end
46
- end
47
- end
48
-
49
- # @return Class object for plugin
50
- def plugin_class(plugin_name_sym)
51
- raise NoSuchElement, "plugin not found: #{plugin_name_sym}" unless @plugins.key?(plugin_name_sym)
52
- require @plugins[plugin_name_sym][:require_stanza]
53
- # Module.nesting[1] is Aspera::Cli
54
- return Object.const_get("#{Module.nesting[1]}::#{PLUGINS_MODULE}::#{plugin_name_sym.to_s.capitalize}")
55
- end
56
-
57
- # Create specified plugin
58
- # @param plugin_name_sym [Symbol] name of plugin
59
- # @param args [Hash] arguments to pass to plugin constructor
60
- def create(plugin_name_sym, **args)
61
- # TODO: check that ancestor is Plugin?
62
- plugin_class(plugin_name_sym).new(**args)
63
- end
64
-
65
- private
66
-
67
- # add plugin information to list
68
- # @param path [String] path to plugin source file
69
- def add_plugin_info(path)
70
- raise Error, "plugin path must end with #{RUBY_FILE_EXT}" if !path.end_with?(RUBY_FILE_EXT)
71
- plugin_symbol = File.basename(path, RUBY_FILE_EXT).to_sym
72
- req = path.sub(/#{RUBY_FILE_EXT}$/o, '')
73
- if @plugins.key?(plugin_symbol)
74
- Log.log.warn{"skipping plugin already registered: #{plugin_symbol}"}
75
- return
76
- end
77
- @plugins[plugin_symbol] = {source: path, require_stanza: req}
78
- end
79
- end
80
- end
81
- end
@@ -1,77 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'singleton'
4
- require 'aspera/log'
5
- require 'aspera/assert'
6
-
7
- module Aspera
8
- # implements a simple resume policy
9
- class Resumer
10
- # list of supported parameters and default values
11
- DEFAULTS = {
12
- iter_max: 7,
13
- sleep_initial: 2,
14
- sleep_factor: 2,
15
- sleep_max: 60
16
- }.freeze
17
-
18
- # @param params see DEFAULTS
19
- def initialize(params = nil)
20
- @parameters = DEFAULTS.dup
21
- if !params.nil?
22
- Aspera.assert_type(params, Hash)
23
- params.each do |k, v|
24
- Aspera.assert_values(k, DEFAULTS.keys){'resume parameter'}
25
- Aspera.assert_type(v, Integer){k}
26
- @parameters[k] = v
27
- end
28
- end
29
- Log.log.debug{"resume params=#{@parameters}"}
30
- end
31
-
32
- # calls block a number of times (resumes) until success or limit reached
33
- # this is re-entrant, one resumer can handle multiple transfers in //
34
- def execute_with_resume
35
- Aspera.assert(block_given?)
36
- # maximum of retry
37
- remaining_resumes = @parameters[:iter_max]
38
- sleep_seconds = @parameters[:sleep_initial]
39
- Log.log.debug{"retries=#{remaining_resumes}"}
40
- # try to send the file until ascp is successful
41
- loop do
42
- Log.log.debug('transfer starting')
43
- begin
44
- # call provided block
45
- yield
46
- # exit retry loop if success
47
- break
48
- rescue Transfer::Error => e
49
- Log.log.warn{"An error occurred during transfer: #{e.message}"}
50
- # failure in ascp
51
- if e.retryable?
52
- # exit if we exceed the max number of retry
53
- raise Transfer::Error, "Maximum number of retry reached (#{@parameters[:iter_max]})" if remaining_resumes <= 0
54
- else
55
- # give one chance only to non retryable errors
56
- unless remaining_resumes.eql?(@parameters[:iter_max])
57
- Log.log.error('non-retryable error'.red.blink)
58
- raise e
59
- end
60
- end
61
- end
62
-
63
- # take this retry in account
64
- remaining_resumes -= 1
65
- Log.log.warn{"Resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})"}
66
-
67
- # wait a bit before retrying, maybe network condition will be better
68
- sleep(sleep_seconds)
69
-
70
- # increase retry period
71
- sleep_seconds *= @parameters[:sleep_factor]
72
- # cap value
73
- sleep_seconds = @parameters[:sleep_max] if sleep_seconds > @parameters[:sleep_max]
74
- end
75
- end
76
- end
77
- end
@@ -1,91 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # cspell:words mnemo PROTO RCVR NOLIC PMTU BRTT VLINK BWMEAS SSEAR FIPS
4
-
5
- module Aspera
6
- module Transfer
7
- # from https://www.google.com/search?q=FASP+error+codes
8
- # Note that the fact that an error is retry-able is not internally defined by protocol, it's client-side responsibility
9
- # rubocop:disable Layout/FirstHashElementLineBreak
10
- ERROR_INFO = {
11
- # id retry-able mnemo message additional info
12
- 1 => {r: false, c: 'FASP_PROTO', m: 'Generic fasp(tm) protocol error', a: 'fasp(tm) error'},
13
- 2 => {r: false, c: 'ASCP', m: 'Generic SCP error', a: 'ASCP error'},
14
- 3 => {r: false, c: 'AMBIGUOUS_TARGET', m: 'Target incorrectly specified', a: 'Ambiguous target'},
15
- 4 => {r: false, c: 'NO_SUCH_FILE', m: 'No such file or directory', a: 'No such file or directory'},
16
- 5 => {r: false, c: 'NO_PERMS', m: 'Insufficient permission to read or write', a: 'Insufficient permissions'},
17
- 6 => {r: false, c: 'NOT_DIR', m: 'Target is not a directory', a: 'Target must be a directory'},
18
- 7 => {r: false, c: 'IS_DIR', m: 'File is a directory - expected regular file', a: 'Expected regular file'},
19
- 8 => {r: false, c: 'USAGE', m: 'Incorrect usage of scp command', a: 'Incorrect usage of Aspera scp command'},
20
- 9 => {r: false, c: 'LIC_DUP', m: 'Duplicate license', a: 'Duplicate license'},
21
- 10 => {r: false, c: 'LIC_RATE_EXCEEDED', m: 'Rate exceeds the cap imposed by license', a: 'Rate exceeds cap imposed by license'},
22
- 11 => {r: false, c: 'INTERNAL_ERROR', m: 'Internal error (unexpected error)', a: 'Internal error'},
23
- 12 => {r: true, c: 'TRANSFER_ERROR', m: 'Error establishing control connection',
24
- a: 'Error establishing SSH connection (check SSH port and firewall)'},
25
- 13 => {r: true, c: 'TRANSFER_TIMEOUT', m: 'Timeout establishing control connection',
26
- a: 'Timeout establishing SSH connection (check SSH port and firewall)'},
27
- 14 => {r: true, c: 'CONNECTION_ERROR', m: 'Error establishing data connection',
28
- a: 'Error establishing UDP connection (check UDP port and firewall)'},
29
- 15 => {r: true, c: 'CONNECTION_TIMEOUT', m: 'Timeout establishing data connection',
30
- a: 'Timeout establishing UDP connection (check UDP port and firewall)'},
31
- 16 => {r: true, c: 'CONNECTION_LOST', m: 'Connection lost', a: 'Connection lost'},
32
- 17 => {r: true, c: 'RCVR_SEND_ERROR', m: 'Receiver fails to send feedback', a: 'Network failure (receiver can\'t send feedback)'},
33
- 18 => {r: true, c: 'RCVR_RECV_ERROR', m: 'Receiver fails to receive data packets', a: 'Network failure (receiver can\'t receive UDP data)'},
34
- 19 => {r: false, c: 'AUTH', m: 'Authentication failure', a: 'Authentication failure'},
35
- 20 => {r: false, c: 'NOTHING', m: 'Nothing to transfer', a: 'Nothing to transfer'},
36
- 21 => {r: false, c: 'NOT_REGULAR', m: 'Not a regular file (special file)', a: 'Not a regular file'},
37
- 22 => {r: false, c: 'FILE_TABLE_OVR', m: 'File table overflow', a: 'File table overflow'},
38
- 23 => {r: true, c: 'TOO_MANY_FILES', m: 'Too many files open', a: 'Too many files open'},
39
- 24 => {r: false, c: 'FILE_TOO_BIG', m: 'File too big for file system', a: 'File too big for filesystem'},
40
- 25 => {r: false, c: 'NO_SPACE_LEFT', m: 'No space left on disk', a: 'No space left on disk'},
41
- 26 => {r: false, c: 'READ_ONLY_FS', m: 'Read only file system', a: 'Read only filesystem'},
42
- 27 => {r: false, c: 'SOME_FILE_ERRS', m: 'Some individual files failed', a: 'One or more files failed'},
43
- 28 => {r: false, c: 'USER_CANCEL', m: 'Cancelled by user', a: 'Cancelled by user'},
44
- 29 => {r: false, c: 'LIC_NOLIC', m: 'License not found or unable to access', a: 'Unable to access license info'},
45
- 30 => {r: false, c: 'LIC_EXPIRED', m: 'License expired', a: 'License expired'},
46
- 31 => {r: false, c: 'SOCK_SETUP', m: 'Unable to setup socket (create, bind, etc ...)', a: 'Unable to set up socket'},
47
- 32 => {r: true, c: 'OUT_OF_MEMORY', m: 'Out of memory, unable to allocate', a: 'Out of memory'},
48
- 33 => {r: true, c: 'THREAD_SPAWN', m: 'Can\'t spawn thread', a: 'Unable to spawn thread'},
49
- 34 => {r: false, c: 'UNAUTHORIZED', m: 'Unauthorized by external auth server', a: 'Unauthorized'},
50
- 35 => {r: true, c: 'DISK_READ', m: 'Error reading source file from disk', a: 'Disk read error'},
51
- 36 => {r: true, c: 'DISK_WRITE', m: 'Error writing to disk', a: 'Disk write error'},
52
- 37 => {r: true, c: 'AUTHORIZATION', m: 'Used interchangeably with ERR_UNAUTHORIZED', a: 'Authorization failure'},
53
- 38 => {r: false, c: 'LIC_ILLEGAL', m: 'Operation not permitted by license', a: 'Operation not permitted by license'},
54
- 39 => {r: true, c: 'PEER_ABORTED_SESSION', m: 'Remote peer terminated session', a: 'Peer aborted session'},
55
- 40 => {r: true, c: 'DATA_TRANSFER_TIMEOUT', m: 'Transfer stalled, timed out', a: 'Data transfer stalled, timed out'},
56
- 41 => {r: false, c: 'BAD_PATH', m: 'Path violates docroot containment', a: 'File location is outside \'docroot\' hierarchy'},
57
- 42 => {r: false, c: 'ALREADY_EXISTS', m: 'File or directory already exists', a: 'File or directory already exists'},
58
- 43 => {r: false, c: 'STAT_FAILS', m: 'Cannot stat file', a: 'Cannot collect details about file or directory'},
59
- 44 => {r: true, c: 'PMTU_BRTT_ERROR', m: 'UDP session initiation fatal error', a: 'UDP session initiation fatal error'},
60
- 45 => {r: true, c: 'BWMEAS_ERROR', m: 'Bandwidth measurement fatal error', a: 'Bandwidth measurement fatal error'},
61
- 46 => {r: false, c: 'VLINK_ERROR', m: 'Virtual link error', a: 'Virtual link error'},
62
- 47 => {r: false, c: 'CONNECTION_ERROR_HTTP', m: 'Error establishing HTTP connection',
63
- a: 'Error establishing HTTP connection (check HTTP port and firewall)'},
64
- 48 => {r: false, c: 'FILE_ENCRYPTION_ERROR', m: 'File encryption error, e.g. corrupt file',
65
- a: 'File encryption/decryption error, e.g. corrupt file'},
66
- 49 => {r: false, c: 'FILE_DECRYPTION_PASS', m: 'File encryption/decryption error, e.g. corrupt file', a: 'File decryption error, bad passphrase'},
67
- 50 => {r: false, c: 'BAD_CONFIGURATION', m: 'Aspera.conf contains invalid data and was rejected', a: 'Invalid configuration'},
68
- 51 => {r: false, c: 'INSECURE_CONNECTION', m: 'Remote-host key check failure', a: 'Remote host is not who we expected'},
69
- 52 => {r: false, c: 'START_VALIDATION_FAILED', m: 'File start validation failed', a: 'File start validation failed'},
70
- 53 => {r: false, c: 'STOP_VALIDATION_FAILED', m: 'File stop validation failed', a: 'File stop validation failed'},
71
- 54 => {r: false, c: 'THRESHOLD_VALIDATION_FAILED', m: 'File threshold validation failed', a: 'File threshold validation failed'},
72
- 55 => {r: false, c: 'FILEPATH_TOO_LONG', m: 'File path/name too long for underlying file system', a: 'File path exceeds underlying file system limit'},
73
- 56 => {r: false, c: 'ILLEGAL_CHARS_IN_PATH', m: 'Windows path contains illegal characters',
74
- a: 'Path being written to Windows file system contains illegal characters'},
75
- 57 => {r: false, c: 'CHUNK_MUST_MATCH_ALIGNMENT', m: 'Chunk size/start must be aligned with storage', a: 'Chunk size/start must be aligned with storage'},
76
- 58 => {r: false, c: 'VALIDATION_SESSION_ABORT', m: 'Session aborted to due to validation error', a: 'Session aborted to due validation error'},
77
- 59 => {r: false, c: 'REMOTE_STORAGE_ERROR', m: 'Remote storage errored', a: 'Remote storage errored'},
78
- 60 => {r: false, c: 'LUA_SCRIPT_ABORTED_SESSION', m: 'Session aborted due to Lua script abort', a: 'Session aborted due to Lua script abort'},
79
- 61 => {r: true, c: 'SSEAR_RETRYABLE', m: 'Transfer failed because of a retryable Encryption at Rest error',
80
- a: 'Transfer failed because of a retryable Encryption at Rest error'},
81
- 62 => {r: false, c: 'SSEAR_FATAL', m: 'Transfer failed because of a fatal Encryption at Rest error',
82
- a: 'Transfer failed because of a fatal Encryption at Rest error'},
83
- 63 => {r: false, c: 'LINK_LOOP', m: 'Path refers to a symbolic link loop', a: 'Path refers to a symbolic link loop'},
84
- 64 => {r: false, c: 'CANNOT_RENAME_PARTIAL_FILES', m: 'Can\'t rename a partial file', a: 'Can\'t rename a partial file.'},
85
- 65 => {r: false, c: 'CIPHER_NON_COMPAT_FIPS', m: 'Can\'t use this cipher with FIPS mode enabled', a: 'Can\'t use this cipher with FIPS mode enabled'},
86
- 66 => {r: false, c: 'PEER_REQUIRES_FIPS', m: 'Peer rejects cipher due to FIPS mode enabled on peer',
87
- a: 'Peer rejects cipher due to FIPS mode enabled on peer'}
88
- }.freeze
89
- # rubocop:enable Layout/FirstHashElementLineBreak
90
- end
91
- end