aspera-cli 4.24.1 → 4.25.0.pre
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +1064 -745
- data/CONTRIBUTING.md +43 -100
- data/README.md +1281 -720
- data/bin/ascli +20 -1
- data/bin/asession +23 -27
- data/lib/aspera/agent/base.rb +10 -21
- data/lib/aspera/agent/connect.rb +2 -3
- data/lib/aspera/agent/desktop.rb +2 -2
- data/lib/aspera/agent/direct.rb +49 -32
- data/lib/aspera/agent/factory.rb +31 -0
- data/lib/aspera/api/aoc.rb +134 -76
- data/lib/aspera/api/cos_node.rb +3 -2
- data/lib/aspera/api/faspex.rb +213 -0
- data/lib/aspera/api/node.rb +107 -94
- data/lib/aspera/ascmd.rb +1 -2
- data/lib/aspera/ascp/installation.rb +73 -58
- data/lib/aspera/ascp/management.rb +119 -23
- data/lib/aspera/assert.rb +39 -11
- data/lib/aspera/cli/error.rb +4 -2
- data/lib/aspera/cli/extended_value.rb +91 -67
- data/lib/aspera/cli/formatter.rb +62 -27
- data/lib/aspera/cli/hints.rb +8 -0
- data/lib/aspera/cli/info.rb +4 -4
- data/lib/aspera/cli/main.rb +76 -84
- data/lib/aspera/cli/manager.rb +352 -248
- data/lib/aspera/cli/plugins/alee.rb +5 -4
- data/lib/aspera/cli/plugins/aoc.rb +175 -195
- data/lib/aspera/cli/plugins/ats.rb +4 -4
- data/lib/aspera/cli/plugins/base.rb +343 -0
- data/lib/aspera/cli/plugins/basic_auth.rb +45 -0
- data/lib/aspera/cli/plugins/config.rb +283 -269
- data/lib/aspera/cli/plugins/console.rb +27 -22
- data/lib/aspera/cli/plugins/cos.rb +3 -3
- data/lib/aspera/cli/plugins/factory.rb +78 -0
- data/lib/aspera/cli/plugins/faspex.rb +49 -46
- data/lib/aspera/cli/plugins/faspex5.rb +113 -225
- data/lib/aspera/cli/plugins/faspio.rb +19 -18
- data/lib/aspera/cli/plugins/httpgw.rb +14 -13
- data/lib/aspera/cli/plugins/node.rb +162 -149
- data/lib/aspera/cli/plugins/oauth.rb +48 -0
- data/lib/aspera/cli/plugins/orchestrator.rb +129 -45
- data/lib/aspera/cli/plugins/preview.rb +30 -50
- data/lib/aspera/cli/plugins/server.rb +21 -21
- data/lib/aspera/cli/plugins/shares.rb +45 -47
- data/lib/aspera/cli/sync_actions.rb +50 -39
- data/lib/aspera/cli/transfer_agent.rb +35 -49
- data/lib/aspera/cli/transfer_progress.rb +6 -6
- data/lib/aspera/cli/version.rb +3 -3
- data/lib/aspera/cli/wizard.rb +70 -55
- data/lib/aspera/colors.rb +6 -0
- data/lib/aspera/command_line_builder.rb +59 -61
- data/lib/aspera/command_line_converter.rb +2 -1
- data/lib/aspera/coverage.rb +2 -2
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +51 -41
- data/lib/aspera/faspex_gw.rb +7 -5
- data/lib/aspera/faspex_postproc.rb +1 -1
- data/lib/aspera/keychain/factory.rb +1 -2
- data/lib/aspera/keychain/macos_security.rb +1 -1
- data/lib/aspera/log.rb +37 -9
- data/lib/aspera/markdown.rb +31 -0
- data/lib/aspera/nagios.rb +7 -6
- data/lib/aspera/oauth/base.rb +25 -28
- data/lib/aspera/oauth/factory.rb +9 -9
- data/lib/aspera/oauth/url_json.rb +2 -1
- data/lib/aspera/oauth/web.rb +2 -2
- data/lib/aspera/preview/file_types.rb +23 -37
- data/lib/aspera/products/connect.rb +7 -6
- data/lib/aspera/products/desktop.rb +1 -4
- data/lib/aspera/products/other.rb +9 -1
- data/lib/aspera/products/transferd.rb +0 -1
- data/lib/aspera/rest.rb +168 -113
- data/lib/aspera/rest_error_analyzer.rb +4 -4
- data/lib/aspera/ssh.rb +7 -4
- data/lib/aspera/ssl.rb +41 -0
- data/lib/aspera/sync/args.schema.yaml +46 -3
- data/lib/aspera/sync/conf.schema.yaml +307 -123
- data/lib/aspera/sync/database.rb +2 -1
- data/lib/aspera/sync/operations.rb +135 -79
- data/lib/aspera/temp_file_manager.rb +17 -5
- data/lib/aspera/transfer/error.rb +16 -7
- data/lib/aspera/transfer/parameters.rb +35 -22
- data/lib/aspera/transfer/resumer.rb +74 -0
- data/lib/aspera/transfer/spec.rb +5 -5
- data/lib/aspera/transfer/spec.schema.yaml +170 -59
- data/lib/aspera/transfer/spec_doc.rb +49 -43
- data/lib/aspera/uri_reader.rb +2 -2
- data/lib/aspera/web_auth.rb +6 -6
- data/lib/transferd_pb.rb +2 -2
- data.tar.gz.sig +0 -0
- metadata +26 -11
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/basic_auth_plugin.rb +0 -43
- data/lib/aspera/cli/plugin.rb +0 -333
- data/lib/aspera/cli/plugin_factory.rb +0 -81
- data/lib/aspera/resumer.rb +0 -77
- data/lib/aspera/transfer/error_info.rb +0 -91
data/lib/aspera/cli/plugin.rb
DELETED
|
@@ -1,333 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'aspera/cli/extended_value'
|
|
4
|
-
require 'aspera/assert'
|
|
5
|
-
|
|
6
|
-
module Aspera
|
|
7
|
-
module Cli
|
|
8
|
-
# Base class for plugins
|
|
9
|
-
class Plugin
|
|
10
|
-
# operations without id
|
|
11
|
-
GLOBAL_OPS = %i[create list].freeze
|
|
12
|
-
# operations with id
|
|
13
|
-
INSTANCE_OPS = %i[modify delete show].freeze
|
|
14
|
-
# all standard operations
|
|
15
|
-
ALL_OPS = (GLOBAL_OPS + INSTANCE_OPS).freeze
|
|
16
|
-
# special query parameter: max number of items for list command
|
|
17
|
-
MAX_ITEMS = 'max'
|
|
18
|
-
# special query parameter: max number of pages for list command
|
|
19
|
-
MAX_PAGES = 'pmax'
|
|
20
|
-
# special identifier format: look for this name to find where supported
|
|
21
|
-
REGEX_LOOKUP_ID_BY_FIELD = /^%([^:]+):(.*)$/
|
|
22
|
-
PER_PAGE_DEFAULT = 1000
|
|
23
|
-
private_constant :PER_PAGE_DEFAULT
|
|
24
|
-
|
|
25
|
-
class << self
|
|
26
|
-
def declare_generic_options(options)
|
|
27
|
-
options.declare(:query, 'Additional filter for for some commands (list/delete)', types: [Hash, Array])
|
|
28
|
-
options.declare(:property, 'Name of property to set (modify operation)')
|
|
29
|
-
options.declare(:bulk, 'Bulk operation (only some)', values: :bool, default: :no)
|
|
30
|
-
options.declare(:bfail, 'Bulk operation error handling', values: :bool, default: :yes)
|
|
31
|
-
end
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
def initialize(context:)
|
|
35
|
-
# check presence in descendant of mandatory method and constant
|
|
36
|
-
Aspera.assert(respond_to?(:execute_action), type: InternalError){"Missing method 'execute_action' in #{self.class}"}
|
|
37
|
-
Aspera.assert(self.class.constants.include?(:ACTIONS), type: InternalError){"Missing constant 'ACTIONS' in #{self.class}"}
|
|
38
|
-
@context = context
|
|
39
|
-
add_manual_header if @context.man_header
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
# Global objects
|
|
43
|
-
attr_reader :context
|
|
44
|
-
|
|
45
|
-
def options; @context.options; end
|
|
46
|
-
def transfer; @context.transfer; end
|
|
47
|
-
def config; @context.config; end
|
|
48
|
-
def formatter; @context.formatter; end
|
|
49
|
-
def persistency; @context.persistency; end
|
|
50
|
-
|
|
51
|
-
def add_manual_header(has_options = true)
|
|
52
|
-
# manual header for all plugins
|
|
53
|
-
options.parser.separator('')
|
|
54
|
-
options.parser.separator("COMMAND: #{self.class.name.split('::').last.downcase}")
|
|
55
|
-
options.parser.separator("SUBCOMMANDS: #{self.class.const_get(:ACTIONS).map(&:to_s).sort.join(' ')}")
|
|
56
|
-
options.parser.separator('OPTIONS:') if has_options
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
# Must be called AFTER the instance action:
|
|
60
|
-
# ... folder browse _call_instance_identifier
|
|
61
|
-
#
|
|
62
|
-
# @param description [String] description of the identifier
|
|
63
|
-
# @param as_option [Symbol] option name to use if identifier is an option
|
|
64
|
-
# @param block [Proc] block to search for identifier based on attribute value
|
|
65
|
-
# @return [String, Array] identifier or list of ids
|
|
66
|
-
def instance_identifier(description: 'identifier', as_option: nil, &block)
|
|
67
|
-
if as_option.nil?
|
|
68
|
-
res_id = options.get_next_argument(description, multiple: options.get_option(:bulk)) if res_id.nil?
|
|
69
|
-
else
|
|
70
|
-
res_id = options.get_option(as_option)
|
|
71
|
-
end
|
|
72
|
-
# can be an Array
|
|
73
|
-
if res_id.is_a?(String) && (m = res_id.match(REGEX_LOOKUP_ID_BY_FIELD))
|
|
74
|
-
if block
|
|
75
|
-
res_id = yield(m[1], ExtendedValue.instance.evaluate(m[2]))
|
|
76
|
-
else
|
|
77
|
-
raise Cli::BadArgument, "Percent syntax for #{description} not supported in this context"
|
|
78
|
-
end
|
|
79
|
-
end
|
|
80
|
-
return res_id
|
|
81
|
-
end
|
|
82
|
-
|
|
83
|
-
# For create and delete operations: execute one actin or multiple if bulk is yes
|
|
84
|
-
# @param command [Symbol] operation: :create, :delete, ...
|
|
85
|
-
# @param descr [String] description of the value
|
|
86
|
-
# @param values [Object] the value(s), or the type of value to get from user
|
|
87
|
-
# @param id_result [String] key in result hash to use as identifier
|
|
88
|
-
# @param fields [Array] fields to display
|
|
89
|
-
# @param &block [Proc] block to execute for each value
|
|
90
|
-
def do_bulk_operation(command:, descr: nil, values: Hash, id_result: 'id', fields: :default)
|
|
91
|
-
Aspera.assert(block_given?){'missing block'}
|
|
92
|
-
is_bulk = options.get_option(:bulk)
|
|
93
|
-
case values
|
|
94
|
-
when :identifier
|
|
95
|
-
values = instance_identifier(description: descr)
|
|
96
|
-
when Class
|
|
97
|
-
values = value_create_modify(command: command, description: descr, type: values, bulk: is_bulk)
|
|
98
|
-
end
|
|
99
|
-
# if not bulk, there is a single value
|
|
100
|
-
params = is_bulk ? values : [values]
|
|
101
|
-
Log.log.warn('Empty list given for bulk operation') if params.empty?
|
|
102
|
-
Log.dump(:bulk_operation, params)
|
|
103
|
-
result_list = []
|
|
104
|
-
params.each do |param|
|
|
105
|
-
# init for delete
|
|
106
|
-
result = {id_result => param}
|
|
107
|
-
begin
|
|
108
|
-
# execute custom code
|
|
109
|
-
res = yield(param)
|
|
110
|
-
# if block returns a hash, let's use this (create)
|
|
111
|
-
result = res if res.is_a?(Hash)
|
|
112
|
-
# TODO: remove when faspio gw api fixes this
|
|
113
|
-
result = res.first if res.is_a?(Array) && res.first.is_a?(Hash)
|
|
114
|
-
# create -> created
|
|
115
|
-
result['status'] = "#{command}#{'e' unless command.to_s.end_with?('e')}d".gsub(/yed$/, 'ied')
|
|
116
|
-
rescue StandardError => e
|
|
117
|
-
raise e if options.get_option(:bfail)
|
|
118
|
-
result['status'] = e.to_s
|
|
119
|
-
end
|
|
120
|
-
result_list.push(result)
|
|
121
|
-
end
|
|
122
|
-
display_fields = [id_result, 'status']
|
|
123
|
-
if is_bulk
|
|
124
|
-
return Main.result_object_list(result_list, fields: display_fields)
|
|
125
|
-
else
|
|
126
|
-
display_fields = fields unless fields.eql?(:default)
|
|
127
|
-
return Main.result_single_object(result_list.first, fields: display_fields)
|
|
128
|
-
end
|
|
129
|
-
end
|
|
130
|
-
|
|
131
|
-
# Operations: Create, Delete, Show, List, Modify
|
|
132
|
-
# @param api [Rest] api to use
|
|
133
|
-
# @param entity [String] sub path in URL to resource relative to base url
|
|
134
|
-
# @param command [Symbol] command to execute: create show list modify delete
|
|
135
|
-
# @param display_fields [Array] fields to display by default
|
|
136
|
-
# @param items_key [String] result is in a sub key of the json
|
|
137
|
-
# @param delete_style [String] if set, the delete operation by array in payload
|
|
138
|
-
# @param id_as_arg [String] if set, the id is provided as url argument ?<id_as_arg>=<id>
|
|
139
|
-
# @param is_singleton [Boolean] if true, entity is the full path to the resource
|
|
140
|
-
# @param tclo [Bool] if set, :list use paging with total_count, limit, offset
|
|
141
|
-
# @param block [Proc] block to search for identifier based on attribute value
|
|
142
|
-
# @return result suitable for CLI result
|
|
143
|
-
def entity_execute(
|
|
144
|
-
api:,
|
|
145
|
-
entity:,
|
|
146
|
-
command: nil,
|
|
147
|
-
display_fields: nil,
|
|
148
|
-
items_key: nil,
|
|
149
|
-
delete_style: nil,
|
|
150
|
-
id_as_arg: false,
|
|
151
|
-
is_singleton: false,
|
|
152
|
-
list_query: nil,
|
|
153
|
-
tclo: false,
|
|
154
|
-
&block
|
|
155
|
-
)
|
|
156
|
-
command = options.get_next_command(ALL_OPS) if command.nil?
|
|
157
|
-
if is_singleton
|
|
158
|
-
one_res_path = entity
|
|
159
|
-
elsif INSTANCE_OPS.include?(command)
|
|
160
|
-
one_res_id = instance_identifier(&block)
|
|
161
|
-
one_res_path = "#{entity}/#{one_res_id}"
|
|
162
|
-
one_res_path = "#{entity}?#{id_as_arg}=#{one_res_id}" if id_as_arg
|
|
163
|
-
end
|
|
164
|
-
|
|
165
|
-
case command
|
|
166
|
-
when :create
|
|
167
|
-
raise BadArgument, 'cannot create singleton' if is_singleton
|
|
168
|
-
return do_bulk_operation(command: command, descr: 'data', fields: display_fields) do |params|
|
|
169
|
-
api.create(entity, params)
|
|
170
|
-
end
|
|
171
|
-
when :delete
|
|
172
|
-
raise BadArgument, 'cannot delete singleton' if is_singleton
|
|
173
|
-
if !delete_style.nil?
|
|
174
|
-
one_res_id = [one_res_id] unless one_res_id.is_a?(Array)
|
|
175
|
-
Aspera.assert_type(one_res_id, Array, type: Cli::BadArgument)
|
|
176
|
-
api.call(
|
|
177
|
-
operation: 'DELETE',
|
|
178
|
-
subpath: entity,
|
|
179
|
-
content_type: Rest::MIME_JSON,
|
|
180
|
-
body: {delete_style => one_res_id},
|
|
181
|
-
headers: {'Accept' => Rest::MIME_JSON}
|
|
182
|
-
)
|
|
183
|
-
return Main.result_status('deleted')
|
|
184
|
-
end
|
|
185
|
-
return do_bulk_operation(command: command, values: one_res_id) do |one_id|
|
|
186
|
-
api.delete("#{entity}/#{one_id}", query_read_delete)
|
|
187
|
-
{'id' => one_id}
|
|
188
|
-
end
|
|
189
|
-
when :show
|
|
190
|
-
return Main.result_single_object(api.read(one_res_path), fields: display_fields)
|
|
191
|
-
when :list
|
|
192
|
-
if tclo
|
|
193
|
-
data, total = list_entities_limit_offset_total_count(api: api, entity:, items_key: items_key, query: list_query)
|
|
194
|
-
return Main.result_object_list(data, total: total, fields: display_fields)
|
|
195
|
-
end
|
|
196
|
-
resp = api.call(operation: 'GET', subpath: entity, headers: {'Accept' => Rest::MIME_JSON}, query: query_read_delete)
|
|
197
|
-
return Main.result_empty if resp[:http].code == '204'
|
|
198
|
-
data = resp[:data]
|
|
199
|
-
# TODO: not generic : which application is this for ?
|
|
200
|
-
if resp[:http]['Content-Type'].start_with?('application/vnd.api+json')
|
|
201
|
-
Log.log.debug('is vnd.api')
|
|
202
|
-
data = data[entity]
|
|
203
|
-
end
|
|
204
|
-
data = data[items_key] if items_key
|
|
205
|
-
case data
|
|
206
|
-
when Hash
|
|
207
|
-
return Main.result_single_object(data, fields: display_fields)
|
|
208
|
-
when Array
|
|
209
|
-
return Main.result_object_list(data, fields: display_fields) if data.empty? || data.first.is_a?(Hash)
|
|
210
|
-
return Main.result_value_list(data)
|
|
211
|
-
else
|
|
212
|
-
raise "An error occurred: unexpected result type for list: #{data.class}"
|
|
213
|
-
end
|
|
214
|
-
when :modify
|
|
215
|
-
parameters = value_create_modify(command: command)
|
|
216
|
-
property = options.get_option(:property)
|
|
217
|
-
parameters = {property => parameters} unless property.nil?
|
|
218
|
-
api.update(one_res_path, parameters)
|
|
219
|
-
return Main.result_status('modified')
|
|
220
|
-
else
|
|
221
|
-
raise "unknown action: #{command}"
|
|
222
|
-
end
|
|
223
|
-
end
|
|
224
|
-
|
|
225
|
-
# Query parameters in URL suitable for REST: list/GET and delete/DELETE
|
|
226
|
-
def query_read_delete(default: nil)
|
|
227
|
-
query = options.get_option(:query)
|
|
228
|
-
# dup default, as it could be frozen
|
|
229
|
-
query = default.dup if query.nil?
|
|
230
|
-
Log.log.debug{"query_read_delete=#{query}".bg_red}
|
|
231
|
-
begin
|
|
232
|
-
# check it is suitable
|
|
233
|
-
URI.encode_www_form(query) unless query.nil?
|
|
234
|
-
rescue StandardError => e
|
|
235
|
-
raise Cli::BadArgument, "Query must be an extended value (Hash, Array) which can be encoded with URI.encode_www_form. Refer to manual. (#{e.message})"
|
|
236
|
-
end
|
|
237
|
-
return query
|
|
238
|
-
end
|
|
239
|
-
|
|
240
|
-
# Retrieves an extended value from command line, used for creation or modification of entities
|
|
241
|
-
# @param command [Symbol] command name for error message
|
|
242
|
-
# @param type [Class] expected type of value, either a Class, an Array of Class
|
|
243
|
-
# @param bulk [Boolean] if true, value must be an Array of <type>
|
|
244
|
-
# @param default [Object] default value if not provided
|
|
245
|
-
def value_create_modify(command:, description: nil, type: Hash, bulk: false, default: nil)
|
|
246
|
-
value = options.get_next_argument(
|
|
247
|
-
"parameters for #{command}#{" (#{description})" unless description.nil?}", mandatory: default.nil?,
|
|
248
|
-
validation: bulk ? Array : type
|
|
249
|
-
)
|
|
250
|
-
value = default if value.nil?
|
|
251
|
-
unless type.nil?
|
|
252
|
-
type = [type] unless type.is_a?(Array)
|
|
253
|
-
Aspera.assert(type.all?(Class)){"check types must be a Class, not #{type.map(&:class).join(',')}"}
|
|
254
|
-
if bulk
|
|
255
|
-
Aspera.assert_type(value, Array, type: Cli::BadArgument)
|
|
256
|
-
value.each do |v|
|
|
257
|
-
Aspera.assert_values(v.class, type, type: Cli::BadArgument)
|
|
258
|
-
end
|
|
259
|
-
else
|
|
260
|
-
Aspera.assert_values(value.class, type, type: Cli::BadArgument)
|
|
261
|
-
end
|
|
262
|
-
end
|
|
263
|
-
return value
|
|
264
|
-
end
|
|
265
|
-
|
|
266
|
-
# Get a (full or partial) list of all entities of a given type with query: offset/limit
|
|
267
|
-
# @param `api` [Rest] the API object
|
|
268
|
-
# @param `entity` [String,Symbol] the API endpoint of entity to list
|
|
269
|
-
# @param `items_key` [String] key in the result to get the list of items
|
|
270
|
-
# @param `query` [Hash,nil] additional query parameters
|
|
271
|
-
# @return [Array] items, total_count
|
|
272
|
-
def list_entities_limit_offset_total_count(
|
|
273
|
-
api:,
|
|
274
|
-
entity:,
|
|
275
|
-
items_key: nil,
|
|
276
|
-
query: nil
|
|
277
|
-
)
|
|
278
|
-
entity = entity.to_s if entity.is_a?(Symbol)
|
|
279
|
-
items_key = entity.split('/').last if items_key.nil?
|
|
280
|
-
query = {} if query.nil?
|
|
281
|
-
Aspera.assert_type(entity, String)
|
|
282
|
-
Aspera.assert_type(items_key, String)
|
|
283
|
-
Aspera.assert_type(query, Hash)
|
|
284
|
-
Log.log.debug{"list_entities t=#{entity} k=#{items_key} q=#{query}"}
|
|
285
|
-
result = []
|
|
286
|
-
offset = 0
|
|
287
|
-
max_items = query.delete(MAX_ITEMS)
|
|
288
|
-
remain_pages = query.delete(MAX_PAGES)
|
|
289
|
-
# merge default parameters, by default 100 per page
|
|
290
|
-
query = {'limit'=> PER_PAGE_DEFAULT}.merge(query)
|
|
291
|
-
total_count = nil
|
|
292
|
-
loop do
|
|
293
|
-
query['offset'] = offset
|
|
294
|
-
page_result = api.read(entity, query)
|
|
295
|
-
Aspera.assert_type(page_result[items_key], Array)
|
|
296
|
-
result.concat(page_result[items_key])
|
|
297
|
-
# reach the limit set by user ?
|
|
298
|
-
if !max_items.nil? && (result.length >= max_items)
|
|
299
|
-
result = result.slice(0, max_items)
|
|
300
|
-
break
|
|
301
|
-
end
|
|
302
|
-
total_count ||= page_result['total_count']
|
|
303
|
-
break if result.length >= total_count
|
|
304
|
-
remain_pages -= 1 unless remain_pages.nil?
|
|
305
|
-
break if remain_pages == 0
|
|
306
|
-
offset += page_result[items_key].length
|
|
307
|
-
formatter.long_operation_running
|
|
308
|
-
end
|
|
309
|
-
formatter.long_operation_terminated
|
|
310
|
-
return result, total_count
|
|
311
|
-
end
|
|
312
|
-
|
|
313
|
-
# Lookup an entity id from its name
|
|
314
|
-
# @param entity [String] the type of entity to lookup, by default it is the path, and it is also the field name in result
|
|
315
|
-
# @param value [String] the value to lookup
|
|
316
|
-
# @param field [String] the field to match, by default it is 'name'
|
|
317
|
-
# @param items_key [String] key in the result to get the list of items (override entity)
|
|
318
|
-
# @param query [Hash] additional query parameters
|
|
319
|
-
def lookup_entity_by_field(api:, entity:, value:, field: 'name', items_key: nil, query: :default)
|
|
320
|
-
if query.eql?(:default)
|
|
321
|
-
Aspera.assert(field.eql?('name')){'Default query is on name only'}
|
|
322
|
-
query = {'q'=> value}
|
|
323
|
-
end
|
|
324
|
-
found = list_entities_limit_offset_total_count(api: api, entity: entity, items_key: items_key, query: query).first.select{ |i| i[field].eql?(value)}
|
|
325
|
-
case found.length
|
|
326
|
-
when 0 then raise "No #{entity} with #{field} = #{value}"
|
|
327
|
-
when 1 then return found.first
|
|
328
|
-
else raise "Found #{found.length} #{entity} with #{field} = #{value}"
|
|
329
|
-
end
|
|
330
|
-
end
|
|
331
|
-
end
|
|
332
|
-
end
|
|
333
|
-
end
|
|
@@ -1,81 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'singleton'
|
|
4
|
-
module Aspera
|
|
5
|
-
module Cli
|
|
6
|
-
# Instantiate plugin from well-known locations
|
|
7
|
-
class PluginFactory
|
|
8
|
-
include Singleton
|
|
9
|
-
|
|
10
|
-
RUBY_FILE_EXT = '.rb'
|
|
11
|
-
PLUGINS_MODULE = 'Plugins'
|
|
12
|
-
private_constant :RUBY_FILE_EXT, :PLUGINS_MODULE
|
|
13
|
-
|
|
14
|
-
attr_reader :lookup_folders
|
|
15
|
-
|
|
16
|
-
def initialize
|
|
17
|
-
@lookup_folders = []
|
|
18
|
-
# information on plugins
|
|
19
|
-
@plugins = {}
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
# @return list of registered plugins
|
|
23
|
-
def plugin_list
|
|
24
|
-
@plugins.keys
|
|
25
|
-
end
|
|
26
|
-
|
|
27
|
-
# @return path to source file of plugin
|
|
28
|
-
def plugin_source(plugin_name_sym)
|
|
29
|
-
@plugins[plugin_name_sym][:source]
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
# add a folder to the list of folders to look for plugins
|
|
33
|
-
def add_lookup_folder(folder)
|
|
34
|
-
@lookup_folders.unshift(folder)
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
# find plugins in defined paths
|
|
38
|
-
def add_plugins_from_lookup_folders
|
|
39
|
-
@lookup_folders.each do |folder|
|
|
40
|
-
next unless File.directory?(folder)
|
|
41
|
-
# TODO: add gem root to load path ? and require short folder ?
|
|
42
|
-
# $LOAD_PATH.push(folder) if i[:add_path]
|
|
43
|
-
Dir.entries(folder).select{ |file| file.end_with?(RUBY_FILE_EXT)}.each do |source|
|
|
44
|
-
add_plugin_info(File.join(folder, source))
|
|
45
|
-
end
|
|
46
|
-
end
|
|
47
|
-
end
|
|
48
|
-
|
|
49
|
-
# @return Class object for plugin
|
|
50
|
-
def plugin_class(plugin_name_sym)
|
|
51
|
-
raise NoSuchElement, "plugin not found: #{plugin_name_sym}" unless @plugins.key?(plugin_name_sym)
|
|
52
|
-
require @plugins[plugin_name_sym][:require_stanza]
|
|
53
|
-
# Module.nesting[1] is Aspera::Cli
|
|
54
|
-
return Object.const_get("#{Module.nesting[1]}::#{PLUGINS_MODULE}::#{plugin_name_sym.to_s.capitalize}")
|
|
55
|
-
end
|
|
56
|
-
|
|
57
|
-
# Create specified plugin
|
|
58
|
-
# @param plugin_name_sym [Symbol] name of plugin
|
|
59
|
-
# @param args [Hash] arguments to pass to plugin constructor
|
|
60
|
-
def create(plugin_name_sym, **args)
|
|
61
|
-
# TODO: check that ancestor is Plugin?
|
|
62
|
-
plugin_class(plugin_name_sym).new(**args)
|
|
63
|
-
end
|
|
64
|
-
|
|
65
|
-
private
|
|
66
|
-
|
|
67
|
-
# add plugin information to list
|
|
68
|
-
# @param path [String] path to plugin source file
|
|
69
|
-
def add_plugin_info(path)
|
|
70
|
-
raise Error, "plugin path must end with #{RUBY_FILE_EXT}" if !path.end_with?(RUBY_FILE_EXT)
|
|
71
|
-
plugin_symbol = File.basename(path, RUBY_FILE_EXT).to_sym
|
|
72
|
-
req = path.sub(/#{RUBY_FILE_EXT}$/o, '')
|
|
73
|
-
if @plugins.key?(plugin_symbol)
|
|
74
|
-
Log.log.warn{"skipping plugin already registered: #{plugin_symbol}"}
|
|
75
|
-
return
|
|
76
|
-
end
|
|
77
|
-
@plugins[plugin_symbol] = {source: path, require_stanza: req}
|
|
78
|
-
end
|
|
79
|
-
end
|
|
80
|
-
end
|
|
81
|
-
end
|
data/lib/aspera/resumer.rb
DELETED
|
@@ -1,77 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'singleton'
|
|
4
|
-
require 'aspera/log'
|
|
5
|
-
require 'aspera/assert'
|
|
6
|
-
|
|
7
|
-
module Aspera
|
|
8
|
-
# implements a simple resume policy
|
|
9
|
-
class Resumer
|
|
10
|
-
# list of supported parameters and default values
|
|
11
|
-
DEFAULTS = {
|
|
12
|
-
iter_max: 7,
|
|
13
|
-
sleep_initial: 2,
|
|
14
|
-
sleep_factor: 2,
|
|
15
|
-
sleep_max: 60
|
|
16
|
-
}.freeze
|
|
17
|
-
|
|
18
|
-
# @param params see DEFAULTS
|
|
19
|
-
def initialize(params = nil)
|
|
20
|
-
@parameters = DEFAULTS.dup
|
|
21
|
-
if !params.nil?
|
|
22
|
-
Aspera.assert_type(params, Hash)
|
|
23
|
-
params.each do |k, v|
|
|
24
|
-
Aspera.assert_values(k, DEFAULTS.keys){'resume parameter'}
|
|
25
|
-
Aspera.assert_type(v, Integer){k}
|
|
26
|
-
@parameters[k] = v
|
|
27
|
-
end
|
|
28
|
-
end
|
|
29
|
-
Log.log.debug{"resume params=#{@parameters}"}
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
# calls block a number of times (resumes) until success or limit reached
|
|
33
|
-
# this is re-entrant, one resumer can handle multiple transfers in //
|
|
34
|
-
def execute_with_resume
|
|
35
|
-
Aspera.assert(block_given?)
|
|
36
|
-
# maximum of retry
|
|
37
|
-
remaining_resumes = @parameters[:iter_max]
|
|
38
|
-
sleep_seconds = @parameters[:sleep_initial]
|
|
39
|
-
Log.log.debug{"retries=#{remaining_resumes}"}
|
|
40
|
-
# try to send the file until ascp is successful
|
|
41
|
-
loop do
|
|
42
|
-
Log.log.debug('transfer starting')
|
|
43
|
-
begin
|
|
44
|
-
# call provided block
|
|
45
|
-
yield
|
|
46
|
-
# exit retry loop if success
|
|
47
|
-
break
|
|
48
|
-
rescue Transfer::Error => e
|
|
49
|
-
Log.log.warn{"An error occurred during transfer: #{e.message}"}
|
|
50
|
-
# failure in ascp
|
|
51
|
-
if e.retryable?
|
|
52
|
-
# exit if we exceed the max number of retry
|
|
53
|
-
raise Transfer::Error, "Maximum number of retry reached (#{@parameters[:iter_max]})" if remaining_resumes <= 0
|
|
54
|
-
else
|
|
55
|
-
# give one chance only to non retryable errors
|
|
56
|
-
unless remaining_resumes.eql?(@parameters[:iter_max])
|
|
57
|
-
Log.log.error('non-retryable error'.red.blink)
|
|
58
|
-
raise e
|
|
59
|
-
end
|
|
60
|
-
end
|
|
61
|
-
end
|
|
62
|
-
|
|
63
|
-
# take this retry in account
|
|
64
|
-
remaining_resumes -= 1
|
|
65
|
-
Log.log.warn{"Resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})"}
|
|
66
|
-
|
|
67
|
-
# wait a bit before retrying, maybe network condition will be better
|
|
68
|
-
sleep(sleep_seconds)
|
|
69
|
-
|
|
70
|
-
# increase retry period
|
|
71
|
-
sleep_seconds *= @parameters[:sleep_factor]
|
|
72
|
-
# cap value
|
|
73
|
-
sleep_seconds = @parameters[:sleep_max] if sleep_seconds > @parameters[:sleep_max]
|
|
74
|
-
end
|
|
75
|
-
end
|
|
76
|
-
end
|
|
77
|
-
end
|
|
@@ -1,91 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
# cspell:words mnemo PROTO RCVR NOLIC PMTU BRTT VLINK BWMEAS SSEAR FIPS
|
|
4
|
-
|
|
5
|
-
module Aspera
|
|
6
|
-
module Transfer
|
|
7
|
-
# from https://www.google.com/search?q=FASP+error+codes
|
|
8
|
-
# Note that the fact that an error is retry-able is not internally defined by protocol, it's client-side responsibility
|
|
9
|
-
# rubocop:disable Layout/FirstHashElementLineBreak
|
|
10
|
-
ERROR_INFO = {
|
|
11
|
-
# id retry-able mnemo message additional info
|
|
12
|
-
1 => {r: false, c: 'FASP_PROTO', m: 'Generic fasp(tm) protocol error', a: 'fasp(tm) error'},
|
|
13
|
-
2 => {r: false, c: 'ASCP', m: 'Generic SCP error', a: 'ASCP error'},
|
|
14
|
-
3 => {r: false, c: 'AMBIGUOUS_TARGET', m: 'Target incorrectly specified', a: 'Ambiguous target'},
|
|
15
|
-
4 => {r: false, c: 'NO_SUCH_FILE', m: 'No such file or directory', a: 'No such file or directory'},
|
|
16
|
-
5 => {r: false, c: 'NO_PERMS', m: 'Insufficient permission to read or write', a: 'Insufficient permissions'},
|
|
17
|
-
6 => {r: false, c: 'NOT_DIR', m: 'Target is not a directory', a: 'Target must be a directory'},
|
|
18
|
-
7 => {r: false, c: 'IS_DIR', m: 'File is a directory - expected regular file', a: 'Expected regular file'},
|
|
19
|
-
8 => {r: false, c: 'USAGE', m: 'Incorrect usage of scp command', a: 'Incorrect usage of Aspera scp command'},
|
|
20
|
-
9 => {r: false, c: 'LIC_DUP', m: 'Duplicate license', a: 'Duplicate license'},
|
|
21
|
-
10 => {r: false, c: 'LIC_RATE_EXCEEDED', m: 'Rate exceeds the cap imposed by license', a: 'Rate exceeds cap imposed by license'},
|
|
22
|
-
11 => {r: false, c: 'INTERNAL_ERROR', m: 'Internal error (unexpected error)', a: 'Internal error'},
|
|
23
|
-
12 => {r: true, c: 'TRANSFER_ERROR', m: 'Error establishing control connection',
|
|
24
|
-
a: 'Error establishing SSH connection (check SSH port and firewall)'},
|
|
25
|
-
13 => {r: true, c: 'TRANSFER_TIMEOUT', m: 'Timeout establishing control connection',
|
|
26
|
-
a: 'Timeout establishing SSH connection (check SSH port and firewall)'},
|
|
27
|
-
14 => {r: true, c: 'CONNECTION_ERROR', m: 'Error establishing data connection',
|
|
28
|
-
a: 'Error establishing UDP connection (check UDP port and firewall)'},
|
|
29
|
-
15 => {r: true, c: 'CONNECTION_TIMEOUT', m: 'Timeout establishing data connection',
|
|
30
|
-
a: 'Timeout establishing UDP connection (check UDP port and firewall)'},
|
|
31
|
-
16 => {r: true, c: 'CONNECTION_LOST', m: 'Connection lost', a: 'Connection lost'},
|
|
32
|
-
17 => {r: true, c: 'RCVR_SEND_ERROR', m: 'Receiver fails to send feedback', a: 'Network failure (receiver can\'t send feedback)'},
|
|
33
|
-
18 => {r: true, c: 'RCVR_RECV_ERROR', m: 'Receiver fails to receive data packets', a: 'Network failure (receiver can\'t receive UDP data)'},
|
|
34
|
-
19 => {r: false, c: 'AUTH', m: 'Authentication failure', a: 'Authentication failure'},
|
|
35
|
-
20 => {r: false, c: 'NOTHING', m: 'Nothing to transfer', a: 'Nothing to transfer'},
|
|
36
|
-
21 => {r: false, c: 'NOT_REGULAR', m: 'Not a regular file (special file)', a: 'Not a regular file'},
|
|
37
|
-
22 => {r: false, c: 'FILE_TABLE_OVR', m: 'File table overflow', a: 'File table overflow'},
|
|
38
|
-
23 => {r: true, c: 'TOO_MANY_FILES', m: 'Too many files open', a: 'Too many files open'},
|
|
39
|
-
24 => {r: false, c: 'FILE_TOO_BIG', m: 'File too big for file system', a: 'File too big for filesystem'},
|
|
40
|
-
25 => {r: false, c: 'NO_SPACE_LEFT', m: 'No space left on disk', a: 'No space left on disk'},
|
|
41
|
-
26 => {r: false, c: 'READ_ONLY_FS', m: 'Read only file system', a: 'Read only filesystem'},
|
|
42
|
-
27 => {r: false, c: 'SOME_FILE_ERRS', m: 'Some individual files failed', a: 'One or more files failed'},
|
|
43
|
-
28 => {r: false, c: 'USER_CANCEL', m: 'Cancelled by user', a: 'Cancelled by user'},
|
|
44
|
-
29 => {r: false, c: 'LIC_NOLIC', m: 'License not found or unable to access', a: 'Unable to access license info'},
|
|
45
|
-
30 => {r: false, c: 'LIC_EXPIRED', m: 'License expired', a: 'License expired'},
|
|
46
|
-
31 => {r: false, c: 'SOCK_SETUP', m: 'Unable to setup socket (create, bind, etc ...)', a: 'Unable to set up socket'},
|
|
47
|
-
32 => {r: true, c: 'OUT_OF_MEMORY', m: 'Out of memory, unable to allocate', a: 'Out of memory'},
|
|
48
|
-
33 => {r: true, c: 'THREAD_SPAWN', m: 'Can\'t spawn thread', a: 'Unable to spawn thread'},
|
|
49
|
-
34 => {r: false, c: 'UNAUTHORIZED', m: 'Unauthorized by external auth server', a: 'Unauthorized'},
|
|
50
|
-
35 => {r: true, c: 'DISK_READ', m: 'Error reading source file from disk', a: 'Disk read error'},
|
|
51
|
-
36 => {r: true, c: 'DISK_WRITE', m: 'Error writing to disk', a: 'Disk write error'},
|
|
52
|
-
37 => {r: true, c: 'AUTHORIZATION', m: 'Used interchangeably with ERR_UNAUTHORIZED', a: 'Authorization failure'},
|
|
53
|
-
38 => {r: false, c: 'LIC_ILLEGAL', m: 'Operation not permitted by license', a: 'Operation not permitted by license'},
|
|
54
|
-
39 => {r: true, c: 'PEER_ABORTED_SESSION', m: 'Remote peer terminated session', a: 'Peer aborted session'},
|
|
55
|
-
40 => {r: true, c: 'DATA_TRANSFER_TIMEOUT', m: 'Transfer stalled, timed out', a: 'Data transfer stalled, timed out'},
|
|
56
|
-
41 => {r: false, c: 'BAD_PATH', m: 'Path violates docroot containment', a: 'File location is outside \'docroot\' hierarchy'},
|
|
57
|
-
42 => {r: false, c: 'ALREADY_EXISTS', m: 'File or directory already exists', a: 'File or directory already exists'},
|
|
58
|
-
43 => {r: false, c: 'STAT_FAILS', m: 'Cannot stat file', a: 'Cannot collect details about file or directory'},
|
|
59
|
-
44 => {r: true, c: 'PMTU_BRTT_ERROR', m: 'UDP session initiation fatal error', a: 'UDP session initiation fatal error'},
|
|
60
|
-
45 => {r: true, c: 'BWMEAS_ERROR', m: 'Bandwidth measurement fatal error', a: 'Bandwidth measurement fatal error'},
|
|
61
|
-
46 => {r: false, c: 'VLINK_ERROR', m: 'Virtual link error', a: 'Virtual link error'},
|
|
62
|
-
47 => {r: false, c: 'CONNECTION_ERROR_HTTP', m: 'Error establishing HTTP connection',
|
|
63
|
-
a: 'Error establishing HTTP connection (check HTTP port and firewall)'},
|
|
64
|
-
48 => {r: false, c: 'FILE_ENCRYPTION_ERROR', m: 'File encryption error, e.g. corrupt file',
|
|
65
|
-
a: 'File encryption/decryption error, e.g. corrupt file'},
|
|
66
|
-
49 => {r: false, c: 'FILE_DECRYPTION_PASS', m: 'File encryption/decryption error, e.g. corrupt file', a: 'File decryption error, bad passphrase'},
|
|
67
|
-
50 => {r: false, c: 'BAD_CONFIGURATION', m: 'Aspera.conf contains invalid data and was rejected', a: 'Invalid configuration'},
|
|
68
|
-
51 => {r: false, c: 'INSECURE_CONNECTION', m: 'Remote-host key check failure', a: 'Remote host is not who we expected'},
|
|
69
|
-
52 => {r: false, c: 'START_VALIDATION_FAILED', m: 'File start validation failed', a: 'File start validation failed'},
|
|
70
|
-
53 => {r: false, c: 'STOP_VALIDATION_FAILED', m: 'File stop validation failed', a: 'File stop validation failed'},
|
|
71
|
-
54 => {r: false, c: 'THRESHOLD_VALIDATION_FAILED', m: 'File threshold validation failed', a: 'File threshold validation failed'},
|
|
72
|
-
55 => {r: false, c: 'FILEPATH_TOO_LONG', m: 'File path/name too long for underlying file system', a: 'File path exceeds underlying file system limit'},
|
|
73
|
-
56 => {r: false, c: 'ILLEGAL_CHARS_IN_PATH', m: 'Windows path contains illegal characters',
|
|
74
|
-
a: 'Path being written to Windows file system contains illegal characters'},
|
|
75
|
-
57 => {r: false, c: 'CHUNK_MUST_MATCH_ALIGNMENT', m: 'Chunk size/start must be aligned with storage', a: 'Chunk size/start must be aligned with storage'},
|
|
76
|
-
58 => {r: false, c: 'VALIDATION_SESSION_ABORT', m: 'Session aborted to due to validation error', a: 'Session aborted to due validation error'},
|
|
77
|
-
59 => {r: false, c: 'REMOTE_STORAGE_ERROR', m: 'Remote storage errored', a: 'Remote storage errored'},
|
|
78
|
-
60 => {r: false, c: 'LUA_SCRIPT_ABORTED_SESSION', m: 'Session aborted due to Lua script abort', a: 'Session aborted due to Lua script abort'},
|
|
79
|
-
61 => {r: true, c: 'SSEAR_RETRYABLE', m: 'Transfer failed because of a retryable Encryption at Rest error',
|
|
80
|
-
a: 'Transfer failed because of a retryable Encryption at Rest error'},
|
|
81
|
-
62 => {r: false, c: 'SSEAR_FATAL', m: 'Transfer failed because of a fatal Encryption at Rest error',
|
|
82
|
-
a: 'Transfer failed because of a fatal Encryption at Rest error'},
|
|
83
|
-
63 => {r: false, c: 'LINK_LOOP', m: 'Path refers to a symbolic link loop', a: 'Path refers to a symbolic link loop'},
|
|
84
|
-
64 => {r: false, c: 'CANNOT_RENAME_PARTIAL_FILES', m: 'Can\'t rename a partial file', a: 'Can\'t rename a partial file.'},
|
|
85
|
-
65 => {r: false, c: 'CIPHER_NON_COMPAT_FIPS', m: 'Can\'t use this cipher with FIPS mode enabled', a: 'Can\'t use this cipher with FIPS mode enabled'},
|
|
86
|
-
66 => {r: false, c: 'PEER_REQUIRES_FIPS', m: 'Peer rejects cipher due to FIPS mode enabled on peer',
|
|
87
|
-
a: 'Peer rejects cipher due to FIPS mode enabled on peer'}
|
|
88
|
-
}.freeze
|
|
89
|
-
# rubocop:enable Layout/FirstHashElementLineBreak
|
|
90
|
-
end
|
|
91
|
-
end
|