aspera-cli 4.26.1 → 4.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +300 -22
- data/CONTRIBUTING.md +100 -12
- data/bin/ascli +5 -5
- data/docs/README.md +10757 -0
- data/docs/test-mcp-with-ai.md +202 -0
- data/lib/aspera/agent/base.rb +20 -6
- data/lib/aspera/agent/connect.rb +53 -16
- data/lib/aspera/agent/desktop.rb +52 -17
- data/lib/aspera/agent/direct.rb +27 -25
- data/lib/aspera/agent/factory.rb +4 -0
- data/lib/aspera/agent/httpgw.rb +2 -2
- data/lib/aspera/agent/node.rb +34 -2
- data/lib/aspera/agent/transferd.rb +39 -3
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +56 -51
- data/lib/aspera/api/cos_node.rb +2 -2
- data/lib/aspera/api/faspex.rb +22 -20
- data/lib/aspera/api/httpgw.rb +4 -4
- data/lib/aspera/api/node.rb +33 -30
- data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
- data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
- data/lib/aspera/ascmd.rb +25 -23
- data/lib/aspera/ascp/installation.rb +43 -40
- data/lib/aspera/ascp/management.rb +2 -3
- data/lib/aspera/cli/ascp_actions.rb +155 -0
- data/lib/aspera/cli/async_transfer_store.rb +81 -0
- data/lib/aspera/cli/bootstrapper.rb +197 -0
- data/lib/aspera/cli/command_registry.rb +145 -0
- data/lib/aspera/cli/command_spec.rb +138 -0
- data/lib/aspera/cli/context.rb +71 -0
- data/lib/aspera/cli/error.rb +16 -0
- data/lib/aspera/cli/extended_value.rb +5 -3
- data/lib/aspera/cli/formatter.rb +122 -209
- data/lib/aspera/cli/gem_checker.rb +65 -0
- data/lib/aspera/cli/hints.rb +2 -2
- data/lib/aspera/cli/http.rb +218 -0
- data/lib/aspera/cli/info.rb +2 -0
- data/lib/aspera/cli/mailer.rb +97 -0
- data/lib/aspera/cli/mcp_tool.rb +198 -0
- data/lib/aspera/cli/options.schema.yaml +364 -7
- data/lib/aspera/cli/{manager.rb → parser.rb} +337 -144
- data/lib/aspera/cli/plugins/alee.rb +20 -22
- data/lib/aspera/cli/plugins/aoc.rb +1173 -849
- data/lib/aspera/cli/plugins/ats.rb +200 -161
- data/lib/aspera/cli/plugins/base.rb +470 -107
- data/lib/aspera/cli/plugins/basic_auth.rb +14 -4
- data/lib/aspera/cli/plugins/config.rb +434 -1032
- data/lib/aspera/cli/plugins/console.rb +106 -64
- data/lib/aspera/cli/plugins/cos.rb +44 -32
- data/lib/aspera/cli/plugins/factory.rb +7 -4
- data/lib/aspera/cli/plugins/faspex.rb +296 -259
- data/lib/aspera/cli/plugins/faspex5.rb +592 -344
- data/lib/aspera/cli/plugins/faspio.rb +49 -51
- data/lib/aspera/cli/plugins/httpgw.rb +18 -25
- data/lib/aspera/cli/plugins/mcp.rb +279 -0
- data/lib/aspera/cli/plugins/node.rb +1001 -797
- data/lib/aspera/cli/plugins/oauth.rb +7 -10
- data/lib/aspera/cli/plugins/orchestrator.rb +111 -139
- data/lib/aspera/cli/plugins/preview.rb +183 -142
- data/lib/aspera/cli/plugins/server.rb +125 -80
- data/lib/aspera/cli/plugins/shares.rb +301 -107
- data/lib/aspera/cli/preset_actions.rb +139 -0
- data/lib/aspera/cli/preset_manager.rb +236 -0
- data/lib/aspera/cli/result.rb +360 -0
- data/lib/aspera/cli/runner.rb +473 -0
- data/lib/aspera/cli/secret_finder.rb +40 -0
- data/lib/aspera/cli/special_values.rb +1 -0
- data/lib/aspera/cli/sync_actions.rb +84 -56
- data/lib/aspera/cli/terminal_formatter.rb +65 -0
- data/lib/aspera/cli/transfer_actions.rb +83 -0
- data/lib/aspera/cli/transfer_agent.rb +116 -51
- data/lib/aspera/cli/transfer_progress.rb +9 -9
- data/lib/aspera/cli/vault_manager.rb +57 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +26 -26
- data/lib/aspera/command_line_builder.rb +23 -23
- data/lib/aspera/data_repository.rb +6 -1
- data/lib/aspera/dot_container.rb +5 -5
- data/lib/aspera/environment.rb +39 -13
- data/lib/aspera/exec_spec.rb +13 -0
- data/lib/aspera/faspex_gw.rb +2 -2
- data/lib/aspera/faspex_postproc.rb +4 -3
- data/lib/aspera/graphql.rb +35 -0
- data/lib/aspera/hash_ext.rb +6 -0
- data/lib/aspera/json_rpc/client.rb +62 -0
- data/lib/aspera/json_rpc/version.rb +7 -0
- data/lib/aspera/keychain/base.rb +1 -1
- data/lib/aspera/keychain/encrypted_hash.rb +2 -2
- data/lib/aspera/keychain/factory.rb +4 -4
- data/lib/aspera/keychain/macos_security.rb +4 -4
- data/lib/aspera/link_header.rb +82 -0
- data/lib/aspera/log.rb +23 -5
- data/lib/aspera/markdown.rb +85 -2
- data/lib/aspera/nagios.rb +2 -2
- data/lib/aspera/node_simulator.rb +3 -1
- data/lib/aspera/oauth/base.rb +10 -10
- data/lib/aspera/oauth/boot.rb +43 -0
- data/lib/aspera/oauth/factory.rb +41 -15
- data/lib/aspera/oauth/jwt.rb +4 -4
- data/lib/aspera/oauth/web.rb +4 -4
- data/lib/aspera/oauth.rb +1 -0
- data/lib/aspera/persistency_action_once.rb +10 -10
- data/lib/aspera/persistency_folder.rb +10 -3
- data/lib/aspera/preview/file_types.rb +7 -3
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/preview/utils.rb +1 -1
- data/lib/aspera/products/connect.rb +4 -1
- data/lib/aspera/products/other.rb +1 -1
- data/lib/aspera/products/transferd.rb +3 -3
- data/lib/aspera/proxy_auto_config.rb +10 -9
- data/lib/aspera/rest.rb +18 -40
- data/lib/aspera/rest_error_analyzer.rb +7 -3
- data/lib/aspera/rest_errors_aspera.rb +0 -10
- data/lib/aspera/rest_list.rb +13 -6
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +14 -4
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +2730 -648
- data/lib/aspera/schema/async_tables.yaml +361 -0
- data/lib/aspera/schema/documentation.rb +57 -26
- data/lib/aspera/schema/reader.rb +60 -11
- data/lib/aspera/schema/registry.rb +50 -10
- data/lib/aspera/secret_hider.rb +11 -2
- data/lib/aspera/ssl.rb +4 -4
- data/lib/aspera/sync/conf.schema.yaml +2 -2
- data/lib/aspera/sync/database.rb +14 -10
- data/lib/aspera/sync/operations.rb +15 -18
- data/lib/aspera/temp_file_manager.rb +5 -1
- data/lib/aspera/timer_limiter.rb +1 -1
- data/lib/aspera/transfer/faux_file.rb +24 -11
- data/lib/aspera/transfer/parameters.rb +27 -25
- data/lib/aspera/transfer/result.rb +74 -0
- data/lib/aspera/transfer/resumer.rb +7 -7
- data/lib/aspera/transfer/spec.rb +18 -1
- data/lib/aspera/transfer/spec.schema.yaml +26 -8
- data/lib/aspera/transfer/uri.rb +1 -1
- data/lib/aspera/uri_reader.rb +2 -2
- data/lib/aspera/web_auth.rb +2 -2
- data/lib/aspera/web_server_simple.rb +8 -5
- data/lib/aspera/yaml.rb +2 -1
- data.tar.gz.sig +0 -0
- metadata +45 -16
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/main.rb +0 -480
- data/lib/aspera/json_rpc.rb +0 -51
data/lib/aspera/api/node.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'aspera/link_header'
|
|
3
4
|
require 'aspera/transfer/spec'
|
|
4
5
|
require 'aspera/rest'
|
|
5
6
|
require 'aspera/oauth'
|
|
@@ -78,12 +79,12 @@ module Aspera
|
|
|
78
79
|
attr_reader :api_options
|
|
79
80
|
attr_reader :use_dynamic_key
|
|
80
81
|
|
|
81
|
-
def api_options=(
|
|
82
|
-
Aspera.assert_type(
|
|
83
|
-
|
|
84
|
-
Aspera.assert(@api_options.key?(
|
|
85
|
-
Aspera.assert_type(
|
|
86
|
-
@api_options[
|
|
82
|
+
def api_options=(options_hash)
|
|
83
|
+
Aspera.assert_type(options_hash, Hash)
|
|
84
|
+
options_hash.each do |key, val|
|
|
85
|
+
Aspera.assert(@api_options.key?(key.to_sym)){"unknown api option: #{key} (#{OPTIONS.join(', ')})"}
|
|
86
|
+
Aspera.assert_type(val, TrueClass, FalseClass){"api options value for #{key} should be boolean"}
|
|
87
|
+
@api_options[key.to_sym] = val
|
|
87
88
|
end
|
|
88
89
|
end
|
|
89
90
|
|
|
@@ -105,30 +106,30 @@ module Aspera
|
|
|
105
106
|
end
|
|
106
107
|
|
|
107
108
|
# Adds fields `public_keys` in provided Hash, if dynamic key is set.
|
|
108
|
-
# @param
|
|
109
|
+
# @param parameters [Hash] Hash to add public key to
|
|
109
110
|
# @return [Hash] Hash with public key added
|
|
110
|
-
def add_public_key(
|
|
111
|
+
def add_public_key(parameters)
|
|
111
112
|
if @dynamic_key
|
|
112
113
|
ssh_key = Net::SSH::Buffer.from(:key, @dynamic_key)
|
|
113
114
|
# Get pub key in OpenSSH public key format (authorized_keys)
|
|
114
|
-
|
|
115
|
+
parameters['public_keys'] = [
|
|
115
116
|
ssh_key.read_string,
|
|
116
117
|
Base64.strict_encode64(ssh_key.to_s)
|
|
117
118
|
].join(' ')
|
|
118
119
|
end
|
|
119
|
-
return
|
|
120
|
+
return parameters
|
|
120
121
|
end
|
|
121
122
|
|
|
122
123
|
# Adds fields `ssh_private_key` in provided Hash, if dynamic key is set.
|
|
123
|
-
# @param
|
|
124
|
-
def add_private_key(
|
|
125
|
-
|
|
126
|
-
return
|
|
124
|
+
# @param parameters [Hash] Hash to add private key to
|
|
125
|
+
def add_private_key(parameters)
|
|
126
|
+
parameters['ssh_private_key'] = @dynamic_key.to_pem if @dynamic_key
|
|
127
|
+
return parameters
|
|
127
128
|
end
|
|
128
129
|
|
|
129
130
|
# For access keys: provide expression to match entry in folder
|
|
130
|
-
# @param match_expression one of supported types
|
|
131
|
-
# @return lambda function
|
|
131
|
+
# @param match_expression [Proc, Regexp, String] one of supported types
|
|
132
|
+
# @return [Proc] lambda function
|
|
132
133
|
def file_matcher(match_expression)
|
|
133
134
|
case match_expression
|
|
134
135
|
when Proc then return match_expression
|
|
@@ -146,7 +147,8 @@ module Aspera
|
|
|
146
147
|
end
|
|
147
148
|
|
|
148
149
|
# Split path into folder + filename
|
|
149
|
-
# @
|
|
150
|
+
# @param path [String] file path to split
|
|
151
|
+
# @return [Array(String, String)] containing folder + inside folder/file
|
|
150
152
|
def split_folder(path)
|
|
151
153
|
folder = path.split(PATH_SEPARATOR)
|
|
152
154
|
inside = folder.pop
|
|
@@ -160,7 +162,8 @@ module Aspera
|
|
|
160
162
|
end
|
|
161
163
|
|
|
162
164
|
# Decode node scope into access key and scope
|
|
163
|
-
# @
|
|
165
|
+
# @param scope [String] scope string to decode
|
|
166
|
+
# @return [Hash] decoded scope with access key and scope parts
|
|
164
167
|
def decode_scope(scope)
|
|
165
168
|
items = scope.split(Scope::SEPARATOR, 2)
|
|
166
169
|
Aspera.assert(items.length.eql?(2)){"invalid scope: #{scope}"}
|
|
@@ -174,9 +177,9 @@ module Aspera
|
|
|
174
177
|
# @param private_key [OpenSSL::PKey::RSA] Private key to sign the token
|
|
175
178
|
def bearer_token(access_key:, payload:, private_key:)
|
|
176
179
|
Aspera.assert_type(payload, Hash)
|
|
177
|
-
Aspera.assert(payload.key?('user_id'))
|
|
180
|
+
Aspera.assert(payload.key?('user_id'), 'payload must have user_id')
|
|
178
181
|
Aspera.assert_type(payload['user_id'], String)
|
|
179
|
-
Aspera.assert(!payload['user_id'].empty
|
|
182
|
+
Aspera.assert(!payload['user_id'].empty?, 'user_id must not be empty')
|
|
180
183
|
Aspera.assert_type(private_key, OpenSSL::PKey::RSA)
|
|
181
184
|
# Manage convenience parameters
|
|
182
185
|
expiration_sec = payload['_validity'] || BEARER_TOKEN_VALIDITY_DEFAULT
|
|
@@ -205,7 +208,7 @@ module Aspera
|
|
|
205
208
|
# If username is not provided, use the access key from the token
|
|
206
209
|
if access_key.nil?
|
|
207
210
|
access_key = Node.decode_scope(Node.decode_bearer_token(OAuth::Factory.bearer_token(bearer_auth))['scope'])[:access_key]
|
|
208
|
-
Aspera.assert(!access_key.nil
|
|
211
|
+
Aspera.assert(!access_key.nil?, 'access_key could not be determined from token')
|
|
209
212
|
end
|
|
210
213
|
return {
|
|
211
214
|
HEADER_X_ASPERA_ACCESS_KEY => access_key,
|
|
@@ -240,7 +243,7 @@ module Aspera
|
|
|
240
243
|
return tspec
|
|
241
244
|
end
|
|
242
245
|
|
|
243
|
-
# @
|
|
246
|
+
# @return [Node, nil] a Node Api object or nil if no App defined
|
|
244
247
|
def node_id_to_node(node_id)
|
|
245
248
|
if !@app_info.nil?
|
|
246
249
|
return self if node_id.eql?(@app_info.node_info['id'])
|
|
@@ -337,7 +340,7 @@ module Aspera
|
|
|
337
340
|
# @param top_file_path [String] path of top folder (default = /)
|
|
338
341
|
# @para query [Hash, nil] optional query for `read`
|
|
339
342
|
def process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/', query: nil)
|
|
340
|
-
Aspera.assert(!top_file_path.nil
|
|
343
|
+
Aspera.assert(!top_file_path.nil?, 'top_file_path not set')
|
|
341
344
|
Log.log.debug{"process_folder_tree: node=#{@app_info ? @app_info.node_info['id'] : 'nil'}, file id=#{top_file_id}, path=#{top_file_path}"}
|
|
342
345
|
# Start at top folder
|
|
343
346
|
folders_to_explore = [{id: top_file_id, path: top_file_path}]
|
|
@@ -390,7 +393,7 @@ module Aspera
|
|
|
390
393
|
return NodeFileId.new(self, top_file_id) if path_elements.empty?
|
|
391
394
|
resolve_state = {path: path_elements, consumed: [], result: nil, process_last_link: process_last_link}
|
|
392
395
|
process_folder_tree(method_sym: :process_api_fid, state: resolve_state, top_file_id: top_file_id)
|
|
393
|
-
|
|
396
|
+
Aspera.assert(!resolve_state[:result].nil?, type: ParameterError){"Entry not found: #{resolve_state[:path].first} in /#{resolve_state[:consumed].join(PATH_SEPARATOR)}"}
|
|
394
397
|
Log.log.debug{"resolve_api_fid: #{path} -> #{resolve_state[:result].node_api.base_url} #{resolve_state[:result].file_id}"}
|
|
395
398
|
return resolve_state[:result]
|
|
396
399
|
end
|
|
@@ -405,7 +408,7 @@ module Aspera
|
|
|
405
408
|
# - 'destination' [String]: Destination path (optional)
|
|
406
409
|
def resolve_api_fid_paths(top_file_id, paths)
|
|
407
410
|
Aspera.assert_type(paths, Array)
|
|
408
|
-
Aspera.assert(paths.size.positive
|
|
411
|
+
Aspera.assert(paths.size.positive?, 'paths must not be empty')
|
|
409
412
|
split_sources = paths.map{ |p| Pathname(p['source']).each_filename.to_a}
|
|
410
413
|
root = []
|
|
411
414
|
split_sources.map(&:size).min.times do |i|
|
|
@@ -490,7 +493,7 @@ module Aspera
|
|
|
490
493
|
case auth_params[:type]
|
|
491
494
|
when :basic
|
|
492
495
|
ak_name = auth_params[:username]
|
|
493
|
-
Aspera.assert(auth_params[:password]
|
|
496
|
+
Aspera.assert(auth_params[:password], 'no secret in node object')
|
|
494
497
|
ak_token = Rest.basic_authorization(auth_params[:username], auth_params[:password])
|
|
495
498
|
when :oauth2
|
|
496
499
|
ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
|
|
@@ -551,7 +554,7 @@ module Aspera
|
|
|
551
554
|
def read_with_paging(subpath, query = nil, iteration: nil, **call_args)
|
|
552
555
|
Aspera.assert_type(iteration, Array, NilClass){'iteration'}
|
|
553
556
|
Aspera.assert_type(query, Hash, NilClass){'query'}
|
|
554
|
-
Aspera.assert(!call_args.key?(:query))
|
|
557
|
+
Aspera.assert(!call_args.key?(:query), ':query must not be in call_args (use query parameter)')
|
|
555
558
|
query = {} if query.nil?
|
|
556
559
|
query[:iteration_token] = iteration[0] unless iteration.nil? || iteration[0].nil?
|
|
557
560
|
max = query.delete(RestList::MAX_ITEMS)
|
|
@@ -572,7 +575,7 @@ module Aspera
|
|
|
572
575
|
# Update progress spinner
|
|
573
576
|
RestParameters.instance.spinner_cb.call(item_list.length)
|
|
574
577
|
# Parse Link header according to RFC 8288 to extract next iteration token
|
|
575
|
-
next_url =
|
|
578
|
+
next_url = LinkHeader.parse(http['Link']).find_href(rel: 'next')
|
|
576
579
|
next_iteration_token = nil
|
|
577
580
|
if next_url
|
|
578
581
|
begin
|
|
@@ -629,7 +632,7 @@ module Aspera
|
|
|
629
632
|
private
|
|
630
633
|
|
|
631
634
|
# Method called in loop for each entry for `resolve_api_fid`
|
|
632
|
-
# @return `true` to continue digging, `false` to stop processing: set state[:result] if found
|
|
635
|
+
# @return [Boolean] `true` to continue digging, `false` to stop processing: set state[:result] if found
|
|
633
636
|
def process_api_fid(entry, path, state)
|
|
634
637
|
# Stop digging here if not in right path
|
|
635
638
|
return false unless entry['name'].eql?(state[:path].first)
|
|
@@ -656,7 +659,7 @@ module Aspera
|
|
|
656
659
|
# We found it
|
|
657
660
|
other_node = nil
|
|
658
661
|
other_node = node_id_to_node(entry['target_node_id']) if entry_has_link_information(entry)
|
|
659
|
-
|
|
662
|
+
Aspera.assert(!other_node.nil?, type: Error){'Cannot resolve link'}
|
|
660
663
|
state[:result] = NodeFileId.new(other_node, entry['target_id'])
|
|
661
664
|
else
|
|
662
665
|
# We found it but we do not process the link
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
query ($organization_id: ID!) {
|
|
2
|
+
aoc (organization_id: $organization_id) {
|
|
3
|
+
bssSubscription {
|
|
4
|
+
aocVersion endDate startDate termMonths plan trial termType
|
|
5
|
+
aocOrganizations { id }
|
|
6
|
+
additionalStorageVolumeGb additionalEgressVolumeGb
|
|
7
|
+
term { startDate endDate transferVolumeGb egressVolumeGb storageVolumeGb transferVolumeOffsetGb }
|
|
8
|
+
paygoRate { transferRate storageRate currency }
|
|
9
|
+
aocPlanData {
|
|
10
|
+
tier trial
|
|
11
|
+
workspaces { max }
|
|
12
|
+
users { planAmount max }
|
|
13
|
+
samlIntegration activity sharedInboxes uniqueUrls support watermarking byok
|
|
14
|
+
automation { planAmount, max }
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
query ($organization_id: ID!, $startDate: Date!, $endDate: Date!, $aggregate: TransferUsageAggregateOption!) {
|
|
2
|
+
aoc (organization_id: $organization_id) {
|
|
3
|
+
bssSubscription {
|
|
4
|
+
aocOrganizations { id }
|
|
5
|
+
additionalStorageVolumeGb additionalEgressVolumeGb
|
|
6
|
+
aocPlanData { tier trial }
|
|
7
|
+
term { transferVolumeGb egressVolumeGb storageVolumeGb startDate endDate transferVolumeOffsetGb }
|
|
8
|
+
termMonths
|
|
9
|
+
transferUsages (startDate: $startDate, endDate: $endDate, aggregate: $aggregate) { mbTotal }
|
|
10
|
+
egressUsages (startDate: $startDate, endDate: $endDate, aggregate: $aggregate) { usageMb }
|
|
11
|
+
}
|
|
12
|
+
subscriptionEntitlements {
|
|
13
|
+
id
|
|
14
|
+
transferUsages (startDate: $startDate, endDate: $endDate, aggregate: $aggregate) { mbTotal }
|
|
15
|
+
egressUsages (startDate: $startDate, endDate: $endDate, aggregate: $aggregate) { usageMb }
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
}
|
data/lib/aspera/ascmd.rb
CHANGED
|
@@ -28,18 +28,18 @@ module Aspera
|
|
|
28
28
|
ENUM_START = 1
|
|
29
29
|
|
|
30
30
|
# Description of result structures (see ascmdtypes.h).
|
|
31
|
-
# Base types are big endian
|
|
32
|
-
#
|
|
33
|
-
#
|
|
34
|
-
#
|
|
35
|
-
#
|
|
36
|
-
#
|
|
37
|
-
#
|
|
38
|
-
#
|
|
39
|
-
#
|
|
40
|
-
#
|
|
41
|
-
# :
|
|
42
|
-
# rubocop:disable Layout/FirstHashElementLineBreak
|
|
31
|
+
# Base types are big endian.
|
|
32
|
+
# Keys are type names; the index in each `fields` array equals (TLV numerical type - `ENUM_START`).
|
|
33
|
+
# Decoding always starts at `result`.
|
|
34
|
+
# Some fields have special handling indicated by `special`:
|
|
35
|
+
# - `:field_list` : a Hash or Array of child TLVs
|
|
36
|
+
# - `:list_tlv_list` : a list of TLV sub-lists (Array of Hash)
|
|
37
|
+
# - `:list_tlv_restart` : same but restarted per entry
|
|
38
|
+
# `decode` values:
|
|
39
|
+
# - `:base` : a single scalar value (unpack with `unpack`)
|
|
40
|
+
# - `:buffer_list` : an Array of `{btype, buffer}` entries
|
|
41
|
+
# - `:field_list` : a Hash or Array built from child TLVs
|
|
42
|
+
# rubocop:disable-next Layout/FirstHashElementLineBreak
|
|
43
43
|
TYPES_DESCR = {
|
|
44
44
|
result: {decode: :field_list,
|
|
45
45
|
fields: [{name: :file, is_a: :stat}, {name: :dir, is_a: :stat, special: :list_tlv_list}, {name: :size, is_a: :size}, {name: :error, is_a: :error},
|
|
@@ -73,7 +73,6 @@ module Aspera
|
|
|
73
73
|
zstr: {decode: :base, unpack: 'Z*'},
|
|
74
74
|
blist: {decode: :buffer_list}
|
|
75
75
|
}.freeze
|
|
76
|
-
# rubocop:enable Layout/FirstHashElementLineBreak
|
|
77
76
|
|
|
78
77
|
private_constant :TYPES_DESCR, :ENUM_START, :OPS_ARGS
|
|
79
78
|
|
|
@@ -87,8 +86,8 @@ module Aspera
|
|
|
87
86
|
|
|
88
87
|
# execute an "as" command on a remote server
|
|
89
88
|
# Version 2 allows use of reverse proxy with multiple addresses.
|
|
90
|
-
# @param [Symbol] one of OPERATIONS
|
|
91
|
-
# @param [Array]
|
|
89
|
+
# @param action_sym [Symbol] one of OPERATIONS
|
|
90
|
+
# @param arguments [Array] parameters for "as" command
|
|
92
91
|
# @return [Boolean,Array,Hash] result of command, type depends on command
|
|
93
92
|
def execute_single(action_sym, arguments, version: 1, host: nil)
|
|
94
93
|
arguments = [] if arguments.nil?
|
|
@@ -129,10 +128,10 @@ module Aspera
|
|
|
129
128
|
Log.log.trace1{"execute_single:#{stdin_input}"}
|
|
130
129
|
# execute, get binary output
|
|
131
130
|
byte_buffer = @command_executor.execute(remote_cmd, input: stdin_input).unpack('C*')
|
|
132
|
-
Aspera.assert(!byte_buffer.empty
|
|
131
|
+
Aspera.assert(!byte_buffer.empty?, 'empty answer from server')
|
|
133
132
|
# get hash or table result
|
|
134
133
|
result = self.class.parse(byte_buffer, :result)
|
|
135
|
-
Aspera.assert(byte_buffer.empty
|
|
134
|
+
Aspera.assert(byte_buffer.empty?, 'unparsed bytes remaining')
|
|
136
135
|
# get and delete info,always present in results
|
|
137
136
|
system_info = result[:info]
|
|
138
137
|
result.delete(:info)
|
|
@@ -169,25 +168,28 @@ module Aspera
|
|
|
169
168
|
end
|
|
170
169
|
|
|
171
170
|
class << self
|
|
172
|
-
# Get description of structure's field
|
|
171
|
+
# Get description of structure's field
|
|
172
|
+
# @param struct_name [Symbol] name of the structure type
|
|
173
|
+
# @param typed_buffer [Hash] typed buffer providing the field name via `:btype`
|
|
174
|
+
# @return [Hash] field description
|
|
173
175
|
def field_description(struct_name, typed_buffer)
|
|
174
176
|
result = TYPES_DESCR[struct_name][:fields][typed_buffer[:btype] - ENUM_START]
|
|
175
|
-
|
|
177
|
+
Aspera.assert(!result.nil?){"Unrecognized field for #{struct_name}: #{typed_buffer[:btype]}\n#{typed_buffer[:buffer]}"}
|
|
176
178
|
return result
|
|
177
179
|
end
|
|
178
180
|
|
|
179
181
|
# Decodes the provided buffer as provided type name
|
|
180
|
-
# @return a decoded type: single, Array, or Hash
|
|
182
|
+
# @return [Boolean, Array, Hash] a decoded type: single, Array, or Hash
|
|
181
183
|
def parse(buffer, type_name, indent_level = nil)
|
|
182
184
|
indent_level = (indent_level || -1) + 1
|
|
183
185
|
type_descr = TYPES_DESCR[type_name]
|
|
184
|
-
|
|
186
|
+
Aspera.assert(!type_descr.nil?){"Unexpected type #{type_name}"}
|
|
185
187
|
Log.log.trace1{"#{' .' * indent_level}parse:#{type_name}:#{type_descr[:decode]}:#{buffer[0, 16]}...".red}
|
|
186
188
|
result = nil
|
|
187
189
|
case type_descr[:decode]
|
|
188
190
|
when :base
|
|
189
191
|
num_bytes = type_name.eql?(:zstr) ? buffer.length : type_descr[:size]
|
|
190
|
-
Aspera.assert(buffer.length >= num_bytes
|
|
192
|
+
Aspera.assert(buffer.length >= num_bytes, 'not enough bytes')
|
|
191
193
|
byte_array = buffer.shift(num_bytes)
|
|
192
194
|
byte_array = [byte_array] unless byte_array.is_a?(Array)
|
|
193
195
|
result = byte_array.pack('C*').unpack1(type_descr[:unpack])
|
|
@@ -200,7 +202,7 @@ module Aspera
|
|
|
200
202
|
until buffer.empty?
|
|
201
203
|
btype = parse(buffer, :int8, indent_level)
|
|
202
204
|
length = parse(buffer, :int32, indent_level)
|
|
203
|
-
Aspera.assert(buffer.length >= length
|
|
205
|
+
Aspera.assert(buffer.length >= length, 'not enough bytes')
|
|
204
206
|
value = buffer.shift(length)
|
|
205
207
|
result.push({btype: btype, buffer: value})
|
|
206
208
|
Log.log.trace1{"#{' .' * indent_level}:buffer_list[#{result.length - 1}] #{result.last}"}
|
|
@@ -27,6 +27,10 @@ module Aspera
|
|
|
27
27
|
# It is used by object : AgentDirect to find necessary resources
|
|
28
28
|
# By default it takes the first Aspera product found
|
|
29
29
|
# The user can specify `ascp` location by calling: `sdk_folder=` method
|
|
30
|
+
#
|
|
31
|
+
# @!method self.instance
|
|
32
|
+
# Returns the singleton instance of Installation
|
|
33
|
+
# @return [Installation] the singleton instance
|
|
30
34
|
class Installation
|
|
31
35
|
include Singleton
|
|
32
36
|
|
|
@@ -57,16 +61,16 @@ module Aspera
|
|
|
57
61
|
# - "product:FIRST" to use ascp from first found product
|
|
58
62
|
def sdk_folder=(ascp_location)
|
|
59
63
|
Aspera.assert_type(ascp_location, String){'ascp_location'}
|
|
60
|
-
Aspera.assert(!ascp_location.empty
|
|
64
|
+
Aspera.assert(!ascp_location.empty?, 'ascp location cannot be empty: check your config file')
|
|
61
65
|
folder =
|
|
62
66
|
if ascp_location.start_with?(USE_PRODUCT_PREFIX)
|
|
63
67
|
product_name = ascp_location.delete_prefix(USE_PRODUCT_PREFIX)
|
|
64
68
|
if product_name.eql?(FIRST_FOUND)
|
|
65
69
|
pl = installed_products.first
|
|
66
|
-
|
|
70
|
+
Aspera.assert(!pl.nil?){"No Aspera transfer module or SDK found.\nRefer to the manual or install SDK with command:\nascli conf transferd install"}
|
|
67
71
|
else
|
|
68
72
|
pl = installed_products.find{ |i| i[:name].eql?(product_name)}
|
|
69
|
-
|
|
73
|
+
Aspera.assert(!pl.nil?){"No such product installed: #{product_name}"}
|
|
70
74
|
end
|
|
71
75
|
File.dirname(pl[:ascp_path])
|
|
72
76
|
else
|
|
@@ -84,14 +88,14 @@ module Aspera
|
|
|
84
88
|
# @return [Hash] with key = file name (String), and value = path to file
|
|
85
89
|
def file_paths
|
|
86
90
|
return SDK_FILES.to_h do |v|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
91
|
+
[v.to_s, begin
|
|
92
|
+
path(v)
|
|
93
|
+
rescue Errno::ENOENT => e
|
|
94
|
+
e.message.gsub(/.*assertion failed: /, '').gsub(/\): .*/, ')')
|
|
95
|
+
rescue => e
|
|
96
|
+
e.message
|
|
97
|
+
end]
|
|
98
|
+
end
|
|
95
99
|
end
|
|
96
100
|
|
|
97
101
|
# TODO: if using another product than SDK, should use files from there
|
|
@@ -102,18 +106,18 @@ module Aspera
|
|
|
102
106
|
|
|
103
107
|
# Get path of one resource file of currently activated product
|
|
104
108
|
# keys and certs are generated locally... (they are well known values, arch. independent)
|
|
105
|
-
# @param
|
|
109
|
+
# @param file_type [Symbol] key of the resource file
|
|
106
110
|
# @return [String, nil] Full path to the resource file or nil if not found
|
|
107
|
-
def path(
|
|
111
|
+
def path(file_type)
|
|
108
112
|
file_is_required = true
|
|
109
|
-
case
|
|
113
|
+
case file_type
|
|
110
114
|
when *EXE_FILES
|
|
111
|
-
file_is_required =
|
|
115
|
+
file_is_required = file_type.eql?(:ascp)
|
|
112
116
|
file = Products::Transferd.transferd_path
|
|
113
|
-
file = File.join(File.dirname(file), Environment.instance.exe_file(
|
|
117
|
+
file = File.join(File.dirname(file), Environment.instance.exe_file(file_type.to_s)) unless file_type.eql?(:transferd)
|
|
114
118
|
when :ssh_private_dsa, :ssh_private_rsa
|
|
115
119
|
# assume last 3 letters are type
|
|
116
|
-
type =
|
|
120
|
+
type = file_type.to_s[-3..].to_sym
|
|
117
121
|
file = check_or_create_sdk_file("aspera_bypass_#{type}.pem"){DataRepository.instance.item(type)}
|
|
118
122
|
when :aspera_license
|
|
119
123
|
file = check_or_create_sdk_file('aspera-license'){DataRepository.instance.item(:license)}
|
|
@@ -130,22 +134,23 @@ module Aspera
|
|
|
130
134
|
check_or_create_sdk_file('aspera_fallback_cert_private_key.pem', force: true){private_key.to_pem}
|
|
131
135
|
check_or_create_sdk_file('aspera_fallback_cert.pem', force: true){cert.to_pem}
|
|
132
136
|
end
|
|
133
|
-
file =
|
|
134
|
-
else Aspera.error_unexpected_value(
|
|
137
|
+
file = file_type.eql?(:fallback_certificate) ? file_cert : file_key
|
|
138
|
+
else Aspera.error_unexpected_value(file_type)
|
|
135
139
|
end
|
|
136
140
|
return unless file_is_required || File.exist?(file)
|
|
137
|
-
Aspera.assert(File.exist?(file), type: Errno::ENOENT){"#{
|
|
141
|
+
Aspera.assert(File.exist?(file), type: Errno::ENOENT){"#{file_type} not found (#{file})"}
|
|
138
142
|
return file
|
|
139
143
|
end
|
|
140
144
|
|
|
141
145
|
# default bypass key phrase
|
|
142
|
-
# @return [String]
|
|
146
|
+
# @return [String] UUID used as bypass key phrase for SSH certificate
|
|
143
147
|
def ssh_cert_uuid
|
|
144
148
|
return DataRepository.instance.item(:uuid)
|
|
145
149
|
end
|
|
146
150
|
|
|
147
151
|
# get paths of SSH keys to use for ascp client
|
|
148
|
-
# @param types [Symbol] types to use
|
|
152
|
+
# @param types [Symbol] types to use (one of CLIENT_SSH_KEY_OPTIONS)
|
|
153
|
+
# @return [Array<String>] list of private key file paths
|
|
149
154
|
def aspera_token_ssh_key_paths(types)
|
|
150
155
|
Aspera.assert_values(types, CLIENT_SSH_KEY_OPTIONS)
|
|
151
156
|
return case types
|
|
@@ -166,13 +171,11 @@ module Aspera
|
|
|
166
171
|
Aspera.assert_type(exe_path, String)
|
|
167
172
|
Aspera.assert_type(vers_arg, String)
|
|
168
173
|
return unless File.exist?(exe_path)
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
raise "An error occurred when testing #{exe_path}: #{cmd_out}" unless $CHILD_STATUS == 0
|
|
174
|
+
cmd_out, _err, status = Environment.secure_execute(exe_path, vers_arg, mode: :capture, exception: false)
|
|
175
|
+
raise "An error occurred when testing #{exe_path}: #{cmd_out}" unless status.success?
|
|
172
176
|
# get version from ascp, only after full extract, as windows requires DLLs (SSL/TLS/etc...)
|
|
173
177
|
m = cmd_out.match(/ version ([0-9.]+)/)
|
|
174
|
-
|
|
175
|
-
return exe_version
|
|
178
|
+
m.nil? ? nil : m[1].gsub(/\.$/, '')
|
|
176
179
|
end
|
|
177
180
|
|
|
178
181
|
# Extract some stings from ascp logs
|
|
@@ -230,15 +233,15 @@ module Aspera
|
|
|
230
233
|
return ascp_data
|
|
231
234
|
end
|
|
232
235
|
|
|
233
|
-
# @return the url for download of SDK archive for the given platform and version
|
|
236
|
+
# @return [String] the url for download of SDK archive for the given platform and version
|
|
234
237
|
def sdk_url_for_platform(platform: nil, version: nil)
|
|
235
238
|
all_locations = sdk_locations
|
|
236
239
|
platform = Environment.instance.architecture if platform.nil?
|
|
237
240
|
locations = all_locations.select{ |l| l['platform'].eql?(platform)}
|
|
238
|
-
|
|
241
|
+
Aspera.assert(!locations.empty?){"No SDK for platform: #{platform}, available: #{all_locations.map{ |i| i['platform']}.uniq}"}
|
|
239
242
|
version = locations.max_by{ |entry| Gem::Version.new(entry['version'])}['version'] if version.nil?
|
|
240
243
|
info = locations.select{ |entry| entry['version'].eql?(version)}
|
|
241
|
-
|
|
244
|
+
Aspera.assert(!info.empty?){"No such version: #{version} for #{platform}"}
|
|
242
245
|
return info.first['url']
|
|
243
246
|
end
|
|
244
247
|
|
|
@@ -247,7 +250,7 @@ module Aspera
|
|
|
247
250
|
# @yieldparam entry_stream [IO, Gem::Package::TarReader::Entry] Data stream
|
|
248
251
|
# @yieldparam link_target [String, nil] Link target if symlink, nil otherwise
|
|
249
252
|
def extract_archive_files(sdk_archive_path)
|
|
250
|
-
Aspera.assert(block_given
|
|
253
|
+
Aspera.assert(block_given?, 'missing block')
|
|
251
254
|
case sdk_archive_path
|
|
252
255
|
# Windows and Mac use zip
|
|
253
256
|
when /\.zip$/
|
|
@@ -278,15 +281,14 @@ module Aspera
|
|
|
278
281
|
end
|
|
279
282
|
end
|
|
280
283
|
|
|
281
|
-
#
|
|
284
|
+
# Downloads and extracts the SDK archive for the current platform into +folder+
|
|
282
285
|
# @param folder [String] Destination folder path
|
|
283
286
|
# @param url [nil, String] URL to SDK archive, if nil: default url for version
|
|
284
287
|
# @param version [nil, String] Specific version, if nil: latest version
|
|
285
288
|
# @param backup [Boolean] If destination folder exists, then rename
|
|
286
289
|
# @yieldparam entry_name [String] File path from archive
|
|
287
290
|
# @yieldreturn [String, nil] Destination sub folder (end with /) or file, or nil to not extract
|
|
288
|
-
|
|
289
|
-
def install_sdk(folder:, url: nil, version: nil, backup: true)
|
|
291
|
+
def download_sdk(folder:, url: nil, version: nil, backup: true)
|
|
290
292
|
url ||= sdk_url_for_platform(version: version)
|
|
291
293
|
# Rename old install
|
|
292
294
|
if backup && Dir.exist?(folder) && !Dir.empty?(folder)
|
|
@@ -368,12 +370,13 @@ module Aspera
|
|
|
368
370
|
end
|
|
369
371
|
end
|
|
370
372
|
|
|
371
|
-
#
|
|
372
|
-
# @param url
|
|
373
|
-
# @param version [nil, String]
|
|
374
|
-
|
|
373
|
+
# Downloads, installs and configures the SDK, returns name, version and install folder
|
|
374
|
+
# @param url [nil, String] URL to SDK archive, if nil: default url for version
|
|
375
|
+
# @param version [nil, String] Specific version, if nil: latest version
|
|
376
|
+
# @return [Array<String>] sdk name, sdk version, install folder
|
|
377
|
+
def install_sdk(url: nil, version: nil)
|
|
375
378
|
folder = Products::Transferd.sdk_directory
|
|
376
|
-
|
|
379
|
+
download_sdk(folder: folder, url: url, version: version)
|
|
377
380
|
# Ensure necessary files are there, or generate them, restrict file access on SDK executables
|
|
378
381
|
SDK_FILES.each do |file_id_sym|
|
|
379
382
|
file_path = path(file_id_sym)
|
|
@@ -429,7 +432,7 @@ module Aspera
|
|
|
429
432
|
|
|
430
433
|
public
|
|
431
434
|
|
|
432
|
-
# @return the list of installed products in format of product_locations_on_current_os
|
|
435
|
+
# @return [Array<Hash>] the list of installed products in format of product_locations_on_current_os
|
|
433
436
|
def installed_products
|
|
434
437
|
return @found_products unless @found_products.nil?
|
|
435
438
|
# :expected M app name is taken from the manifest if present, else defaults to this value
|
|
@@ -12,7 +12,7 @@ module Aspera
|
|
|
12
12
|
# https://www.ibm.com/support/pages/error-code-reference-tables
|
|
13
13
|
# mgmtmess.c : as_mgmt_err_is_retryable
|
|
14
14
|
# Note that the fact that an error is retry-able is not internally defined by protocol, it's client-side responsibility
|
|
15
|
-
# rubocop:disable Layout/FirstHashElementLineBreak
|
|
15
|
+
# rubocop:disable-next Layout/FirstHashElementLineBreak
|
|
16
16
|
ERRORS = {
|
|
17
17
|
# id retry-able mnemo/code message additional info
|
|
18
18
|
0 => {r: false, c: 'UNKNOWN', m: 'unknown', a: 'unknown'},
|
|
@@ -93,7 +93,6 @@ module Aspera
|
|
|
93
93
|
66 => {r: false, c: 'PEER_REQUIRES_FIPS', m: 'Peer rejects cipher due to FIPS mode enabled on peer',
|
|
94
94
|
a: 'Peer rejects cipher due to FIPS mode enabled on peer'}
|
|
95
95
|
}.freeze
|
|
96
|
-
# rubocop:enable Layout/FirstHashElementLineBreak
|
|
97
96
|
# cspell: disable
|
|
98
97
|
OPERATIONS = %w[
|
|
99
98
|
NOP
|
|
@@ -307,7 +306,7 @@ module Aspera
|
|
|
307
306
|
def field_snake_to_native(name)
|
|
308
307
|
field = name.delete('_')
|
|
309
308
|
result = PARAMETERS.find{ |w| w.casecmp?(field)}
|
|
310
|
-
|
|
309
|
+
Aspera.assert(!result.nil?){"No such field: #{name}"}
|
|
311
310
|
result
|
|
312
311
|
end
|
|
313
312
|
|