aspera-cli 4.27.2 → 4.27.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +50 -0
- data/bin/ascli +2 -1
- data/docs/README.md +804 -746
- data/lib/aspera/agent/connect.rb +6 -4
- data/lib/aspera/agent/desktop.rb +2 -2
- data/lib/aspera/agent/direct.rb +3 -1
- data/lib/aspera/agent/node.rb +3 -3
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +14 -12
- data/lib/aspera/api/ats.rb +1 -1
- data/lib/aspera/api/cos_node.rb +2 -2
- data/lib/aspera/api/faspex.rb +9 -7
- data/lib/aspera/api/httpgw.rb +37 -33
- data/lib/aspera/api/node.rb +38 -33
- data/lib/aspera/ascmd.rb +3 -1
- data/lib/aspera/ascp/installation.rb +62 -27
- data/lib/aspera/ascp/management.rb +1 -0
- data/lib/aspera/assert.rb +4 -0
- data/lib/aspera/cli/ascp_actions.rb +20 -41
- data/lib/aspera/cli/async_transfer_store.rb +2 -2
- data/lib/aspera/cli/bootstrapper.rb +11 -15
- data/lib/aspera/cli/command_line.rb +252 -0
- data/lib/aspera/cli/command_registry.rb +149 -33
- data/lib/aspera/cli/command_spec.rb +103 -14
- data/lib/aspera/cli/completion/ascli.bash +12 -0
- data/lib/aspera/cli/completion/ascli.fish +16 -0
- data/lib/aspera/cli/completion/ascli.zsh +19 -0
- data/lib/aspera/cli/context.rb +3 -0
- data/lib/aspera/cli/deprecation.rb +37 -0
- data/lib/aspera/cli/extended_value.rb +2 -0
- data/lib/aspera/cli/formatter.rb +87 -75
- data/lib/aspera/cli/gem_checker.rb +1 -1
- data/lib/aspera/cli/hints.rb +7 -6
- data/lib/aspera/cli/http.rb +21 -21
- data/lib/aspera/cli/info.rb +3 -0
- data/lib/aspera/cli/mcp_tool.rb +47 -83
- data/lib/aspera/cli/option_declarator.rb +33 -42
- data/lib/aspera/cli/option_registry.rb +69 -0
- data/lib/aspera/cli/option_types.rb +103 -0
- data/lib/aspera/cli/option_value.rb +281 -0
- data/lib/aspera/cli/options.schema.yaml +38 -5
- data/lib/aspera/cli/parser.rb +307 -848
- data/lib/aspera/cli/plugins/alee.rb +7 -4
- data/lib/aspera/cli/plugins/aoc.rb +430 -380
- data/lib/aspera/cli/plugins/ats.rb +58 -73
- data/lib/aspera/cli/plugins/base.rb +190 -240
- data/lib/aspera/cli/plugins/basic_auth.rb +2 -10
- data/lib/aspera/cli/plugins/config.rb +244 -178
- data/lib/aspera/cli/plugins/console.rb +102 -38
- data/lib/aspera/cli/plugins/cos.rb +6 -23
- data/lib/aspera/cli/plugins/factory.rb +3 -0
- data/lib/aspera/cli/plugins/faspex5.rb +176 -173
- data/lib/aspera/cli/plugins/faspio.rb +5 -10
- data/lib/aspera/cli/plugins/httpgw.rb +8 -11
- data/lib/aspera/cli/plugins/mcp.rb +20 -55
- data/lib/aspera/cli/plugins/node.rb +277 -311
- data/lib/aspera/cli/plugins/orchestrator.rb +90 -77
- data/lib/aspera/cli/plugins/preview.rb +79 -90
- data/lib/aspera/cli/plugins/server.rb +76 -50
- data/lib/aspera/cli/plugins/shares.rb +68 -116
- data/lib/aspera/cli/preset_actions.rb +17 -10
- data/lib/aspera/cli/preset_manager.rb +12 -2
- data/lib/aspera/cli/prompt.rb +35 -0
- data/lib/aspera/cli/result.rb +13 -18
- data/lib/aspera/cli/runner.rb +31 -54
- data/lib/aspera/cli/special_values.rb +5 -0
- data/lib/aspera/cli/sync_actions.rb +41 -37
- data/lib/aspera/cli/terminal_formatter.rb +9 -3
- data/lib/aspera/cli/transfer_actions.rb +0 -6
- data/lib/aspera/cli/transfer_agent.rb +29 -35
- data/lib/aspera/cli/vault_manager.rb +0 -17
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +4 -2
- data/lib/aspera/coverage.rb +1 -0
- data/lib/aspera/environment.rb +6 -0
- data/lib/aspera/faspex_gw.rb +2 -1
- data/lib/aspera/faspex_postproc.rb +1 -0
- data/lib/aspera/graphql.rb +5 -5
- data/lib/aspera/json_rpc/client.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +1 -1
- data/lib/aspera/keychain/one_password_api.rb +1 -1
- data/lib/aspera/link_header.rb +2 -2
- data/lib/aspera/log.rb +22 -25
- data/lib/aspera/markdown.rb +2 -0
- data/lib/aspera/mime.rb +25 -0
- data/lib/aspera/node_simulator.rb +1 -0
- data/lib/aspera/oauth/base.rb +35 -25
- data/lib/aspera/oauth/factory.rb +1 -0
- data/lib/aspera/oauth/generic.rb +1 -1
- data/lib/aspera/oauth/jwt.rb +1 -1
- data/lib/aspera/oauth/web.rb +9 -8
- data/lib/aspera/preview/file_types.rb +4 -4
- data/lib/aspera/preview/generator.rb +7 -0
- data/lib/aspera/preview/options.rb +4 -4
- data/lib/aspera/preview/terminal.rb +4 -3
- data/lib/aspera/preview/utils.rb +9 -6
- data/lib/aspera/products/connect.rb +1 -1
- data/lib/aspera/rainbow.rb +7 -0
- data/lib/aspera/rest/aspera_errors.rb +60 -0
- data/lib/aspera/rest/call_error.rb +27 -0
- data/lib/aspera/rest/client.rb +514 -0
- data/lib/aspera/rest/error_analyzer.rb +113 -0
- data/lib/aspera/rest/list.rb +143 -0
- data/lib/aspera/rest/parameters.rb +55 -0
- data/lib/aspera/rest/util.rb +176 -0
- data/lib/aspera/rest.rb +7 -621
- data/lib/aspera/schema/IBM Aspera Console-enhanced.yaml +1125 -0
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2395 -0
- data/lib/aspera/schema/documentation.rb +13 -3
- data/lib/aspera/schema/registry.rb +18 -1
- data/lib/aspera/schema/validator.rb +92 -0
- data/lib/aspera/secret_hider.rb +36 -25
- data/lib/aspera/string_ext.rb +15 -0
- data/lib/aspera/temp_file_manager.rb +6 -5
- data/lib/aspera/transfer/parameters.rb +2 -0
- data/lib/aspera/transfer/spec.rb +1 -0
- data/lib/aspera/uri_reader.rb +11 -11
- data/lib/aspera/web_auth/index.html +147 -0
- data/lib/aspera/web_auth/server.rb +81 -0
- data.tar.gz.sig +0 -0
- metadata +39 -7
- metadata.gz.sig +0 -0
- data/lib/aspera/colors.rb +0 -79
- data/lib/aspera/rest_call_error.rb +0 -25
- data/lib/aspera/rest_error_analyzer.rb +0 -111
- data/lib/aspera/rest_errors_aspera.rb +0 -58
- data/lib/aspera/rest_list.rb +0 -136
- data/lib/aspera/web_auth.rb +0 -211
data/lib/aspera/api/node.rb
CHANGED
|
@@ -12,6 +12,8 @@ require 'openssl'
|
|
|
12
12
|
require 'pathname'
|
|
13
13
|
require 'zlib'
|
|
14
14
|
require 'net/ssh/buffer'
|
|
15
|
+
require 'aspera/rainbow'
|
|
16
|
+
using Rainbow
|
|
15
17
|
|
|
16
18
|
module Aspera
|
|
17
19
|
module Api
|
|
@@ -33,7 +35,7 @@ module Aspera
|
|
|
33
35
|
|
|
34
36
|
# Aspera Node API client
|
|
35
37
|
# with gen4 extensions (access keys)
|
|
36
|
-
class Node < Rest
|
|
38
|
+
class Node < Rest::Client
|
|
37
39
|
# Format of node scope : node.<access key>:<scope>
|
|
38
40
|
module Scope
|
|
39
41
|
# Node sub-scopes
|
|
@@ -42,6 +44,26 @@ module Aspera
|
|
|
42
44
|
# Separator between node.AK and user:all
|
|
43
45
|
SEPARATOR = ':'
|
|
44
46
|
NODE_PREFIX = 'node.'
|
|
47
|
+
|
|
48
|
+
extend self
|
|
49
|
+
|
|
50
|
+
# Encode node API scope
|
|
51
|
+
# @param access_key [String] Access Key
|
|
52
|
+
# @param scope [String] internal scope
|
|
53
|
+
# @return [String] full node scope
|
|
54
|
+
def join(access_key:, scope:)
|
|
55
|
+
["#{NODE_PREFIX}#{access_key}", scope].join(SEPARATOR)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Decode node scope into access key and scope
|
|
59
|
+
# @param scope [String] scope string to split
|
|
60
|
+
# @return [Hash] decoded scope with access key and scope parts
|
|
61
|
+
def split(scope)
|
|
62
|
+
items = scope.split(Scope::SEPARATOR, 2)
|
|
63
|
+
Aspera.assert(items.length.eql?(2)) { "invalid scope: #{scope}" }
|
|
64
|
+
Aspera.assert(items[0].start_with?(NODE_PREFIX)) { "invalid scope: #{scope}" }
|
|
65
|
+
{access_key: items[0].delete_prefix(NODE_PREFIX), scope: items[1]}
|
|
66
|
+
end
|
|
45
67
|
end
|
|
46
68
|
# Delimiter in decoded node token
|
|
47
69
|
SIGNATURE_DELIMITER = '==SIGNATURE=='
|
|
@@ -54,10 +76,10 @@ module Aspera
|
|
|
54
76
|
HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
|
|
55
77
|
HEADER_X_CACHE_CONTROL = 'X-Aspera-Cache-Control'
|
|
56
78
|
HEADER_X_NEXT_ITER_TOKEN = 'X-Aspera-Next-Iteration-Token'
|
|
79
|
+
HEADER_X_TOTAL_COUNT = 'X-Total-Count'
|
|
57
80
|
HEADER_ACCEPT_VERSION = 'Accept-Version'
|
|
58
81
|
# / in cloud
|
|
59
82
|
PATH_SEPARATOR = '/'
|
|
60
|
-
HEADER_X_TOTAL_COUNT = 'X-Total-Count'
|
|
61
83
|
|
|
62
84
|
# Register node special token decoder
|
|
63
85
|
OAuth::Factory.instance.register_decoder(lambda { |token| Node.decode_bearer_token(token) })
|
|
@@ -133,22 +155,6 @@ module Aspera
|
|
|
133
155
|
[folder.join(PATH_SEPARATOR), inside]
|
|
134
156
|
end
|
|
135
157
|
|
|
136
|
-
# Node API scopes
|
|
137
|
-
# @return [String] node scope
|
|
138
|
-
def token_scope(access_key, scope)
|
|
139
|
-
return [Scope::NODE_PREFIX, access_key, Scope::SEPARATOR, scope].join('')
|
|
140
|
-
end
|
|
141
|
-
|
|
142
|
-
# Decode node scope into access key and scope
|
|
143
|
-
# @param scope [String] scope string to decode
|
|
144
|
-
# @return [Hash] decoded scope with access key and scope parts
|
|
145
|
-
def decode_scope(scope)
|
|
146
|
-
items = scope.split(Scope::SEPARATOR, 2)
|
|
147
|
-
Aspera.assert(items.length.eql?(2)) { "invalid scope: #{scope}" }
|
|
148
|
-
Aspera.assert(items[0].start_with?(Scope::NODE_PREFIX)) { "invalid scope: #{scope}" }
|
|
149
|
-
return {access_key: items[0].delete_prefix(Scope::NODE_PREFIX), scope: items[1]}
|
|
150
|
-
end
|
|
151
|
-
|
|
152
158
|
# Create an Aspera Node bearer token
|
|
153
159
|
# @param access_key [String] Access key identifier
|
|
154
160
|
# @param payload [String] JSON payload to be included in the token
|
|
@@ -162,9 +168,8 @@ module Aspera
|
|
|
162
168
|
# Manage convenience parameters
|
|
163
169
|
expiration_sec = payload['_validity'] || BEARER_TOKEN_VALIDITY_DEFAULT
|
|
164
170
|
payload.delete('_validity')
|
|
165
|
-
scope = payload
|
|
166
|
-
payload.
|
|
167
|
-
payload['scope'] ||= token_scope(access_key, scope)
|
|
171
|
+
scope = payload.delete('_scope') || Scope::USER
|
|
172
|
+
payload['scope'] ||= Scope.join(access_key: access_key, scope: scope)
|
|
168
173
|
payload['auth_type'] ||= 'access_key'
|
|
169
174
|
payload['expires_at'] ||= (Time.now + expiration_sec).utc.strftime('%FT%TZ')
|
|
170
175
|
payload_json = JSON.generate(payload)
|
|
@@ -185,7 +190,7 @@ module Aspera
|
|
|
185
190
|
def bearer_headers(bearer_auth, access_key: nil)
|
|
186
191
|
# If username is not provided, use the access key from the token
|
|
187
192
|
if access_key.nil?
|
|
188
|
-
access_key = Node.
|
|
193
|
+
access_key = Node::Scope.split(Node.decode_bearer_token(OAuth::Factory.bearer_token(bearer_auth))['scope'])[:access_key]
|
|
189
194
|
Aspera.assert(!access_key.nil?, 'access_key could not be determined from token')
|
|
190
195
|
end
|
|
191
196
|
return {
|
|
@@ -233,7 +238,7 @@ module Aspera
|
|
|
233
238
|
end
|
|
234
239
|
Log.log.warn { "Cannot resolve link with node id #{node_id}, no resolver" }
|
|
235
240
|
return
|
|
236
|
-
rescue
|
|
241
|
+
rescue Rest::CallError => e
|
|
237
242
|
Log.log.warn { "Cannot resolve link with node id #{node_id}: #{e.message}" }
|
|
238
243
|
return
|
|
239
244
|
end
|
|
@@ -287,7 +292,7 @@ module Aspera
|
|
|
287
292
|
query['page'] = 0
|
|
288
293
|
end
|
|
289
294
|
loop do
|
|
290
|
-
|
|
295
|
+
Rest::Parameters.instance.spinner_cb.call(folder_items.count)
|
|
291
296
|
data, http = read("files/#{file_id}/files", query, headers: headers, ret: :both)
|
|
292
297
|
folder_items.concat(data)
|
|
293
298
|
if use_v4
|
|
@@ -304,7 +309,7 @@ module Aspera
|
|
|
304
309
|
Log.log.warn { "#{path || file_id}: #{e.class} #{e.message}" }
|
|
305
310
|
Log.log.debug { (['Backtrace:'] + e.backtrace).join("\n") }
|
|
306
311
|
ensure
|
|
307
|
-
|
|
312
|
+
Rest::Parameters.instance.spinner_cb.call(folder_items.count, action: :success)
|
|
308
313
|
end
|
|
309
314
|
folder_items
|
|
310
315
|
end
|
|
@@ -326,7 +331,7 @@ module Aspera
|
|
|
326
331
|
until folders_to_explore.empty?
|
|
327
332
|
# Consume first in job list
|
|
328
333
|
current_item = folders_to_explore.shift
|
|
329
|
-
Log.log.debug { "Exploring #{current_item[:path]}".
|
|
334
|
+
Log.log.debug { "Exploring #{current_item[:path]}".bg(:green) }
|
|
330
335
|
# Get folder content
|
|
331
336
|
folder_contents = read_folder_content(current_item[:id], query, exception: false, path: current_item[:path])
|
|
332
337
|
Log.dump(:folder_contents, folder_contents)
|
|
@@ -527,7 +532,7 @@ module Aspera
|
|
|
527
532
|
|
|
528
533
|
# Executes `GET` call in loop using `iteration_token` (`/ops/transfers`)
|
|
529
534
|
# @param iteration [Array] a single element array with the iteration token or nil
|
|
530
|
-
# @param call_args [Hash] additional arguments to pass to `Rest
|
|
535
|
+
# @param call_args [Hash] additional arguments to pass to `Rest::Client#call`
|
|
531
536
|
# @return [Array] list of items returned by the API call
|
|
532
537
|
def read_with_paging(subpath, query = nil, iteration: nil, **call_args)
|
|
533
538
|
Aspera.assert_type(iteration, Array, NilClass) { 'iteration' }
|
|
@@ -535,7 +540,7 @@ module Aspera
|
|
|
535
540
|
Aspera.assert(!call_args.key?(:query), ':query must not be in call_args (use query parameter)')
|
|
536
541
|
query = {} if query.nil?
|
|
537
542
|
query[:iteration_token] = iteration[0] unless iteration.nil? || iteration[0].nil?
|
|
538
|
-
max = query.delete(
|
|
543
|
+
max = query.delete(Rest::List::MAX_ITEMS)
|
|
539
544
|
# Return empty list immediately if max is 0
|
|
540
545
|
return [] if max&.zero?
|
|
541
546
|
item_list = []
|
|
@@ -551,7 +556,7 @@ module Aspera
|
|
|
551
556
|
break
|
|
552
557
|
end
|
|
553
558
|
# Update progress spinner
|
|
554
|
-
|
|
559
|
+
Rest::Parameters.instance.spinner_cb.call(item_list.length)
|
|
555
560
|
# Parse Link header according to RFC 8288 to extract next iteration token
|
|
556
561
|
next_url = LinkHeader.parse(http['Link']).find_href(rel: 'next')
|
|
557
562
|
next_iteration_token = nil
|
|
@@ -572,7 +577,7 @@ module Aspera
|
|
|
572
577
|
query[:iteration_token] = next_iteration_token
|
|
573
578
|
end
|
|
574
579
|
# Signal completion
|
|
575
|
-
|
|
580
|
+
Rest::Parameters.instance.spinner_cb.call(action: :success)
|
|
576
581
|
# save iteration token if needed
|
|
577
582
|
iteration[0] = query[:iteration_token] unless iteration.nil?
|
|
578
583
|
item_list
|
|
@@ -582,7 +587,7 @@ module Aspera
|
|
|
582
587
|
#
|
|
583
588
|
# @param subpath [String] API Path
|
|
584
589
|
# @param query [Hash, nil] Optional query parameters for the API request
|
|
585
|
-
# @param kwargs [Hash] Other parameters for Rest
|
|
590
|
+
# @param kwargs [Hash] Other parameters for Rest::Client#read
|
|
586
591
|
#
|
|
587
592
|
# @return [Array<Hash>] List of folder entries (files, folders, links)
|
|
588
593
|
def read_with_pages(subpath, query = nil, **kwargs)
|
|
@@ -592,7 +597,7 @@ module Aspera
|
|
|
592
597
|
query['page'] ||= 1
|
|
593
598
|
suffix = nil
|
|
594
599
|
loop do
|
|
595
|
-
|
|
600
|
+
Rest::Parameters.instance.spinner_cb.call("#{items.count}#{suffix}")
|
|
596
601
|
data, http = read(subpath, query, **kwargs, ret: :both)
|
|
597
602
|
items.concat(data)
|
|
598
603
|
break if data.length < query['per_page']
|
|
@@ -603,7 +608,7 @@ module Aspera
|
|
|
603
608
|
Log.log.warn { "#{e.class} #{e.message}" }
|
|
604
609
|
Log.log.debug { (['Backtrace:'] + e.backtrace).join("\n") }
|
|
605
610
|
ensure
|
|
606
|
-
|
|
611
|
+
Rest::Parameters.instance.spinner_cb.call(items.count, action: :success)
|
|
607
612
|
return items # rubocop:disable Lint/EnsureReturn
|
|
608
613
|
end
|
|
609
614
|
|
data/lib/aspera/ascmd.rb
CHANGED
|
@@ -3,9 +3,11 @@
|
|
|
3
3
|
# cspell:ignore ascmd smode errstr zstr zmode zuid zgid zctime zatime zmtime fcount dcount btype blist codeset lc_ctype ascmdtypes
|
|
4
4
|
require 'aspera/log'
|
|
5
5
|
require 'aspera/assert'
|
|
6
|
+
require 'aspera/rainbow'
|
|
7
|
+
using Rainbow
|
|
6
8
|
|
|
7
9
|
module Aspera
|
|
8
|
-
# Run
|
|
10
|
+
# Run `ascmd` commands using specified executor (usually, remotely on transfer node)
|
|
9
11
|
# Equivalent of SDK "command client"
|
|
10
12
|
# execute: "ascmd -h" to get syntax
|
|
11
13
|
# Note: "ls" can take filters: as_ls -f *.txt -f *.bin /
|
|
@@ -42,6 +42,14 @@ module Aspera
|
|
|
42
42
|
# policy for product selection
|
|
43
43
|
FIRST_FOUND = 'FIRST'
|
|
44
44
|
|
|
45
|
+
class << self
|
|
46
|
+
# @param value [String, nil] value of option `sdk_folder`
|
|
47
|
+
# @return [Boolean] true if value selects a product instead of a folder
|
|
48
|
+
def product_selector?(value)
|
|
49
|
+
value.is_a?(String) && value.start_with?(USE_PRODUCT_PREFIX)
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
45
53
|
# Loads YAML from cloud with locations of SDK archives for all platforms
|
|
46
54
|
# @return [Hash] location structure
|
|
47
55
|
def sdk_locations
|
|
@@ -55,30 +63,25 @@ module Aspera
|
|
|
55
63
|
end
|
|
56
64
|
end
|
|
57
65
|
|
|
58
|
-
# Set `ascp` executable "location"
|
|
66
|
+
# Set `ascp` executable "location" (option `sdk_folder`)
|
|
59
67
|
# It can be:
|
|
60
|
-
# - Full path to folder where `ascp` executable is located
|
|
61
|
-
# - "product:PRODUCT_NAME" to use ascp from named product
|
|
68
|
+
# - Full path to folder where `ascp` executable is located, also used as SDK folder (keys, conf)
|
|
69
|
+
# - "product:PRODUCT_NAME" to use ascp from named product, SDK folder stays the default one
|
|
62
70
|
# - "product:FIRST" to use ascp from first found product
|
|
71
|
+
# @param ascp_location [String] folder or product selector
|
|
72
|
+
# @return [nil]
|
|
63
73
|
def sdk_folder=(ascp_location)
|
|
64
74
|
Aspera.assert_type(ascp_location, String) { 'ascp_location' }
|
|
65
75
|
Aspera.assert(!ascp_location.empty?, 'ascp location cannot be empty: check your config file')
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
end
|
|
76
|
-
File.dirname(pl[:ascp_path])
|
|
77
|
-
else
|
|
78
|
-
ascp_location.include?('/ascp') ? File.dirname(ascp_location) : ascp_location
|
|
79
|
-
end
|
|
80
|
-
Log.log.debug { "ascp_folder=#{folder}" }
|
|
81
|
-
Products::Transferd.sdk_directory = folder
|
|
76
|
+
@ascp_folder = nil
|
|
77
|
+
if self.class.product_selector?(ascp_location)
|
|
78
|
+
# Resolved on first use: product scan requires the SDK folder, which is set later
|
|
79
|
+
@ascp_product = ascp_location.delete_prefix(USE_PRODUCT_PREFIX)
|
|
80
|
+
Log.log.debug { "ascp_product=#{@ascp_product}" }
|
|
81
|
+
else
|
|
82
|
+
@ascp_product = nil
|
|
83
|
+
Products::Transferd.sdk_directory = ascp_location.include?('/ascp') ? File.dirname(ascp_location) : ascp_location
|
|
84
|
+
end
|
|
82
85
|
nil
|
|
83
86
|
end
|
|
84
87
|
|
|
@@ -114,8 +117,9 @@ module Aspera
|
|
|
114
117
|
case file_type
|
|
115
118
|
when *EXE_FILES
|
|
116
119
|
file_is_required = file_type.eql?(:ascp)
|
|
117
|
-
file =
|
|
118
|
-
|
|
120
|
+
file = sdk_exe_path(file_type)
|
|
121
|
+
# transferd always comes from SDK
|
|
122
|
+
file = File.join(product_ascp_folder, File.basename(file)) unless @ascp_product.nil? || file_type.eql?(:transferd)
|
|
119
123
|
when :ssh_private_dsa, :ssh_private_rsa
|
|
120
124
|
# assume last 3 letters are type
|
|
121
125
|
type = file_type.to_s[-3..].to_sym
|
|
@@ -282,7 +286,7 @@ module Aspera
|
|
|
282
286
|
end
|
|
283
287
|
end
|
|
284
288
|
|
|
285
|
-
# Downloads and extracts the SDK archive for the current platform into
|
|
289
|
+
# Downloads and extracts the SDK archive for the current platform into `folder`
|
|
286
290
|
# @param folder [String] Destination folder path
|
|
287
291
|
# @param url [nil, String] URL to SDK archive, if nil: default url for version
|
|
288
292
|
# @param version [nil, String] Specific version, if nil: latest version
|
|
@@ -306,7 +310,7 @@ module Aspera
|
|
|
306
310
|
if UriReader.file?(url)
|
|
307
311
|
archive_io.write(File.binread(UriReader.file_path(url)))
|
|
308
312
|
else
|
|
309
|
-
Rest.new(base_url: url, redirect_max: 3).call(operation: 'GET', save_to: archive_io)
|
|
313
|
+
Rest::Client.new(base_url: url, redirect_max: 3).call(operation: 'GET', save_to: archive_io)
|
|
310
314
|
archive_io.rewind
|
|
311
315
|
end
|
|
312
316
|
extract_archive_files(url, archive_io) do |entry_name, entry_stream, link_target|
|
|
@@ -385,10 +389,13 @@ module Aspera
|
|
|
385
389
|
folder = Products::Transferd.sdk_directory
|
|
386
390
|
download_sdk(folder: folder, url: url, version: version)
|
|
387
391
|
# Ensure necessary files are there, or generate them, restrict file access on SDK executables
|
|
392
|
+
# (executables of SDK, even if `ascp` of another product is selected)
|
|
388
393
|
SDK_FILES.each do |file_id_sym|
|
|
389
|
-
|
|
390
|
-
|
|
394
|
+
if EXE_FILES.include?(file_id_sym)
|
|
395
|
+
file_path = sdk_exe_path(file_id_sym)
|
|
391
396
|
Environment.restrict_file_access(file_path, mode: 0o755) if File.exist?(file_path)
|
|
397
|
+
else
|
|
398
|
+
path(file_id_sym)
|
|
392
399
|
end
|
|
393
400
|
end
|
|
394
401
|
# Generate meta data XML file in SDK if needed, based on actual binaries versions
|
|
@@ -399,8 +406,8 @@ module Aspera
|
|
|
399
406
|
sdk_name = meta_data.scan(%r{<name>(.*?)</name>}).flatten.first || 'unknown'
|
|
400
407
|
sdk_version = meta_data.scan(%r{<version>(.*?)</version>}).flatten.first || 'unknown'
|
|
401
408
|
else
|
|
402
|
-
sdk_ascp_version = get_ascp_version(
|
|
403
|
-
transferd_version = get_exe_version(
|
|
409
|
+
sdk_ascp_version = get_ascp_version(sdk_exe_path(:ascp))
|
|
410
|
+
transferd_version = get_exe_version(sdk_exe_path(:transferd), 'version')
|
|
404
411
|
sdk_name = 'IBM Aspera Transfer SDK'
|
|
405
412
|
sdk_version = transferd_version || sdk_ascp_version
|
|
406
413
|
File.write(meta_data_file, "<product><name>#{sdk_name}</name><version>#{sdk_version}</version></product>")
|
|
@@ -435,6 +442,34 @@ module Aspera
|
|
|
435
442
|
# cache for installed products found
|
|
436
443
|
@found_products = nil
|
|
437
444
|
@transferd_urls = TRANSFERD_ARCHIVE_LOCATION_URL
|
|
445
|
+
# product selected with option `sdk_folder`, or nil to use SDK folder
|
|
446
|
+
@ascp_product = nil
|
|
447
|
+
# cache for folder of `ascp` of selected product
|
|
448
|
+
@ascp_folder = nil
|
|
449
|
+
end
|
|
450
|
+
|
|
451
|
+
# @param file_type [Symbol] one of EXE_FILES
|
|
452
|
+
# @return [String] path to executable in SDK folder
|
|
453
|
+
def sdk_exe_path(file_type)
|
|
454
|
+
file = Products::Transferd.transferd_path
|
|
455
|
+
file = File.join(File.dirname(file), Environment.instance.exe_file(file_type.to_s)) unless file_type.eql?(:transferd)
|
|
456
|
+
return file
|
|
457
|
+
end
|
|
458
|
+
|
|
459
|
+
# @return [String] folder of `ascp` of product selected with option `sdk_folder`
|
|
460
|
+
def product_ascp_folder
|
|
461
|
+
if @ascp_folder.nil?
|
|
462
|
+
if @ascp_product.eql?(FIRST_FOUND)
|
|
463
|
+
pl = installed_products.first
|
|
464
|
+
Aspera.assert(!pl.nil?) { "No Aspera transfer module or SDK found.\nRefer to the manual or install SDK with command:\nascli config ascp install" }
|
|
465
|
+
else
|
|
466
|
+
pl = installed_products.find { |i| i[:name].eql?(@ascp_product) }
|
|
467
|
+
Aspera.assert(!pl.nil?) { "No such product installed: #{@ascp_product}" }
|
|
468
|
+
end
|
|
469
|
+
@ascp_folder = File.dirname(pl[:ascp_path])
|
|
470
|
+
Log.log.debug { "ascp_folder=#{@ascp_folder}" }
|
|
471
|
+
end
|
|
472
|
+
return @ascp_folder
|
|
438
473
|
end
|
|
439
474
|
|
|
440
475
|
public
|
data/lib/aspera/assert.rb
CHANGED
|
@@ -17,6 +17,10 @@ module Aspera
|
|
|
17
17
|
class AssertError < Error
|
|
18
18
|
end
|
|
19
19
|
|
|
20
|
+
# Raised when a looked up entity is not found
|
|
21
|
+
class EntityNotFound < Error
|
|
22
|
+
end
|
|
23
|
+
|
|
20
24
|
class << self
|
|
21
25
|
# Replaces `raise` in assertion
|
|
22
26
|
# Allows sending exception, or just error log, when type is `:error`
|
|
@@ -21,26 +21,31 @@ module Aspera
|
|
|
21
21
|
private_constant :AGENT_SCHEMA_KEY
|
|
22
22
|
|
|
23
23
|
# Set the SDK directory, checking default and former locations
|
|
24
|
+
# If a product is selected for `ascp`, the SDK directory is still needed (keys, conf, transferd)
|
|
24
25
|
def set_sdk_dir
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
26
|
+
sdk_folder = options.get_option(:sdk_folder)
|
|
27
|
+
product_selected = Ascp::Installation.product_selector?(sdk_folder)
|
|
28
|
+
return unless sdk_folder.nil? || product_selected
|
|
29
|
+
@sdk_default_location = true
|
|
30
|
+
Log.log.debug('SDK folder is not set, checking default')
|
|
31
|
+
sdk_dir = self.class.default_app_main_folder(app_name: TRANSFERD_APP_NAME)
|
|
32
|
+
Log.log.debug { "Checking: #{sdk_dir}" }
|
|
33
|
+
if !Dir.exist?(sdk_dir)
|
|
34
|
+
Log.log.debug { "No such folder: #{sdk_dir}" }
|
|
35
|
+
former_sdk_folder = File.join(self.class.default_app_main_folder(app_name: Info::CMD_NAME), TRANSFERD_APP_NAME)
|
|
36
|
+
Log.log.debug { "Checking: #{former_sdk_folder}" }
|
|
37
|
+
sdk_dir = former_sdk_folder if Dir.exist?(former_sdk_folder)
|
|
38
|
+
end
|
|
39
|
+
Log.log.debug { "Using: #{sdk_dir}" }
|
|
40
|
+
if product_selected
|
|
38
41
|
Products::Transferd.sdk_directory = sdk_dir
|
|
42
|
+
else
|
|
43
|
+
options.set_option(:sdk_folder, sdk_dir, source: :default)
|
|
39
44
|
end
|
|
40
45
|
end
|
|
41
46
|
|
|
42
47
|
# Install the transfer SDK (ascp + transferd) from a URL or using the default source.
|
|
43
|
-
# Version defaults to
|
|
48
|
+
# Version defaults to `Info::SDK_VERSION`; pass `LATEST` as argument to install the latest available version.
|
|
44
49
|
# @param version [String, nil] version to install; nil means use the default SDK version
|
|
45
50
|
# @return [Result::Status] installation result message
|
|
46
51
|
def install_transfer_sdk(version: nil)
|
|
@@ -52,27 +57,14 @@ module Aspera
|
|
|
52
57
|
return Result::Status.new("Installed #{name} version #{ver} in #{folder}")
|
|
53
58
|
end
|
|
54
59
|
|
|
55
|
-
def action_ascp_show(**)
|
|
56
|
-
Result::Text.new(Ascp::Installation.instance.path(:ascp))
|
|
57
|
-
end
|
|
58
|
-
|
|
59
60
|
def action_ascp_info(**)
|
|
61
|
+
SecretHider.instance.add_secret_keys(DataRepository::ELEMENTS)
|
|
60
62
|
data = Ascp::Installation.instance.ascp_info
|
|
61
63
|
data['ts'] = transfer.user_transfer_spec
|
|
62
64
|
DataRepository::ELEMENTS.each_with_object(data) { |i, h| h[i.to_s] = DataRepository.instance.item(i) }
|
|
63
|
-
SecretHider::ADDITIONAL_KEYS_TO_HIDE.concat(DataRepository::ELEMENTS.map(&:to_s))
|
|
64
65
|
Result::SingleObject.new(data)
|
|
65
66
|
end
|
|
66
67
|
|
|
67
|
-
def action_ascp_install(version: nil, **)
|
|
68
|
-
install_transfer_sdk(version: version)
|
|
69
|
-
end
|
|
70
|
-
|
|
71
|
-
def action_ascp_spec(**)
|
|
72
|
-
builder = Schema::Documentation.new(TerminalFormatter, Transfer::Spec::SCHEMA, include_option: true, agent_columns: true).build
|
|
73
|
-
Result::ObjectList.new(builder.rows, fields: builder.columns)
|
|
74
|
-
end
|
|
75
|
-
|
|
76
68
|
def action_ascp_schema(agent_name: nil, **)
|
|
77
69
|
schema = Transfer::Spec::SCHEMA.current.merge({'$comment'=>'DO NOT EDIT, this file was generated from the YAML.'})
|
|
78
70
|
schema['properties'] = schema['properties'].select { |_k, v| CommandLineBuilder.supported_by_agent(agent_name, v) } unless agent_name.nil?
|
|
@@ -88,10 +80,6 @@ module Aspera
|
|
|
88
80
|
Result::ObjectList.new(error_data)
|
|
89
81
|
end
|
|
90
82
|
|
|
91
|
-
def action_ascp_products_list(**)
|
|
92
|
-
Result::ObjectList.new(Ascp::Installation.instance.installed_products, fields: %w[name app_root])
|
|
93
|
-
end
|
|
94
|
-
|
|
95
83
|
def action_agents_list(**)
|
|
96
84
|
rows = Agent::Factory::ALL.map do |sym, names|
|
|
97
85
|
schema_key = AGENT_SCHEMA_KEY[sym]
|
|
@@ -141,15 +129,6 @@ module Aspera
|
|
|
141
129
|
end
|
|
142
130
|
Result::ObjectList.new(rows, fields: %w[name type description])
|
|
143
131
|
end
|
|
144
|
-
|
|
145
|
-
def action_transferd_install(**)
|
|
146
|
-
install_transfer_sdk
|
|
147
|
-
end
|
|
148
|
-
|
|
149
|
-
def action_transferd_list(**)
|
|
150
|
-
sdk_list = Ascp::Installation.instance.sdk_locations
|
|
151
|
-
Result::ObjectList.new(sdk_list, fields: sdk_list.first.keys - ['url'])
|
|
152
|
-
end
|
|
153
132
|
end
|
|
154
133
|
end
|
|
155
134
|
end
|
|
@@ -10,7 +10,7 @@ module Aspera
|
|
|
10
10
|
#
|
|
11
11
|
# Each entry is a JSON document identified by a `job_id` (UUID generated by ascli).
|
|
12
12
|
# The underlying storage uses PersistencyFolder with the category prefix
|
|
13
|
-
#
|
|
13
|
+
# `CATEGORY` so all async-transfer files are grouped together and can be
|
|
14
14
|
# garbage-collected independently.
|
|
15
15
|
#
|
|
16
16
|
# Schema of a stored entry:
|
|
@@ -81,7 +81,7 @@ module Aspera
|
|
|
81
81
|
end
|
|
82
82
|
|
|
83
83
|
# List all async transfer entries.
|
|
84
|
-
# @return [Array<Hash>] each entry includes the
|
|
84
|
+
# @return [Array<Hash>] each entry includes the `job_id` field
|
|
85
85
|
def list
|
|
86
86
|
@persistency.current_items(CATEGORY).map do |key, raw|
|
|
87
87
|
data = JSON.parse(raw)
|
|
@@ -31,8 +31,8 @@ module Aspera
|
|
|
31
31
|
# - populate context.persistency, context.presets, context.http_config, context.progress_bar
|
|
32
32
|
# - register plugin lookup folders
|
|
33
33
|
# - register @preset / @vault extended-value handlers
|
|
34
|
-
# - configure global singletons:
|
|
35
|
-
#
|
|
34
|
+
# - configure global singletons: Rest::Parameters, OAuth::Factory, SSL, Transfer::Parameters,
|
|
35
|
+
# Rest::ErrorAnalyzer
|
|
36
36
|
# - set up the PAC proxy executor (option :fpac)
|
|
37
37
|
class Bootstrapper
|
|
38
38
|
# Folder name inside $HOME for all Aspera tool data (~/.aspera)
|
|
@@ -88,16 +88,13 @@ module Aspera
|
|
|
88
88
|
setup_rest_and_transfer_runtime
|
|
89
89
|
end
|
|
90
90
|
|
|
91
|
-
# Public accessor used as option handler for :config_file
|
|
92
|
-
attr_accessor :config_file_option
|
|
93
|
-
|
|
94
91
|
private
|
|
95
92
|
|
|
96
93
|
# Declare + parse :home option: sets context.main_folder
|
|
97
94
|
def setup_main_folder
|
|
98
95
|
@context.options.declare(
|
|
99
96
|
:home, description: 'Home folder for tool',
|
|
100
|
-
|
|
97
|
+
on_set: @context.method(:main_folder=),
|
|
101
98
|
default: default_app_main_folder(app_name: Info::CMD_NAME)
|
|
102
99
|
)
|
|
103
100
|
@context.options.parse_options!
|
|
@@ -115,11 +112,10 @@ module Aspera
|
|
|
115
112
|
def setup_config_file
|
|
116
113
|
@context.options.declare(
|
|
117
114
|
:config_file, description: 'Path to YAML file with preset configuration',
|
|
118
|
-
handler: {o: self, m: :config_file_option},
|
|
119
115
|
default: File.join(@context.main_folder, DEFAULT_CONFIG_FILENAME)
|
|
120
116
|
)
|
|
121
117
|
@context.options.parse_options!
|
|
122
|
-
@context.presets = PresetManager.new(config_file: @
|
|
118
|
+
@context.presets = PresetManager.new(config_file: @context.options.get_option(:config_file))
|
|
123
119
|
@context.http_config = Http.new
|
|
124
120
|
end
|
|
125
121
|
|
|
@@ -165,18 +161,18 @@ module Aspera
|
|
|
165
161
|
end
|
|
166
162
|
end
|
|
167
163
|
|
|
168
|
-
# Configure global singletons:
|
|
164
|
+
# Configure global singletons: Rest::Parameters, SSL, Transfer, Rest::ErrorAnalyzer.
|
|
169
165
|
# OAuth persist_mgr is NOT set here: it depends on :cache_tokens which is parsed later
|
|
170
166
|
# by Config#initialize. Runner sets it after Config.new.
|
|
171
167
|
def setup_rest_and_transfer_runtime
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
168
|
+
Rest::Parameters.instance.user_agent = Info::CMD_NAME
|
|
169
|
+
Rest::Parameters.instance.progress_bar = @context.progress_bar
|
|
170
|
+
Rest::Parameters.instance.session_cb = ->(http_session) { @context.http_config.update_session(http_session) }
|
|
171
|
+
Rest::Parameters.instance.spinner_cb = ->(title = nil, action: :spin) { @context.formatter.long_operation(title, action: action) }
|
|
176
172
|
OAuth::Web.additional_info = "#{Info::CMD_NAME} v#{Cli::VERSION}"
|
|
177
173
|
Transfer::Parameters.file_list_folder = File.join(@context.main_folder, FILE_LIST_FOLDER_NAME)
|
|
178
|
-
|
|
179
|
-
|
|
174
|
+
Rest::ErrorAnalyzer.instance.log_file = File.join(@context.main_folder, REST_EXCEPTIONS_LOG_FILENAME)
|
|
175
|
+
Rest::AsperaErrors.register_handlers
|
|
180
176
|
end
|
|
181
177
|
|
|
182
178
|
# @return [String] ~/.aspera
|