aspera-cli 4.21.2 → 4.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +1 -1
- data/CHANGELOG.md +402 -374
- data/CONTRIBUTING.md +6 -10
- data/README.md +1018 -687
- data/lib/aspera/agent/base.rb +9 -5
- data/lib/aspera/agent/connect.rb +30 -28
- data/lib/aspera/agent/desktop.rb +29 -25
- data/lib/aspera/agent/direct.rb +137 -125
- data/lib/aspera/agent/httpgw.rb +22 -26
- data/lib/aspera/agent/node.rb +14 -11
- data/lib/aspera/agent/transferd.rb +6 -2
- data/lib/aspera/api/aoc.rb +15 -18
- data/lib/aspera/api/cos_node.rb +1 -1
- data/lib/aspera/api/httpgw.rb +15 -7
- data/lib/aspera/api/node.rb +6 -4
- data/lib/aspera/ascmd.rb +17 -9
- data/lib/aspera/ascp/installation.rb +21 -19
- data/lib/aspera/ascp/management.rb +1 -1
- data/lib/aspera/assert.rb +14 -5
- data/lib/aspera/cli/error.rb +2 -2
- data/lib/aspera/cli/extended_value.rb +38 -19
- data/lib/aspera/cli/formatter.rb +48 -48
- data/lib/aspera/cli/hints.rb +10 -2
- data/lib/aspera/cli/main.rb +190 -168
- data/lib/aspera/cli/manager.rb +16 -16
- data/lib/aspera/cli/plugin.rb +24 -21
- data/lib/aspera/cli/plugin_factory.rb +1 -1
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +173 -126
- data/lib/aspera/cli/plugins/ats.rb +19 -17
- data/lib/aspera/cli/plugins/config.rb +87 -98
- data/lib/aspera/cli/plugins/console.rb +5 -3
- data/lib/aspera/cli/plugins/faspex.rb +39 -35
- data/lib/aspera/cli/plugins/faspex5.rb +104 -80
- data/lib/aspera/cli/plugins/faspio.rb +13 -1
- data/lib/aspera/cli/plugins/httpgw.rb +13 -1
- data/lib/aspera/cli/plugins/node.rb +336 -205
- data/lib/aspera/cli/plugins/orchestrator.rb +34 -40
- data/lib/aspera/cli/plugins/preview.rb +3 -3
- data/lib/aspera/cli/plugins/server.rb +7 -6
- data/lib/aspera/cli/plugins/shares.rb +5 -5
- data/lib/aspera/cli/sync_actions.rb +19 -18
- data/lib/aspera/cli/transfer_agent.rb +11 -15
- data/lib/aspera/cli/transfer_progress.rb +2 -2
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +116 -95
- data/lib/aspera/coverage.rb +4 -3
- data/lib/aspera/data_repository.rb +1 -0
- data/lib/aspera/environment.rb +7 -6
- data/lib/aspera/faspex_gw.rb +14 -14
- data/lib/aspera/faspex_postproc.rb +7 -6
- data/lib/aspera/hash_ext.rb +2 -2
- data/lib/aspera/json_rpc.rb +1 -1
- data/lib/aspera/keychain/encrypted_hash.rb +47 -34
- data/lib/aspera/keychain/factory.rb +41 -0
- data/lib/aspera/keychain/hashicorp_vault.rb +71 -0
- data/lib/aspera/keychain/macos_security.rb +19 -11
- data/lib/aspera/log.rb +29 -34
- data/lib/aspera/nagios.rb +6 -6
- data/lib/aspera/node_simulator.rb +8 -8
- data/lib/aspera/oauth/base.rb +10 -6
- data/lib/aspera/oauth/factory.rb +6 -6
- data/lib/aspera/oauth/url_json.rb +6 -6
- data/lib/aspera/persistency_action_once.rb +6 -4
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/file_types.rb +40 -33
- data/lib/aspera/preview/generator.rb +1 -1
- data/lib/aspera/preview/options.rb +16 -16
- data/lib/aspera/preview/terminal.rb +3 -3
- data/lib/aspera/preview/utils.rb +11 -13
- data/lib/aspera/products/connect.rb +2 -1
- data/lib/aspera/products/desktop.rb +1 -1
- data/lib/aspera/products/transferd.rb +1 -1
- data/lib/aspera/proxy_auto_config.rb +2 -2
- data/lib/aspera/rest.rb +70 -50
- data/lib/aspera/rest_error_analyzer.rb +1 -0
- data/lib/aspera/rest_errors_aspera.rb +1 -1
- data/lib/aspera/secret_hider.rb +5 -5
- data/lib/aspera/ssh.rb +5 -5
- data/lib/aspera/temp_file_manager.rb +1 -0
- data/lib/aspera/timer_limiter.rb +7 -5
- data/lib/aspera/transfer/async_conf.schema.yaml +716 -0
- data/lib/aspera/transfer/convert.rb +29 -0
- data/lib/aspera/transfer/error_info.rb +66 -66
- data/lib/aspera/transfer/parameters.rb +13 -68
- data/lib/aspera/transfer/spec.rb +5 -6
- data/lib/aspera/transfer/spec.schema.yaml +753 -0
- data/lib/aspera/transfer/spec_doc.rb +62 -0
- data/lib/aspera/transfer/sync.rb +37 -76
- data/lib/aspera/transfer/sync_instance.schema.yaml +20 -0
- data/lib/aspera/transfer/sync_session.schema.yaml +86 -0
- data/lib/aspera/transfer/uri.rb +6 -6
- data/lib/aspera/uri_reader.rb +1 -1
- data/lib/aspera/web_auth.rb +1 -1
- data/lib/aspera/web_server_simple.rb +53 -44
- data.tar.gz.sig +0 -0
- metadata +38 -7
- metadata.gz.sig +0 -0
- data/examples/build_package.sh +0 -28
- data/examples/dascli +0 -30
- data/examples/get_proto_file.rb +0 -8
- data/examples/proxy.pac +0 -60
- data/lib/aspera/transfer/spec.yaml +0 -718
data/lib/aspera/agent/httpgw.rb
CHANGED
@@ -9,9 +9,28 @@ require 'aspera/assert'
|
|
9
9
|
module Aspera
|
10
10
|
module Agent
|
11
11
|
class Httpgw < Base
|
12
|
-
|
13
|
-
|
12
|
+
def initialize(
|
13
|
+
url:,
|
14
|
+
api_version: Api::Httpgw::API_V2,
|
15
|
+
upload_chunk_size: 64_000,
|
16
|
+
synchronous: false,
|
17
|
+
**base_options
|
18
|
+
)
|
19
|
+
super(**base_options)
|
20
|
+
@gw_api = Api::Httpgw.new(
|
21
|
+
# remove /v1 from end of user-provided GW url: we need the base url only
|
22
|
+
url: url,
|
23
|
+
api_version: api_version,
|
24
|
+
upload_chunk_size: upload_chunk_size,
|
25
|
+
synchronous: synchronous,
|
26
|
+
notify_cb: ->(*pa, **ka){notify_progress(*pa, **ka)}
|
27
|
+
)
|
28
|
+
end
|
29
|
+
|
30
|
+
# Start FASP transfer based on transfer spec (hash table)
|
31
|
+
# note that this should be asynchronous, but it is not
|
14
32
|
# HTTP download only supports file list
|
33
|
+
# :reek:UnusedParameters token_regenerator
|
15
34
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
16
35
|
raise 'GW URL must be set' if @gw_api.nil?
|
17
36
|
Aspera.assert_type(transfer_spec['paths'], Array){'paths'}
|
@@ -27,35 +46,12 @@ module Aspera
|
|
27
46
|
end
|
28
47
|
end
|
29
48
|
|
30
|
-
#
|
49
|
+
# Wait for completion of all jobs started
|
31
50
|
# @return list of :success or error message
|
32
51
|
def wait_for_transfers_completion
|
33
52
|
# well ... transfer was done in "start"
|
34
53
|
return [:success]
|
35
54
|
end
|
36
|
-
|
37
|
-
# TODO: is that useful?
|
38
|
-
# def url=(api_url); end
|
39
|
-
|
40
|
-
private
|
41
|
-
|
42
|
-
def initialize(
|
43
|
-
url:,
|
44
|
-
api_version: Api::Httpgw::API_V2,
|
45
|
-
upload_chunk_size: 64_000,
|
46
|
-
synchronous: false,
|
47
|
-
**base_options
|
48
|
-
)
|
49
|
-
super(**base_options)
|
50
|
-
@gw_api = Api::Httpgw.new(
|
51
|
-
# remove /v1 from end of user-provided GW url: we need the base url only
|
52
|
-
url: url,
|
53
|
-
api_version: api_version,
|
54
|
-
upload_chunk_size: upload_chunk_size,
|
55
|
-
synchronous: synchronous,
|
56
|
-
notify_cb: ->(*pa, **ka) { notify_progress(*pa, **ka) }
|
57
|
-
)
|
58
|
-
end
|
59
55
|
end
|
60
56
|
end
|
61
57
|
end
|
data/lib/aspera/agent/node.rb
CHANGED
@@ -13,10 +13,10 @@ module Aspera
|
|
13
13
|
# this singleton class is used by the CLI to provide a common interface to start a transfer
|
14
14
|
# before using it, the use must set the `node_api` member.
|
15
15
|
class Node < Base
|
16
|
-
# @param url
|
17
|
-
# @param username
|
18
|
-
# @param password
|
19
|
-
# @param root_id
|
16
|
+
# @param url [String] the base url of the node api
|
17
|
+
# @param username [String] the username to use for the node api
|
18
|
+
# @param password [String] the password to use for the node api
|
19
|
+
# @param root_id [String] root file id if the node is an access key
|
20
20
|
# @param base_options [Hash] options for base class
|
21
21
|
def initialize(
|
22
22
|
url:,
|
@@ -28,7 +28,7 @@ module Aspera
|
|
28
28
|
super(**base_options)
|
29
29
|
# root id is required for access key
|
30
30
|
@root_id = root_id
|
31
|
-
rest_params = {
|
31
|
+
rest_params = {base_url: url}
|
32
32
|
if OAuth::Factory.bearer?(password)
|
33
33
|
Aspera.assert(!@root_id.nil?){'root_id not allowed for access key'}
|
34
34
|
rest_params[:headers] = Api::Node.bearer_headers(password, access_key: username)
|
@@ -44,13 +44,8 @@ module Aspera
|
|
44
44
|
@transfer_id = nil
|
45
45
|
end
|
46
46
|
|
47
|
-
# used internally to ensure node api is set before using.
|
48
|
-
def node_api_
|
49
|
-
Aspera.assert(!@node_api.nil?){'Before using this object, set the node_api attribute to a Aspera::Rest object'}
|
50
|
-
return @node_api
|
51
|
-
end
|
52
|
-
|
53
47
|
# generic method
|
48
|
+
# :reek:UnusedParameters token_regenerator
|
54
49
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
55
50
|
# add root id if access key
|
56
51
|
if !@root_id.nil?
|
@@ -119,6 +114,14 @@ module Aspera
|
|
119
114
|
# TODO: get status of sessions
|
120
115
|
return []
|
121
116
|
end
|
117
|
+
|
118
|
+
private
|
119
|
+
|
120
|
+
# used internally to ensure node api is set before using.
|
121
|
+
def node_api_
|
122
|
+
Aspera.assert(!@node_api.nil?){'Before using this object, set the node_api attribute to a Aspera::Rest object'}
|
123
|
+
return @node_api
|
124
|
+
end
|
122
125
|
end
|
123
126
|
end
|
124
127
|
end
|
@@ -31,6 +31,7 @@ module Aspera
|
|
31
31
|
**base
|
32
32
|
)
|
33
33
|
super(**base)
|
34
|
+
@transfer_id = nil
|
34
35
|
@stop = stop
|
35
36
|
is_local_auto_port = url.eql?(AUTO_LOCAL_TCP_PORT)
|
36
37
|
raise 'Cannot set options `stop` or `start` to false with port zero' if is_local_auto_port && (!@stop || !start)
|
@@ -94,7 +95,7 @@ module Aspera
|
|
94
95
|
end
|
95
96
|
Log.log.debug{"Daemon started with pid #{@daemon_pid}"}
|
96
97
|
Process.detach(@daemon_pid) unless @stop
|
97
|
-
at_exit
|
98
|
+
at_exit{shutdown}
|
98
99
|
# update port for next connection attempt (if auto high port was requested)
|
99
100
|
daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{PORT_SEP}#{Products::Transferd.daemon_port_from_log(log_stdout)}" if is_local_auto_port
|
100
101
|
# local daemon started, try again
|
@@ -102,6 +103,7 @@ module Aspera
|
|
102
103
|
end
|
103
104
|
end
|
104
105
|
|
106
|
+
# :reek:UnusedParameters token_regenerator
|
105
107
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
106
108
|
# create a transfer request
|
107
109
|
transfer_request = ::Transferd::Api::TransferRequest.new(
|
@@ -156,10 +158,12 @@ module Aspera
|
|
156
158
|
stop_daemon if @stop
|
157
159
|
end
|
158
160
|
|
161
|
+
private
|
162
|
+
|
159
163
|
def stop_daemon
|
160
164
|
if !@daemon_pid.nil?
|
161
165
|
Log.log.debug("Stopping daemon #{@daemon_pid}")
|
162
|
-
Process.kill(
|
166
|
+
Process.kill(:INT, @daemon_pid)
|
163
167
|
_, status = Process.wait2(@daemon_pid)
|
164
168
|
Log.log.debug("daemon stopped #{status}")
|
165
169
|
@daemon_pid = nil
|
data/lib/aspera/api/aoc.rb
CHANGED
@@ -22,7 +22,7 @@ module Aspera
|
|
22
22
|
MAX_AOC_URL_REDIRECT = 10
|
23
23
|
CLIENT_ID_PREFIX = 'aspera.'
|
24
24
|
# Well-known AoC global client apps
|
25
|
-
GLOBAL_CLIENT_APPS = DataRepository::ELEMENTS.select{|i|i.to_s.start_with?(CLIENT_ID_PREFIX)}.freeze
|
25
|
+
GLOBAL_CLIENT_APPS = DataRepository::ELEMENTS.select{ |i| i.to_s.start_with?(CLIENT_ID_PREFIX)}.freeze
|
26
26
|
# cookie prefix so that console can decode identity
|
27
27
|
COOKIE_PREFIX_CONSOLE_AOC = 'aspera.aoc'
|
28
28
|
# path in URL of public links
|
@@ -34,6 +34,8 @@ module Aspera
|
|
34
34
|
# types of events for shared folder creation
|
35
35
|
# Node events: permission.created permission.modified permission.deleted
|
36
36
|
PERMISSIONS_CREATED = ['permission.created'].freeze
|
37
|
+
# Special name when creating workspace shared folders
|
38
|
+
ID_AK_ADMIN = 'ASPERA_ACCESS_KEY_ADMIN'
|
37
39
|
|
38
40
|
private_constant :MAX_AOC_URL_REDIRECT,
|
39
41
|
:CLIENT_ID_PREFIX,
|
@@ -43,7 +45,8 @@ module Aspera
|
|
43
45
|
:JWT_AUDIENCE,
|
44
46
|
:OAUTH_API_SUBPATH,
|
45
47
|
:USER_INFO_FIELDS_MIN,
|
46
|
-
:PERMISSIONS_CREATED
|
48
|
+
:PERMISSIONS_CREATED,
|
49
|
+
:ID_AK_ADMIN
|
47
50
|
|
48
51
|
# various API scopes supported
|
49
52
|
SCOPE_FILES_SELF = 'self'
|
@@ -237,7 +240,7 @@ module Aspera
|
|
237
240
|
Log.log.debug{"ignoring error: #{e}"}
|
238
241
|
{}
|
239
242
|
end
|
240
|
-
USER_INFO_FIELDS_MIN.each{|f
|
243
|
+
USER_INFO_FIELDS_MIN.each{ |f| @cache_user_info[f] = nil if @cache_user_info[f].nil?}
|
241
244
|
return @cache_user_info
|
242
245
|
end
|
243
246
|
|
@@ -283,7 +286,7 @@ module Aspera
|
|
283
286
|
if ws_info.nil?
|
284
287
|
{
|
285
288
|
id: nil,
|
286
|
-
name:
|
289
|
+
name: "Shared #{application}"
|
287
290
|
}
|
288
291
|
else
|
289
292
|
{
|
@@ -386,10 +389,10 @@ module Aspera
|
|
386
389
|
Aspera.assert(field.key?('name')){'metadata field must have name'}
|
387
390
|
Aspera.assert(field.key?('values')){'metadata field must have values'}
|
388
391
|
Aspera.assert_type(field['values'], Array){'metadata field values'}
|
389
|
-
Aspera.assert(!meta_schema.none?{|i|i['name'].eql?(field['name'])}){"unknown metadata field: #{field['name']}"}
|
392
|
+
Aspera.assert(!meta_schema.none?{ |i| i['name'].eql?(field['name'])}){"unknown metadata field: #{field['name']}"}
|
390
393
|
end
|
391
394
|
meta_schema.each do |field|
|
392
|
-
provided = pkg_meta.select{|i|i['name'].eql?(field['name'])}
|
395
|
+
provided = pkg_meta.select{ |i| i['name'].eql?(field['name'])}
|
393
396
|
raise "only one field with name #{field['name']} allowed" if provided.count > 1
|
394
397
|
raise "missing mandatory field: #{field['name']}" if field['required'] && provided.empty?
|
395
398
|
end
|
@@ -519,7 +522,7 @@ module Aspera
|
|
519
522
|
# Console cookie
|
520
523
|
################
|
521
524
|
# we are sure that fields are not nil
|
522
|
-
cookie_elements = [app_info[:app], current_user_info['name'] || 'public link', current_user_info['email'] || 'none'].map{|e|Base64.strict_encode64(e)}
|
525
|
+
cookie_elements = [app_info[:app], current_user_info['name'] || 'public link', current_user_info['email'] || 'none'].map{ |e| Base64.strict_encode64(e)}
|
523
526
|
cookie_elements.unshift(COOKIE_PREFIX_CONSOLE_AOC)
|
524
527
|
transfer_spec['cookie'] = cookie_elements.join(':')
|
525
528
|
# Application tags
|
@@ -546,17 +549,12 @@ module Aspera
|
|
546
549
|
transfer_spec['tags'][Transfer::Spec::TAG_RESERVED]['app'] = app_info[:app]
|
547
550
|
end
|
548
551
|
|
549
|
-
ID_AK_ADMIN = 'ASPERA_ACCESS_KEY_ADMIN'
|
550
552
|
# Callback from Plugins::Node
|
551
553
|
# add application specific tags to permissions creation
|
552
554
|
# @param perm_data [Hash] parameters for creating permissions
|
553
555
|
# @param app_info [Hash] application information
|
554
556
|
def permissions_set_create_params(perm_data:, app_info:)
|
555
|
-
# workspace shared folder:
|
556
|
-
# access_id = "#{ID_AK_ADMIN}_WS_#{app_info[:workspace_id]}"
|
557
557
|
defaults = {
|
558
|
-
# 'access_type' => 'user', # mandatory: user or group
|
559
|
-
# 'access_id' => access_id, # id of user or group or special
|
560
558
|
'tags' => {
|
561
559
|
Transfer::Spec::TAG_RESERVED => {
|
562
560
|
'files' => {
|
@@ -567,8 +565,6 @@ module Aspera
|
|
567
565
|
'shared_by_user_id' => current_user_info['id'],
|
568
566
|
'shared_by_name' => current_user_info['name'],
|
569
567
|
'shared_by_email' => current_user_info['email'],
|
570
|
-
# 'shared_with_name' => access_id,
|
571
|
-
# 'share_as' => new_name_for_folder,
|
572
568
|
'access_key' => app_info[:node_info]['access_key'],
|
573
569
|
'node' => app_info[:node_info]['name']
|
574
570
|
}
|
@@ -578,19 +574,20 @@ module Aspera
|
|
578
574
|
}
|
579
575
|
perm_data.deep_merge!(defaults)
|
580
576
|
tag_workspace = perm_data['tags'][Transfer::Spec::TAG_RESERVED]['files']['workspace']
|
581
|
-
|
577
|
+
shared_with = perm_data.delete('with')
|
578
|
+
case shared_with
|
582
579
|
when NilClass
|
583
580
|
when ''
|
581
|
+
# workspace shared folder
|
584
582
|
perm_data['access_type'] = 'user'
|
585
583
|
perm_data['access_id'] = "#{ID_AK_ADMIN}_WS_#{app_info[:workspace_id]}"
|
586
584
|
tag_workspace['shared_with_name'] = perm_data['access_id']
|
587
585
|
else
|
588
|
-
entity_info = lookup_by_name('contacts',
|
586
|
+
entity_info = lookup_by_name('contacts', shared_with, query: {'current_workspace_id' => app_info[:workspace_id]})
|
589
587
|
perm_data['access_type'] = entity_info['source_type']
|
590
588
|
perm_data['access_id'] = entity_info['source_id']
|
591
|
-
tag_workspace['shared_with_name'] = entity_info['email']
|
589
|
+
tag_workspace['shared_with_name'] = entity_info['email'] # TODO: check that ???
|
592
590
|
end
|
593
|
-
perm_data.delete('with')
|
594
591
|
if perm_data.key?('as')
|
595
592
|
tag_workspace['share_as'] = perm_data['as']
|
596
593
|
perm_data.delete('as')
|
data/lib/aspera/api/cos_node.rb
CHANGED
@@ -88,7 +88,7 @@ module Aspera
|
|
88
88
|
)
|
89
89
|
# get delegated token to be placed in rest call header and in transfer tags
|
90
90
|
@storage_credentials['token'][TOKEN_FIELD] = delegated_oauth.token
|
91
|
-
|
91
|
+
headers['X-Aspera-Storage-Credentials'] = JSON.generate(@storage_credentials)
|
92
92
|
end
|
93
93
|
end
|
94
94
|
end
|
data/lib/aspera/api/httpgw.rb
CHANGED
@@ -171,7 +171,7 @@ module Aspera
|
|
171
171
|
cond_var: ConditionVariable.new
|
172
172
|
}
|
173
173
|
# start read thread after handshake
|
174
|
-
@ws_read_thread = Thread.new
|
174
|
+
@ws_read_thread = Thread.new{process_read_thread}
|
175
175
|
@notify_cb&.call(:session_start, session_id: session_id)
|
176
176
|
@notify_cb&.call(:session_size, session_id: session_id, info: total_bytes_to_transfer)
|
177
177
|
sleep(1)
|
@@ -244,18 +244,22 @@ module Aspera
|
|
244
244
|
end
|
245
245
|
|
246
246
|
def download(transfer_spec)
|
247
|
-
transfer_spec['zip_required'] ||= false
|
248
247
|
transfer_spec['source_root'] ||= '/'
|
248
|
+
default_file_name = transfer_spec['paths'].first['source']
|
249
|
+
source_is_folder = %w[. /].include?(default_file_name)
|
250
|
+
default_file_name = 'http_download' if source_is_folder
|
251
|
+
transfer_spec['zip_required'] ||= source_is_folder || transfer_spec['paths'].length > 1
|
249
252
|
# is normally provided by application, like package name
|
250
253
|
if !transfer_spec.key?('download_name')
|
251
254
|
# by default it is the name of first file
|
252
|
-
download_name = File.basename(
|
253
|
-
#
|
255
|
+
download_name = File.basename(default_file_name, '.*')
|
256
|
+
# add indication of number of files if there is more than one
|
254
257
|
if transfer_spec['paths'].length > 1
|
255
258
|
download_name += " #{transfer_spec['paths'].length} Files"
|
256
259
|
end
|
257
260
|
transfer_spec['download_name'] = download_name
|
258
261
|
end
|
262
|
+
# start transfer session on httpgw
|
259
263
|
creation = create('download', {'transfer_spec' => transfer_spec})
|
260
264
|
transfer_uuid = creation['url'].split('/').last
|
261
265
|
file_name =
|
@@ -264,7 +268,7 @@ module Aspera
|
|
264
268
|
transfer_spec['download_name'] + '.zip'
|
265
269
|
else
|
266
270
|
# it is a plain file if we don't require zip and there is only one file
|
267
|
-
File.basename(
|
271
|
+
File.basename(default_file_name)
|
268
272
|
end
|
269
273
|
file_path = File.join(transfer_spec['destination_root'], file_name)
|
270
274
|
call(operation: 'GET', subpath: "download/#{transfer_uuid}", save_to_file: file_path)
|
@@ -309,11 +313,15 @@ module Aspera
|
|
309
313
|
# web socket endpoint: by default use v2 (newer gateways), without base64 encoding
|
310
314
|
# is the latest supported? else revert to old api
|
311
315
|
if !@upload_version.eql?(API_V1)
|
312
|
-
if !@api_info['endpoints'].any?{|i|i.include?(@upload_version)}
|
316
|
+
if !@api_info['endpoints'].any?{ |i| i.include?(@upload_version)}
|
313
317
|
Log.log.warn{"API version #{@upload_version} not supported, reverting to #{API_V1}"}
|
314
318
|
@upload_version = API_V1
|
315
319
|
end
|
316
320
|
end
|
321
|
+
@shared_info = nil
|
322
|
+
@ws_handshake = nil
|
323
|
+
@ws_io = nil
|
324
|
+
@ws_read_thread = nil
|
317
325
|
end
|
318
326
|
|
319
327
|
private
|
@@ -373,7 +381,7 @@ module Aspera
|
|
373
381
|
raise "File not found: #{source_path}"
|
374
382
|
end
|
375
383
|
end
|
376
|
-
transfer_spec['paths'] = files_to_send.map{|i|{'source' => i[:name]}}
|
384
|
+
transfer_spec['paths'] = files_to_send.map{ |i| {'source' => i[:name]}}
|
377
385
|
files_to_send.push(total_bytes_to_transfer)
|
378
386
|
return files_to_send
|
379
387
|
end
|
data/lib/aspera/api/node.rb
CHANGED
@@ -27,7 +27,7 @@ module Aspera
|
|
27
27
|
:SIGNATURE_DELIMITER, :BEARER_TOKEN_VALIDITY_DEFAULT,
|
28
28
|
:REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
|
29
29
|
|
30
|
-
#
|
30
|
+
# Node API permissions
|
31
31
|
ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
|
32
32
|
HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
|
33
33
|
HEADER_X_TOTAL_COUNT = 'X-Total-Count'
|
@@ -38,7 +38,7 @@ module Aspera
|
|
38
38
|
PATH_SEPARATOR = '/'
|
39
39
|
|
40
40
|
# register node special token decoder
|
41
|
-
OAuth::Factory.instance.register_decoder(lambda{|token|Node.decode_bearer_token(token)})
|
41
|
+
OAuth::Factory.instance.register_decoder(lambda{ |token| Node.decode_bearer_token(token)})
|
42
42
|
|
43
43
|
# class instance variable, access with accessors on class
|
44
44
|
@use_standard_ports = true
|
@@ -81,6 +81,7 @@ module Aspera
|
|
81
81
|
end
|
82
82
|
|
83
83
|
# Create an Aspera Node bearer token
|
84
|
+
# @param access_key [String] Access key identifier
|
84
85
|
# @param payload [String] JSON payload to be included in the token
|
85
86
|
# @param private_key [OpenSSL::PKey::RSA] Private key to sign the token
|
86
87
|
def bearer_token(access_key:, payload:, private_key:)
|
@@ -106,6 +107,7 @@ module Aspera
|
|
106
107
|
].join("\n")))
|
107
108
|
end
|
108
109
|
|
110
|
+
# Decode an Aspera Node bearer token
|
109
111
|
def decode_bearer_token(token)
|
110
112
|
return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition(SIGNATURE_DELIMITER).first)
|
111
113
|
end
|
@@ -149,7 +151,7 @@ module Aspera
|
|
149
151
|
|
150
152
|
# Call node API, possibly adding cache control header, as globally specified
|
151
153
|
def read_with_cache(subpath, query=nil)
|
152
|
-
headers = {'Accept' =>
|
154
|
+
headers = {'Accept' => Rest::MIME_JSON}
|
153
155
|
headers[HEADER_X_CACHE_CONTROL] = 'no-cache' unless self.class.use_node_cache
|
154
156
|
return call(
|
155
157
|
operation: 'GET',
|
@@ -352,7 +354,7 @@ module Aspera
|
|
352
354
|
# get the transfer user from info on access key
|
353
355
|
transfer_spec['remote_user'] = info['transfer_user'] if info['transfer_user']
|
354
356
|
# get settings from name.value array to hash key.value
|
355
|
-
settings = info['settings']&.each_with_object({}){|i, h|h[i['name']] = i['value']}
|
357
|
+
settings = info['settings']&.each_with_object({}){ |i, h| h[i['name']] = i['value']}
|
356
358
|
# check WSS ports
|
357
359
|
Transfer::Spec::WSS_FIELDS.each do |i|
|
358
360
|
transfer_spec[i] = settings[i] if settings.key?(i)
|
data/lib/aspera/ascmd.rb
CHANGED
@@ -10,7 +10,7 @@ module Aspera
|
|
10
10
|
# execute: "ascmd -h" to get syntax
|
11
11
|
# Note: "ls" can take filters: as_ls -f *.txt -f *.bin /
|
12
12
|
class AsCmd
|
13
|
-
# number of arguments for each operation
|
13
|
+
# number of arguments for each operation (to allow splitting into batches)
|
14
14
|
OPS_ARGS = {
|
15
15
|
cp: 2,
|
16
16
|
df: 0,
|
@@ -23,11 +23,11 @@ module Aspera
|
|
23
23
|
rm: 1
|
24
24
|
}.freeze
|
25
25
|
|
26
|
-
#
|
27
|
-
#
|
26
|
+
# Protocol is based on Type-Length-Value
|
27
|
+
# Type start at one, but array index start at zero
|
28
28
|
ENUM_START = 1
|
29
29
|
|
30
|
-
#
|
30
|
+
# Description of result structures (see ascmdtypes.h).
|
31
31
|
# Base types are big endian
|
32
32
|
# key = name of type
|
33
33
|
# index in array `fields` is the type (minus ENUM_START)
|
@@ -79,17 +79,25 @@ module Aspera
|
|
79
79
|
end
|
80
80
|
|
81
81
|
# execute an "as" command on a remote server
|
82
|
+
# Version 2 allows use of reverse proxy with multiple addresses.
|
82
83
|
# @param [Symbol] one of OPERATIONS
|
83
84
|
# @param [Array] parameters for "as" command
|
84
85
|
# @return result of command, type depends on command (bool, array, hash)
|
85
|
-
def execute_single(action_sym, arguments)
|
86
|
+
def execute_single(action_sym, arguments, version: 1, host: nil)
|
86
87
|
arguments = [] if arguments.nil?
|
87
88
|
Log.log.debug{"execute_single:#{action_sym}:#{arguments}"}
|
88
89
|
Aspera.assert_type(action_sym, Symbol)
|
89
90
|
Aspera.assert_type(arguments, Array)
|
90
91
|
Aspera.assert(arguments.all?(String), 'arguments must be strings')
|
92
|
+
remote_cmd = 'ascmd'
|
91
93
|
# lines of commands (String's)
|
92
94
|
command_lines = []
|
95
|
+
if version.eql?(2)
|
96
|
+
cmd = "as_session_init --protocol=#{version}"
|
97
|
+
cmd += " --host=#{host}" if host
|
98
|
+
command_lines.push(cmd)
|
99
|
+
remote_cmd += ' -V2'
|
100
|
+
end
|
93
101
|
# add "as_" command
|
94
102
|
main_command = "as_#{action_sym}"
|
95
103
|
arg_batches =
|
@@ -104,7 +112,7 @@ module Aspera
|
|
104
112
|
# enclose arguments in double quotes, protect backslash and double quotes
|
105
113
|
# ascmd uses space as token separator, and optional quotes ('") or \ to escape
|
106
114
|
args.each do |v|
|
107
|
-
command.push(%Q{"#{v.gsub(/["\\]/){|s|"\\#{s}"}}"})
|
115
|
+
command.push(%Q{"#{v.gsub(/["\\]/){ |s| "\\#{s}"}}"})
|
108
116
|
end
|
109
117
|
command_lines.push(command.join(' '))
|
110
118
|
end
|
@@ -114,7 +122,7 @@ module Aspera
|
|
114
122
|
stdin_input = command_lines.join("\n")
|
115
123
|
Log.log.trace1{"execute_single:#{stdin_input}"}
|
116
124
|
# execute, get binary output
|
117
|
-
byte_buffer = @command_executor.execute(
|
125
|
+
byte_buffer = @command_executor.execute(remote_cmd, stdin_input).unpack('C*')
|
118
126
|
raise 'ERROR: empty answer from server' if byte_buffer.empty?
|
119
127
|
# get hash or table result
|
120
128
|
result = self.class.parse(byte_buffer, :result)
|
@@ -141,7 +149,7 @@ module Aspera
|
|
141
149
|
end
|
142
150
|
# raise error as exception
|
143
151
|
raise Error.new(result[:errno], result[:errstr], action_sym, arguments) if
|
144
|
-
result.is_a?(Hash) && (result.keys.sort == TYPES_DESCR[:error][:fields].map{|i|i[:name]}.sort)
|
152
|
+
result.is_a?(Hash) && (result.keys.sort == TYPES_DESCR[:error][:fields].map{ |i| i[:name]}.sort)
|
145
153
|
return result
|
146
154
|
end
|
147
155
|
|
@@ -209,7 +217,7 @@ module Aspera
|
|
209
217
|
result[field_info[:name]] ||= []
|
210
218
|
result[field_info[:name]].push(parse(typed_buffer[:buffer], field_info[:is_a], indent_level))
|
211
219
|
when :list_tlv_list # field is an array of values in a list of buffers
|
212
|
-
result[field_info[:name]] = parse(typed_buffer[:buffer], :blist, indent_level).map{|r|parse(r[:buffer], field_info[:is_a], indent_level)}
|
220
|
+
result[field_info[:name]] = parse(typed_buffer[:buffer], :blist, indent_level).map{ |r| parse(r[:buffer], field_info[:is_a], indent_level)}
|
213
221
|
when :list_tlv_restart # field is an array of values, but a new value is started on index 1
|
214
222
|
fl = result[field_info[:name]] = []
|
215
223
|
parse(typed_buffer[:buffer], :blist, indent_level).map do |tb|
|
@@ -32,6 +32,7 @@ module Aspera
|
|
32
32
|
# Installation.instance.ascp_path=""
|
33
33
|
class Installation
|
34
34
|
include Singleton
|
35
|
+
|
35
36
|
DEFAULT_ASPERA_CONF = <<~END_OF_CONFIG_FILE
|
36
37
|
<?xml version='1.0' encoding='UTF-8'?>
|
37
38
|
<CONF version="2">
|
@@ -68,8 +69,8 @@ module Aspera
|
|
68
69
|
# set ascp executable path
|
69
70
|
def ascp_path=(v)
|
70
71
|
Aspera.assert_type(v, String)
|
71
|
-
Aspera.assert(!v.empty?)
|
72
|
-
Aspera.assert(File.exist?(v))
|
72
|
+
Aspera.assert(!v.empty?){'ascp path cannot be empty: check your config file'}
|
73
|
+
Aspera.assert(File.exist?(v)){"No such file: [#{v}]"}
|
73
74
|
@path_to_ascp = v
|
74
75
|
end
|
75
76
|
|
@@ -89,7 +90,7 @@ module Aspera
|
|
89
90
|
pl = installed_products.first
|
90
91
|
raise "no Aspera transfer module or SDK found.\nRefer to the manual or install SDK with command:\nascli conf ascp install" if pl.nil?
|
91
92
|
else
|
92
|
-
pl = installed_products.find{|i|i[:name].eql?(product_name)}
|
93
|
+
pl = installed_products.find{ |i| i[:name].eql?(product_name)}
|
93
94
|
raise "no such product installed: #{product_name}" if pl.nil?
|
94
95
|
end
|
95
96
|
self.ascp_path = pl[:ascp_path]
|
@@ -130,22 +131,21 @@ module Aspera
|
|
130
131
|
when :ssh_private_dsa, :ssh_private_rsa
|
131
132
|
# assume last 3 letters are type
|
132
133
|
type = k.to_s[-3..-1].to_sym
|
133
|
-
file = check_or_create_sdk_file("aspera_bypass_#{type}.pem")
|
134
|
+
file = check_or_create_sdk_file("aspera_bypass_#{type}.pem"){DataRepository.instance.item(type)}
|
134
135
|
when :aspera_license
|
135
|
-
file = check_or_create_sdk_file('aspera-license')
|
136
|
+
file = check_or_create_sdk_file('aspera-license'){DataRepository.instance.item(:license)}
|
136
137
|
when :aspera_conf
|
137
|
-
file = check_or_create_sdk_file('aspera.conf')
|
138
|
+
file = check_or_create_sdk_file('aspera.conf'){DEFAULT_ASPERA_CONF}
|
138
139
|
when :fallback_certificate, :fallback_private_key
|
139
140
|
file_key = File.join(Products::Transferd.sdk_directory, 'aspera_fallback_cert_private_key.pem')
|
140
141
|
file_cert = File.join(Products::Transferd.sdk_directory, 'aspera_fallback_cert.pem')
|
141
142
|
if !File.exist?(file_key) || !File.exist?(file_cert)
|
142
143
|
require 'openssl'
|
143
144
|
# create new self signed certificate for http fallback
|
144
|
-
cert = OpenSSL::X509::Certificate.new
|
145
145
|
private_key = OpenSSL::PKey::RSA.new(4096)
|
146
|
-
WebServerSimple.
|
147
|
-
check_or_create_sdk_file('aspera_fallback_cert_private_key.pem', force: true)
|
148
|
-
check_or_create_sdk_file('aspera_fallback_cert.pem', force: true)
|
146
|
+
cert = WebServerSimple.self_signed_cert(private_key)
|
147
|
+
check_or_create_sdk_file('aspera_fallback_cert_private_key.pem', force: true){private_key.to_pem}
|
148
|
+
check_or_create_sdk_file('aspera_fallback_cert.pem', force: true){cert.to_pem}
|
149
149
|
end
|
150
150
|
file = k.eql?(:fallback_certificate) ? file_cert : file_key
|
151
151
|
else Aspera.error_unexpected_value(k)
|
@@ -166,7 +166,7 @@ module Aspera
|
|
166
166
|
Aspera.assert_values(types, CLIENT_SSH_KEY_OPTIONS)
|
167
167
|
return case types
|
168
168
|
when :dsa_rsa, :rsa
|
169
|
-
types.to_s.split('_').map{|i|Installation.instance.path("ssh_private_#{i}".to_sym)}
|
169
|
+
types.to_s.split('_').map{ |i| Installation.instance.path("ssh_private_#{i}".to_sym)}
|
170
170
|
when :per_client
|
171
171
|
raise 'Not yet implemented'
|
172
172
|
end
|
@@ -197,6 +197,8 @@ module Aspera
|
|
197
197
|
last_line = ''
|
198
198
|
while (line = stderr.gets)
|
199
199
|
line.chomp!
|
200
|
+
# skip lines that may have accents
|
201
|
+
next unless line.valid_encoding?
|
200
202
|
last_line = line
|
201
203
|
case line
|
202
204
|
when /^DBG Path ([^ ]+) (dir|file) +: (.*)$/
|
@@ -212,7 +214,7 @@ module Aspera
|
|
212
214
|
data['product_name'] = Regexp.last_match(1)
|
213
215
|
data['product_version'] = Regexp.last_match(2)
|
214
216
|
when /^LOG Initializing FASP version ([^,]+),/
|
215
|
-
data['
|
217
|
+
data['ascp_version'] = Regexp.last_match(1)
|
216
218
|
end
|
217
219
|
end
|
218
220
|
if !thread.value.exitstatus.eql?(1) && !data.key?('root')
|
@@ -227,9 +229,9 @@ module Aspera
|
|
227
229
|
data = {}
|
228
230
|
File.binread(ascp_path).scan(/[\x20-\x7E]{10,}/) do |bin_string|
|
229
231
|
if (m = bin_string.match(/OPENSSLDIR.*"(.*)"/))
|
230
|
-
data['
|
232
|
+
data['ascp_openssl_dir'] = m[1]
|
231
233
|
elsif (m = bin_string.match(/OpenSSL (\d[^ -]+)/))
|
232
|
-
data['
|
234
|
+
data['ascp_openssl_version'] = m[1]
|
233
235
|
end
|
234
236
|
end if File.file?(ascp_path)
|
235
237
|
return data
|
@@ -247,10 +249,10 @@ module Aspera
|
|
247
249
|
def sdk_url_for_platform(platform: nil, version: nil)
|
248
250
|
locations = sdk_locations
|
249
251
|
platform = Environment.architecture if platform.nil?
|
250
|
-
locations = locations.select{|l|l['platform'].eql?(platform)}
|
252
|
+
locations = locations.select{ |l| l['platform'].eql?(platform)}
|
251
253
|
raise "No SDK for platform: #{platform}" if locations.empty?
|
252
|
-
version = locations.max_by
|
253
|
-
info = locations.select{|entry| entry['version'].eql?(version)}
|
254
|
+
version = locations.max_by{ |entry| Gem::Version.new(entry['version'])}['version'] if version.nil?
|
255
|
+
info = locations.select{ |entry| entry['version'].eql?(version)}
|
254
256
|
raise "No such version: #{version} for #{platform}" if info.empty?
|
255
257
|
return info.first['url']
|
256
258
|
end
|
@@ -300,7 +302,7 @@ module Aspera
|
|
300
302
|
if subfolder_lambda.nil?
|
301
303
|
# default files to extract directly to main folder if in selected source folders
|
302
304
|
subfolder_lambda = ->(name) do
|
303
|
-
Products::Transferd::RUNTIME_FOLDERS.any?{|i|name.match?(%r{^[^/]*/#{i}/})} ? '/' : nil
|
305
|
+
Products::Transferd::RUNTIME_FOLDERS.any?{ |i| name.match?(%r{^[^/]*/#{i}/})} ? '/' : nil
|
304
306
|
end
|
305
307
|
end
|
306
308
|
# rename old install
|
@@ -322,7 +324,7 @@ module Aspera
|
|
322
324
|
end
|
323
325
|
FileUtils.mkdir_p(dest_folder)
|
324
326
|
if link_target.nil?
|
325
|
-
File.open(dest_file, 'wb')
|
327
|
+
File.open(dest_file, 'wb'){ |output_stream| IO.copy_stream(entry_stream, output_stream)}
|
326
328
|
else
|
327
329
|
File.symlink(link_target, dest_file)
|
328
330
|
end
|