aspera-cli 4.17.0 → 4.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +2 -4
- data/CHANGELOG.md +23 -0
- data/CONTRIBUTING.md +15 -1
- data/README.md +620 -378
- data/bin/ascli +5 -0
- data/bin/asession +2 -2
- data/lib/aspera/agent/alpha.rb +6 -4
- data/lib/aspera/agent/base.rb +9 -6
- data/lib/aspera/agent/connect.rb +4 -4
- data/lib/aspera/agent/direct.rb +56 -37
- data/lib/aspera/agent/httpgw.rb +23 -324
- data/lib/aspera/agent/node.rb +19 -20
- data/lib/aspera/agent/trsdk.rb +19 -20
- data/lib/aspera/api/aoc.rb +17 -14
- data/lib/aspera/api/cos_node.rb +4 -4
- data/lib/aspera/api/httpgw.rb +339 -0
- data/lib/aspera/api/node.rb +34 -21
- data/lib/aspera/ascmd.rb +4 -3
- data/lib/aspera/ascp/installation.rb +15 -7
- data/lib/aspera/ascp/management.rb +2 -2
- data/lib/aspera/cli/basic_auth_plugin.rb +5 -9
- data/lib/aspera/cli/extended_value.rb +12 -6
- data/lib/aspera/cli/formatter.rb +155 -65
- data/lib/aspera/cli/hints.rb +18 -0
- data/lib/aspera/cli/main.rb +22 -29
- data/lib/aspera/cli/manager.rb +53 -36
- data/lib/aspera/cli/plugin.rb +26 -17
- data/lib/aspera/cli/plugin_factory.rb +31 -20
- data/lib/aspera/cli/plugins/alee.rb +14 -2
- data/lib/aspera/cli/plugins/aoc.rb +141 -131
- data/lib/aspera/cli/plugins/ats.rb +1 -1
- data/lib/aspera/cli/plugins/config.rb +52 -46
- data/lib/aspera/cli/plugins/console.rb +8 -5
- data/lib/aspera/cli/plugins/faspex.rb +27 -19
- data/lib/aspera/cli/plugins/faspex5.rb +222 -149
- data/lib/aspera/cli/plugins/faspio.rb +85 -0
- data/lib/aspera/cli/plugins/httpgw.rb +55 -0
- data/lib/aspera/cli/plugins/node.rb +86 -29
- data/lib/aspera/cli/plugins/orchestrator.rb +31 -29
- data/lib/aspera/cli/plugins/preview.rb +6 -2
- data/lib/aspera/cli/plugins/server.rb +5 -5
- data/lib/aspera/cli/plugins/shares.rb +16 -14
- data/lib/aspera/cli/sync_actions.rb +6 -6
- data/lib/aspera/cli/transfer_agent.rb +5 -4
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/environment.rb +7 -6
- data/lib/aspera/faspex_gw.rb +5 -4
- data/lib/aspera/faspex_postproc.rb +2 -2
- data/lib/aspera/log.rb +6 -3
- data/lib/aspera/node_simulator.rb +2 -2
- data/lib/aspera/oauth/base.rb +31 -19
- data/lib/aspera/oauth/factory.rb +12 -13
- data/lib/aspera/oauth/generic.rb +1 -0
- data/lib/aspera/oauth/jwt.rb +18 -15
- data/lib/aspera/oauth/url_json.rb +8 -6
- data/lib/aspera/open_application.rb +5 -7
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/preview/options.rb +3 -3
- data/lib/aspera/preview/terminal.rb +4 -4
- data/lib/aspera/preview/utils.rb +3 -3
- data/lib/aspera/proxy_auto_config.rb +5 -1
- data/lib/aspera/rest.rb +60 -74
- data/lib/aspera/rest_call_error.rb +1 -1
- data/lib/aspera/rest_error_analyzer.rb +2 -2
- data/lib/aspera/rest_errors_aspera.rb +1 -1
- data/lib/aspera/resumer.rb +1 -1
- data/lib/aspera/secret_hider.rb +2 -4
- data/lib/aspera/ssh.rb +1 -1
- data/lib/aspera/transfer/parameters.rb +39 -36
- data/lib/aspera/transfer/spec.rb +2 -0
- data/lib/aspera/transfer/sync.rb +2 -1
- data/lib/aspera/transfer/uri.rb +1 -1
- data/lib/aspera/uri_reader.rb +5 -4
- data/lib/aspera/web_auth.rb +1 -1
- data/lib/aspera/web_server_simple.rb +4 -3
- data.tar.gz.sig +0 -0
- metadata +5 -3
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/plugins/bss.rb +0 -71
data/lib/aspera/faspex_gw.rb
CHANGED
@@ -50,8 +50,9 @@ module Aspera
|
|
50
50
|
operation: 'POST',
|
51
51
|
subpath: "packages/#{package['id']}/transfer_spec/upload",
|
52
52
|
headers: {'Accept' => 'application/json'},
|
53
|
-
|
54
|
-
|
53
|
+
query: {transfer_type: Cli::Plugins::Faspex5::TRANSFER_CONNECT},
|
54
|
+
body: {paths: [{'destination'=>'/'}]},
|
55
|
+
body_type: :json
|
55
56
|
)[:data]
|
56
57
|
transfer_spec.delete('authentication')
|
57
58
|
# but we place it in a Faspex package creation response
|
@@ -94,5 +95,5 @@ module Aspera
|
|
94
95
|
response.body = {error: 'Unsupported endpoint'}.to_json
|
95
96
|
end
|
96
97
|
end
|
97
|
-
end
|
98
|
-
end
|
98
|
+
end
|
99
|
+
end
|
data/lib/aspera/log.rb
CHANGED
@@ -2,13 +2,14 @@
|
|
2
2
|
|
3
3
|
require 'aspera/colors'
|
4
4
|
require 'aspera/secret_hider'
|
5
|
-
require 'aspera/environment'
|
6
|
-
require 'aspera/assert'
|
7
5
|
require 'logger'
|
8
6
|
require 'pp'
|
9
7
|
require 'json'
|
10
8
|
require 'singleton'
|
11
9
|
|
10
|
+
old_verbose = $VERBOSE
|
11
|
+
$VERBOSE = nil
|
12
|
+
|
12
13
|
# extend Ruby logger with trace levels
|
13
14
|
class Logger
|
14
15
|
TRACE_MAX = 2
|
@@ -44,6 +45,8 @@ class Logger
|
|
44
45
|
Logger::Severity.constants.each { |severity| make_methods(severity) }
|
45
46
|
end
|
46
47
|
|
48
|
+
$VERBOSE = old_verbose
|
49
|
+
|
47
50
|
module Aspera
|
48
51
|
# Singleton object for logging
|
49
52
|
class Log
|
@@ -84,7 +87,7 @@ module Aspera
|
|
84
87
|
ensure
|
85
88
|
$stderr = real_stderr
|
86
89
|
end
|
87
|
-
end
|
90
|
+
end
|
88
91
|
|
89
92
|
attr_reader :logger_type, :logger
|
90
93
|
attr_writer :program_name
|
data/lib/aspera/oauth/base.rb
CHANGED
@@ -9,9 +9,7 @@ require 'date'
|
|
9
9
|
module Aspera
|
10
10
|
module OAuth
|
11
11
|
# Implement OAuth 2 for the REST client and generate a bearer token
|
12
|
-
#
|
13
|
-
# bearer tokens are kept in memory and also in a file cache for later re-use
|
14
|
-
# if a token is expired (api returns 4xx), call again get_authorization(refresh: true)
|
12
|
+
# bearer tokens are cached in memory and in a file cache for later re-use
|
15
13
|
# https://tools.ietf.org/html/rfc6749
|
16
14
|
class Base
|
17
15
|
# scope can be modified after creation
|
@@ -31,6 +29,7 @@ module Aspera
|
|
31
29
|
client_id: nil,
|
32
30
|
client_secret: nil,
|
33
31
|
scope: nil,
|
32
|
+
use_query: false,
|
34
33
|
path_token: 'token', # default endpoint for /token to generate token
|
35
34
|
token_field: 'access_token' # field with token in result of call to path_token
|
36
35
|
)
|
@@ -42,6 +41,7 @@ module Aspera
|
|
42
41
|
@client_id = client_id
|
43
42
|
@client_secret = client_secret
|
44
43
|
@scope = scope
|
44
|
+
@use_query = use_query
|
45
45
|
@identifiers = []
|
46
46
|
@identifiers.push(auth[:username]) if auth.is_a?(Hash) && auth.key?(:username)
|
47
47
|
# this is the OAuth API
|
@@ -52,13 +52,22 @@ module Aspera
|
|
52
52
|
end
|
53
53
|
|
54
54
|
# helper method to create token as per RFC
|
55
|
-
def create_token_call(
|
55
|
+
def create_token_call(creation_params)
|
56
56
|
Log.log.debug{'Generating a new token'.bg_green}
|
57
|
+
payload = {
|
58
|
+
body: creation_params,
|
59
|
+
body_type: :www
|
60
|
+
}
|
61
|
+
if @use_query
|
62
|
+
payload[:query] = creation_params
|
63
|
+
payload[:body] = {}
|
64
|
+
end
|
57
65
|
return @api.call(
|
58
|
-
operation:
|
59
|
-
subpath:
|
60
|
-
headers:
|
61
|
-
|
66
|
+
operation: 'POST',
|
67
|
+
subpath: @path_token,
|
68
|
+
headers: {'Accept' => 'application/json'},
|
69
|
+
**payload
|
70
|
+
)
|
62
71
|
end
|
63
72
|
|
64
73
|
# @return Hash with optional general parameters
|
@@ -70,24 +79,27 @@ module Aspera
|
|
70
79
|
return call_params
|
71
80
|
end
|
72
81
|
|
73
|
-
# OAuth v2 token
|
74
|
-
#
|
75
|
-
|
82
|
+
# get an OAuth v2 token (generated, cached, refreshed)
|
83
|
+
# call token() to get a token.
|
84
|
+
# if a token is expired (api returns 4xx), call again token(refresh: true)
|
85
|
+
# @param cache set to false to disable cache
|
86
|
+
# @param refresh set to true to force refresh or re-generation (if previous failed)
|
87
|
+
def token(cache: true, refresh: false)
|
76
88
|
# generate token unique identifier for persistency (memory/disk cache)
|
77
89
|
token_id = IdGenerator.from_list(Factory.id(
|
78
90
|
@base_url,
|
79
|
-
|
91
|
+
Factory.class_to_id(self.class),
|
80
92
|
@identifiers,
|
81
93
|
@scope
|
82
94
|
))
|
83
95
|
|
84
96
|
# get token_data from cache (or nil), token_data is what is returned by /token
|
85
|
-
token_data = Factory.instance.persist_mgr.get(token_id) if
|
97
|
+
token_data = Factory.instance.persist_mgr.get(token_id) if cache
|
86
98
|
token_data = JSON.parse(token_data) unless token_data.nil?
|
87
99
|
# Optional optimization: check if node token is expired based on decoded content then force refresh if close enough
|
88
100
|
# might help in case the transfer agent cannot refresh himself
|
89
101
|
# `direct` agent is equipped with refresh code
|
90
|
-
if !
|
102
|
+
if !refresh && !token_data.nil?
|
91
103
|
decoded_token = OAuth::Factory.instance.decode_token(token_data[@token_field])
|
92
104
|
Log.log.debug{Log.dump('decoded_token', decoded_token)} unless decoded_token.nil?
|
93
105
|
if decoded_token.is_a?(Hash)
|
@@ -96,13 +108,13 @@ module Aspera
|
|
96
108
|
elsif decoded_token['exp'].is_a?(Integer) then Time.at(decoded_token['exp'])
|
97
109
|
end
|
98
110
|
# force refresh if we see a token too close from expiration
|
99
|
-
|
100
|
-
Log.log.debug{"Expiration: #{expires_at_sec} / #{
|
111
|
+
refresh = true if expires_at_sec.is_a?(Time) && (expires_at_sec - Time.now) < OAuth::Factory.instance.parameters[:token_expiration_guard_sec]
|
112
|
+
Log.log.debug{"Expiration: #{expires_at_sec} / #{refresh}"}
|
101
113
|
end
|
102
114
|
end
|
103
115
|
|
104
116
|
# an API was already called, but failed, we need to regenerate or refresh
|
105
|
-
if
|
117
|
+
if refresh
|
106
118
|
if token_data.is_a?(Hash) && token_data.key?('refresh_token') && !token_data['refresh_token'].eql?('not_supported')
|
107
119
|
# save possible refresh token, before deleting the cache
|
108
120
|
refresh_token = token_data['refresh_token']
|
@@ -133,11 +145,11 @@ module Aspera
|
|
133
145
|
json_data = resp[:http].body
|
134
146
|
token_data = JSON.parse(json_data)
|
135
147
|
Factory.instance.persist_mgr.put(token_id, json_data)
|
136
|
-
end
|
148
|
+
end
|
137
149
|
Aspera.assert(token_data.key?(@token_field)){"API error: No such field in answer: #{@token_field}"}
|
138
150
|
# ok we shall have a token here
|
139
151
|
return OAuth::Factory.bearer_build(token_data[@token_field])
|
140
152
|
end
|
141
|
-
end
|
153
|
+
end
|
142
154
|
end
|
143
155
|
end
|
data/lib/aspera/oauth/factory.rb
CHANGED
@@ -5,6 +5,7 @@ require 'aspera/assert'
|
|
5
5
|
require 'base64'
|
6
6
|
module Aspera
|
7
7
|
module OAuth
|
8
|
+
# Factory to create tokens and manage their cache
|
8
9
|
class Factory
|
9
10
|
include Singleton
|
10
11
|
# a prefix for persistency of tokens (simplify garbage collect)
|
@@ -16,7 +17,11 @@ module Aspera
|
|
16
17
|
|
17
18
|
class << self
|
18
19
|
def bearer_build(token)
|
19
|
-
return BEARER_PREFIX
|
20
|
+
return "#{BEARER_PREFIX}#{token}"
|
21
|
+
end
|
22
|
+
|
23
|
+
def bearer?(token)
|
24
|
+
return token.start_with?(BEARER_PREFIX)
|
20
25
|
end
|
21
26
|
|
22
27
|
def bearer_extract(token)
|
@@ -24,14 +29,11 @@ module Aspera
|
|
24
29
|
return token[BEARER_PREFIX.length..-1]
|
25
30
|
end
|
26
31
|
|
27
|
-
def bearer?(token)
|
28
|
-
return token.start_with?(BEARER_PREFIX)
|
29
|
-
end
|
30
|
-
|
31
32
|
def id(*params)
|
32
33
|
return [PERSIST_CATEGORY_TOKEN, *params].flatten
|
33
34
|
end
|
34
35
|
|
36
|
+
# snake version of class name is the identifier
|
35
37
|
def class_to_id(creator_class)
|
36
38
|
return creator_class.name.split('::').last.capital_to_snake.to_sym
|
37
39
|
end
|
@@ -45,11 +47,8 @@ module Aspera
|
|
45
47
|
# token creation methods
|
46
48
|
@token_type_classes = {}
|
47
49
|
@decoders = []
|
48
|
-
|
49
|
-
|
50
|
-
jwt_accepted_offset_sec: 300,
|
51
|
-
# one hour validity (TODO: configurable?)
|
52
|
-
jwt_expiry_offset_sec: 3600,
|
50
|
+
# default parameters, others can be added by handlers
|
51
|
+
@parameters = {
|
53
52
|
# tokens older than 30 minutes will be discarded from cache
|
54
53
|
token_cache_expiry_sec: 1800,
|
55
54
|
# tokens valid for less than this duration will be regenerated
|
@@ -59,12 +58,12 @@ module Aspera
|
|
59
58
|
|
60
59
|
public
|
61
60
|
|
62
|
-
attr_reader :
|
61
|
+
attr_reader :parameters
|
63
62
|
|
64
63
|
def persist_mgr=(manager)
|
65
64
|
@persist = manager
|
66
65
|
# cleanup expired tokens
|
67
|
-
@persist.garbage_collect(PERSIST_CATEGORY_TOKEN, @
|
66
|
+
@persist.garbage_collect(PERSIST_CATEGORY_TOKEN, @parameters[:token_cache_expiry_sec])
|
68
67
|
end
|
69
68
|
|
70
69
|
def persist_mgr
|
@@ -104,7 +103,7 @@ module Aspera
|
|
104
103
|
# @param id_create called to generate unique id for token, for cache
|
105
104
|
def register_token_creator(creator_class)
|
106
105
|
Aspera.assert_type(creator_class, Class)
|
107
|
-
id =
|
106
|
+
id = Factory.class_to_id(creator_class)
|
108
107
|
Log.log.debug{"registering token creator #{id}"}
|
109
108
|
@token_type_classes[id] = creator_class
|
110
109
|
end
|
data/lib/aspera/oauth/generic.rb
CHANGED
data/lib/aspera/oauth/jwt.rb
CHANGED
@@ -5,41 +5,44 @@ require 'aspera/assert'
|
|
5
5
|
require 'securerandom'
|
6
6
|
module Aspera
|
7
7
|
module OAuth
|
8
|
+
# remove 5 minutes to account for time offset between client and server (TODO: configurable?)
|
9
|
+
Factory.instance.parameters[:jwt_accepted_offset_sec] = 300
|
10
|
+
# one hour validity (TODO: configurable?)
|
11
|
+
Factory.instance.parameters[:jwt_expiry_offset_sec] = 3600
|
8
12
|
# Authentication using private key
|
13
|
+
# https://tools.ietf.org/html/rfc7523
|
14
|
+
# https://tools.ietf.org/html/rfc7519
|
9
15
|
class Jwt < Base
|
10
|
-
# @param
|
11
|
-
# @param
|
12
|
-
# @param
|
16
|
+
# @param private_key_obj private key object
|
17
|
+
# @param payload payload to be included in the JWT
|
18
|
+
# @param headers headers to be included in the JWT
|
13
19
|
def initialize(
|
14
|
-
payload:,
|
15
20
|
private_key_obj:,
|
21
|
+
payload:,
|
16
22
|
headers: {},
|
17
23
|
**base_params
|
18
24
|
)
|
19
|
-
Aspera.assert_type(payload, Hash){'payload'}
|
20
25
|
Aspera.assert_type(private_key_obj, OpenSSL::PKey::RSA){'private_key_obj'}
|
26
|
+
Aspera.assert_type(payload, Hash){'payload'}
|
21
27
|
Aspera.assert_type(headers, Hash){'headers'}
|
22
28
|
super(**base_params)
|
23
29
|
@private_key_obj = private_key_obj
|
24
|
-
@
|
30
|
+
@additional_payload = payload
|
25
31
|
@headers = headers
|
26
|
-
@identifiers.push(@
|
32
|
+
@identifiers.push(@additional_payload[:sub])
|
27
33
|
end
|
28
34
|
|
29
35
|
def create_token
|
30
|
-
# https://tools.ietf.org/html/rfc7523
|
31
|
-
# https://tools.ietf.org/html/rfc7519
|
32
36
|
require 'jwt'
|
33
37
|
seconds_since_epoch = Time.new.to_i
|
34
38
|
Log.log.info{"seconds=#{seconds_since_epoch}"}
|
35
|
-
Aspera.assert(@payload.is_a?(Hash)){'missing JWT payload'}
|
36
39
|
jwt_payload = {
|
37
|
-
exp: seconds_since_epoch + OAuth::Factory.instance.
|
38
|
-
nbf: seconds_since_epoch - OAuth::Factory.instance.
|
39
|
-
iat: seconds_since_epoch - OAuth::Factory.instance.
|
40
|
+
exp: seconds_since_epoch + OAuth::Factory.instance.parameters[:jwt_expiry_offset_sec], # expiration time
|
41
|
+
nbf: seconds_since_epoch - OAuth::Factory.instance.parameters[:jwt_accepted_offset_sec], # not before
|
42
|
+
iat: seconds_since_epoch - OAuth::Factory.instance.parameters[:jwt_accepted_offset_sec] + 1, # issued at
|
40
43
|
jti: SecureRandom.uuid # JWT id
|
41
|
-
}.merge(@
|
42
|
-
Log.log.debug{
|
44
|
+
}.merge(@additional_payload)
|
45
|
+
Log.log.debug{Log.dump(:jwt_payload, jwt_payload)}
|
43
46
|
Log.log.debug{"private=[#{@private_key_obj}]"}
|
44
47
|
assertion = JWT.encode(jwt_payload, @private_key_obj, 'RS256', @headers)
|
45
48
|
Log.log.debug{"assertion=[#{assertion}]"}
|
@@ -4,16 +4,17 @@ require 'aspera/oauth/base'
|
|
4
4
|
|
5
5
|
module Aspera
|
6
6
|
module OAuth
|
7
|
+
# This class is used to create a token using a JSON body and a URL
|
7
8
|
class UrlJson < Base
|
8
9
|
def initialize(
|
9
|
-
json:,
|
10
10
|
url:,
|
11
|
+
json:,
|
11
12
|
**generic_params
|
12
13
|
)
|
13
14
|
super(**generic_params)
|
14
|
-
@
|
15
|
-
@
|
16
|
-
@identifiers.push(@
|
15
|
+
@body = json
|
16
|
+
@query = url
|
17
|
+
@identifiers.push(@body[:url_token])
|
17
18
|
end
|
18
19
|
|
19
20
|
def create_token
|
@@ -21,8 +22,9 @@ module Aspera
|
|
21
22
|
operation: 'POST',
|
22
23
|
subpath: @path_token,
|
23
24
|
headers: {'Accept' => 'application/json'},
|
24
|
-
|
25
|
-
|
25
|
+
query: @query.merge(scope: @scope), # scope is here because it may change over time (node)
|
26
|
+
body: @body,
|
27
|
+
body_type: :json
|
26
28
|
)
|
27
29
|
end
|
28
30
|
end
|
@@ -11,12 +11,10 @@ module Aspera
|
|
11
11
|
# if method is "graphical", then the URL will be opened with the default browser.
|
12
12
|
class OpenApplication
|
13
13
|
include Singleton
|
14
|
+
USER_INTERFACES = %i[text graphical].freeze
|
14
15
|
class << self
|
15
|
-
USER_INTERFACES = %i[text graphical].freeze
|
16
|
-
# User Interfaces
|
17
|
-
def user_interfaces; USER_INTERFACES; end
|
18
|
-
|
19
16
|
def default_gui_mode
|
17
|
+
# assume not remotely connected on macos and windows
|
20
18
|
return :graphical if [Environment::OS_WINDOWS, Environment::OS_X].include?(Environment.os)
|
21
19
|
# unix family
|
22
20
|
return :graphical if ENV.key?('DISPLAY') && !ENV['DISPLAY'].empty?
|
@@ -43,7 +41,7 @@ module Aspera
|
|
43
41
|
uri_graphical(file_path.to_s)
|
44
42
|
end
|
45
43
|
end
|
46
|
-
end
|
44
|
+
end
|
47
45
|
|
48
46
|
attr_accessor :url_method
|
49
47
|
|
@@ -67,5 +65,5 @@ module Aspera
|
|
67
65
|
raise StandardError, "unsupported url open method: #{@url_method}"
|
68
66
|
end
|
69
67
|
end
|
70
|
-
end
|
71
|
-
end
|
68
|
+
end
|
69
|
+
end
|
data/lib/aspera/preview/utils.rb
CHANGED
@@ -36,7 +36,11 @@ module Aspera
|
|
36
36
|
def pac_dns_functions(context_host)
|
37
37
|
context_self = '127.0.0.1'
|
38
38
|
context_ip = nil
|
39
|
-
Resolv::DNS.open
|
39
|
+
Resolv::DNS.open do |dns|
|
40
|
+
dns.each_address(context_host) do |r_addr|
|
41
|
+
context_ip = r_addr.to_s if r_addr.is_a?(Resolv::IPv4)
|
42
|
+
end
|
43
|
+
end
|
40
44
|
raise "DNS name not found: #{context_host}" if context_ip.nil?
|
41
45
|
# NOTE: Javascript code here with string inclusions
|
42
46
|
javascript = <<END_OF_JAVASCRIPT
|