aspera-cli 4.15.0 → 4.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +29 -3
- data/CHANGELOG.md +375 -280
- data/CONTRIBUTING.md +71 -18
- data/README.md +1978 -1656
- data/bin/ascli +13 -31
- data/bin/asession +32 -22
- data/examples/dascli +2 -2
- data/lib/aspera/agent/alpha.rb +117 -0
- data/lib/aspera/agent/base.rb +61 -0
- data/lib/aspera/{fasp/agent_connect.rb → agent/connect.rb} +13 -11
- data/lib/aspera/{fasp/agent_direct.rb → agent/direct.rb} +116 -116
- data/lib/aspera/{fasp/agent_httpgw.rb → agent/httpgw.rb} +21 -19
- data/lib/aspera/{fasp/agent_node.rb → agent/node.rb} +21 -33
- data/lib/aspera/agent/trsdk.rb +188 -0
- data/lib/aspera/api/aoc.rb +586 -0
- data/lib/aspera/api/ats.rb +46 -0
- data/lib/aspera/api/cos_node.rb +95 -0
- data/lib/aspera/api/node.rb +344 -0
- data/lib/aspera/ascmd.rb +47 -14
- data/lib/aspera/{fasp → ascp}/installation.rb +54 -15
- data/lib/aspera/{fasp → ascp}/management.rb +14 -14
- data/lib/aspera/{fasp → ascp}/products.rb +1 -1
- data/lib/aspera/assert.rb +45 -0
- data/lib/aspera/cli/basic_auth_plugin.rb +11 -10
- data/lib/aspera/cli/extended_value.rb +5 -5
- data/lib/aspera/cli/formatter.rb +27 -14
- data/lib/aspera/cli/hints.rb +7 -6
- data/lib/aspera/cli/main.rb +49 -29
- data/lib/aspera/cli/manager.rb +46 -36
- data/lib/aspera/cli/plugin.rb +34 -20
- data/lib/aspera/cli/plugin_factory.rb +61 -0
- data/lib/aspera/cli/plugins/alee.rb +7 -7
- data/lib/aspera/cli/plugins/aoc.rb +168 -132
- data/lib/aspera/cli/plugins/ats.rb +33 -33
- data/lib/aspera/cli/plugins/bss.rb +3 -4
- data/lib/aspera/cli/plugins/config.rb +250 -272
- data/lib/aspera/cli/plugins/console.rb +8 -6
- data/lib/aspera/cli/plugins/cos.rb +20 -19
- data/lib/aspera/cli/plugins/faspex.rb +71 -60
- data/lib/aspera/cli/plugins/faspex5.rb +212 -133
- data/lib/aspera/cli/plugins/node.rb +83 -75
- data/lib/aspera/cli/plugins/orchestrator.rb +36 -44
- data/lib/aspera/cli/plugins/preview.rb +33 -31
- data/lib/aspera/cli/plugins/server.rb +33 -32
- data/lib/aspera/cli/plugins/shares.rb +39 -33
- data/lib/aspera/cli/sync_actions.rb +9 -9
- data/lib/aspera/cli/transfer_agent.rb +45 -25
- data/lib/aspera/cli/transfer_progress.rb +2 -3
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +5 -0
- data/lib/aspera/command_line_builder.rb +16 -14
- data/lib/aspera/coverage.rb +21 -0
- data/lib/aspera/data_repository.rb +33 -2
- data/lib/aspera/environment.rb +5 -4
- data/lib/aspera/faspex_gw.rb +13 -11
- data/lib/aspera/faspex_postproc.rb +6 -5
- data/lib/aspera/id_generator.rb +4 -2
- data/lib/aspera/json_rpc.rb +10 -8
- data/lib/aspera/keychain/encrypted_hash.rb +46 -11
- data/lib/aspera/keychain/macos_security.rb +29 -22
- data/lib/aspera/log.rb +5 -4
- data/lib/aspera/nagios.rb +7 -2
- data/lib/aspera/node_simulator.rb +213 -0
- data/lib/aspera/oauth/base.rb +143 -0
- data/lib/aspera/oauth/factory.rb +124 -0
- data/lib/aspera/oauth/generic.rb +34 -0
- data/lib/aspera/oauth/jwt.rb +51 -0
- data/lib/aspera/oauth/url_json.rb +31 -0
- data/lib/aspera/oauth/web.rb +50 -0
- data/lib/aspera/oauth.rb +5 -328
- data/lib/aspera/open_application.rb +7 -7
- data/lib/aspera/persistency_action_once.rb +13 -14
- data/lib/aspera/persistency_folder.rb +3 -2
- data/lib/aspera/preview/file_types.rb +53 -267
- data/lib/aspera/preview/generator.rb +7 -5
- data/lib/aspera/preview/terminal.rb +17 -7
- data/lib/aspera/preview/utils.rb +8 -7
- data/lib/aspera/proxy_auto_config.rb +6 -3
- data/lib/aspera/rest.rb +187 -140
- data/lib/aspera/rest_error_analyzer.rb +1 -0
- data/lib/aspera/rest_errors_aspera.rb +5 -3
- data/lib/aspera/resumer.rb +77 -0
- data/lib/aspera/secret_hider.rb +5 -2
- data/lib/aspera/ssh.rb +15 -8
- data/lib/aspera/temp_file_manager.rb +1 -1
- data/lib/aspera/{fasp → transfer}/error.rb +3 -3
- data/lib/aspera/{fasp → transfer}/error_info.rb +1 -1
- data/lib/aspera/{fasp → transfer}/faux_file.rb +1 -1
- data/lib/aspera/{fasp → transfer}/parameters.rb +95 -120
- data/lib/aspera/{fasp/transfer_spec.rb → transfer/spec.rb} +23 -19
- data/lib/aspera/{fasp/parameters.yaml → transfer/spec.yaml} +4 -99
- data/lib/aspera/transfer/sync.rb +273 -0
- data/lib/aspera/{fasp → transfer}/uri.rb +10 -9
- data/lib/aspera/web_server_simple.rb +12 -3
- data.tar.gz.sig +0 -0
- metadata +92 -68
- metadata.gz.sig +0 -0
- data/lib/aspera/aoc.rb +0 -606
- data/lib/aspera/ats_api.rb +0 -47
- data/lib/aspera/cos_node.rb +0 -93
- data/lib/aspera/fasp/agent_aspera.rb +0 -126
- data/lib/aspera/fasp/agent_base.rb +0 -48
- data/lib/aspera/fasp/agent_trsdk.rb +0 -146
- data/lib/aspera/fasp/resume_policy.rb +0 -77
- data/lib/aspera/node.rb +0 -338
- data/lib/aspera/sync.rb +0 -219
|
@@ -2,6 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
# https://github.com/fastlane-community/security
|
|
4
4
|
require 'aspera/cli/info'
|
|
5
|
+
require 'aspera/log'
|
|
6
|
+
require 'aspera/assert'
|
|
7
|
+
require 'open3'
|
|
5
8
|
|
|
6
9
|
# enhance the gem to support other key chains
|
|
7
10
|
module Aspera
|
|
@@ -9,6 +12,8 @@ module Aspera
|
|
|
9
12
|
module MacosSecurity
|
|
10
13
|
# keychain based on macOS keychain, using `security` command line
|
|
11
14
|
class Keychain
|
|
15
|
+
# https://www.unix.com/man-page/osx/1/security/
|
|
16
|
+
SECURITY_UTILITY = 'security'
|
|
12
17
|
DOMAINS = %i[user system common dynamic].freeze
|
|
13
18
|
LIST_OPTIONS = {
|
|
14
19
|
domain: :c
|
|
@@ -36,25 +41,27 @@ module Aspera
|
|
|
36
41
|
url = options&.delete(:url)
|
|
37
42
|
if !url.nil?
|
|
38
43
|
uri = URI.parse(url)
|
|
39
|
-
|
|
44
|
+
Aspera.assert(uri.scheme.eql?('https')){'only https'}
|
|
40
45
|
options[:protocol] = 'htps' # cspell: disable-line
|
|
41
46
|
raise 'host required in URL' if uri.host.nil?
|
|
42
47
|
options[:server] = uri.host
|
|
43
48
|
options[:path] = uri.path unless ['', '/'].include?(uri.path)
|
|
44
49
|
options[:port] = uri.port unless uri.port.eql?(443) && !url.include?(':443/')
|
|
45
50
|
end
|
|
46
|
-
|
|
51
|
+
command_line = [SECURITY_UTILITY, command]
|
|
47
52
|
options&.each do |k, v|
|
|
48
|
-
|
|
53
|
+
Aspera.assert(supported.key?(k)){"unknown option: #{k}"}
|
|
49
54
|
next if v.nil?
|
|
50
|
-
|
|
51
|
-
|
|
55
|
+
command_line.push("-#{supported[k]}")
|
|
56
|
+
command_line.push(v.shellescape) unless v.empty?
|
|
52
57
|
end
|
|
53
|
-
|
|
54
|
-
Log.log.debug{"executing>>#{
|
|
55
|
-
|
|
56
|
-
Log.log.debug{"
|
|
57
|
-
|
|
58
|
+
command_line.push(last_opt) unless last_opt.nil?
|
|
59
|
+
Log.log.debug{"executing>>#{command_line.join(' ')}"}
|
|
60
|
+
stdout, stderr, status = Open3.capture3(*command_line)
|
|
61
|
+
Log.log.debug{"status=#{status}, stderr=#{stderr}"}
|
|
62
|
+
Log.log.trace1{"stdout=#{stdout}"}
|
|
63
|
+
raise "#{SECURITY_UTILITY} failed: #{status.exitstatus} : #{stderr}" unless status.success?
|
|
64
|
+
return stdout
|
|
58
65
|
end
|
|
59
66
|
|
|
60
67
|
def key_chains(output)
|
|
@@ -70,7 +77,7 @@ module Aspera
|
|
|
70
77
|
end
|
|
71
78
|
|
|
72
79
|
def list(options={})
|
|
73
|
-
|
|
80
|
+
Aspera.assert_values(options[:domain], DOMAINS, exception_class: ArgumentError){'domain'} unless options[:domain].nil?
|
|
74
81
|
key_chains(execute('list-key_chains', options, LIST_OPTIONS))
|
|
75
82
|
end
|
|
76
83
|
|
|
@@ -89,15 +96,15 @@ module Aspera
|
|
|
89
96
|
end
|
|
90
97
|
|
|
91
98
|
def password(operation, pass_type, options)
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
99
|
+
Aspera.assert_values(operation, %i[add find delete]){'operation'}
|
|
100
|
+
Aspera.assert_values(pass_type, %i[generic internet]){'pass_type'}
|
|
101
|
+
Aspera.assert_type(options, Hash)
|
|
95
102
|
missing = (operation.eql?(:add) ? %i[account service password] : %i[label]) - options.keys
|
|
96
|
-
|
|
103
|
+
Aspera.assert(missing.empty?){"missing options: #{missing}"}
|
|
97
104
|
options[:getpass] = '' if operation.eql?(:find)
|
|
98
105
|
output = self.class.execute("#{operation}-#{pass_type}-password", options, ADD_PASS_OPTIONS, @path)
|
|
99
106
|
raise output.gsub(/^.*: /, '') if output.start_with?('security: ')
|
|
100
|
-
return
|
|
107
|
+
return unless operation.eql?(:find)
|
|
101
108
|
attributes = {}
|
|
102
109
|
output.split("\n").each do |line|
|
|
103
110
|
case line
|
|
@@ -127,18 +134,18 @@ module Aspera
|
|
|
127
134
|
end
|
|
128
135
|
|
|
129
136
|
def set(options)
|
|
130
|
-
|
|
137
|
+
Aspera.assert_type(options, Hash){'options'}
|
|
131
138
|
unsupported = options.keys - %i[label username password url description]
|
|
132
|
-
|
|
139
|
+
Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}"}
|
|
133
140
|
@keychain.password(
|
|
134
141
|
:add, :generic, service: options[:label],
|
|
135
142
|
account: options[:username] || 'none', password: options[:password], comment: options[:description])
|
|
136
143
|
end
|
|
137
144
|
|
|
138
145
|
def get(options)
|
|
139
|
-
|
|
146
|
+
Aspera.assert_type(options, Hash){'options'}
|
|
140
147
|
unsupported = options.keys - %i[label]
|
|
141
|
-
|
|
148
|
+
Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}"}
|
|
142
149
|
info = @keychain.password(:find, :generic, label: options[:label])
|
|
143
150
|
raise 'not found' if info.nil?
|
|
144
151
|
result = options.clone
|
|
@@ -153,9 +160,9 @@ module Aspera
|
|
|
153
160
|
end
|
|
154
161
|
|
|
155
162
|
def delete(options)
|
|
156
|
-
|
|
163
|
+
Aspera.assert_type(options, Hash){'options'}
|
|
157
164
|
unsupported = options.keys - %i[label]
|
|
158
|
-
|
|
165
|
+
Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}"}
|
|
159
166
|
raise 'delete not implemented, use macos keychain app'
|
|
160
167
|
end
|
|
161
168
|
end
|
data/lib/aspera/log.rb
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require 'aspera/colors'
|
|
4
4
|
require 'aspera/secret_hider'
|
|
5
5
|
require 'aspera/environment'
|
|
6
|
+
require 'aspera/assert'
|
|
6
7
|
require 'logger'
|
|
7
8
|
require 'pp'
|
|
8
9
|
require 'json'
|
|
@@ -13,7 +14,7 @@ class Logger
|
|
|
13
14
|
TRACE_MAX = 2
|
|
14
15
|
# add custom level to logger severity
|
|
15
16
|
module Severity
|
|
16
|
-
1.upto(TRACE_MAX).each { |level| const_set("TRACE#{level}", - level)}
|
|
17
|
+
1.upto(TRACE_MAX).each { |level| const_set(:"TRACE#{level}", - level)}
|
|
17
18
|
end
|
|
18
19
|
# quick access to label
|
|
19
20
|
SEVERITY_LABEL = Severity.constants.each_with_object({}) { |name, hash| hash[Severity.const_get(name)] = name}
|
|
@@ -39,6 +40,7 @@ class Logger
|
|
|
39
40
|
end
|
|
40
41
|
EOM
|
|
41
42
|
end
|
|
43
|
+
# declare methods for all levels
|
|
42
44
|
Logger::Severity.constants.each { |severity| make_methods(severity) }
|
|
43
45
|
end
|
|
44
46
|
|
|
@@ -97,8 +99,7 @@ module Aspera
|
|
|
97
99
|
Logger::Severity.constants.each do |name|
|
|
98
100
|
return name.downcase.to_sym if @logger.level.eql?(Logger::Severity.const_get(name))
|
|
99
101
|
end
|
|
100
|
-
|
|
101
|
-
raise "INTERNAL ERROR: unexpected level #{@logger.level}"
|
|
102
|
+
Aspera.error_unexpected_value(@logger.level){'log level'}
|
|
102
103
|
end
|
|
103
104
|
|
|
104
105
|
# change underlying logger, but keep log level
|
|
@@ -123,7 +124,7 @@ module Aspera
|
|
|
123
124
|
end
|
|
124
125
|
@logger = Syslog::Logger.new(@program_name, Syslog::LOG_LOCAL2)
|
|
125
126
|
else
|
|
126
|
-
raise "unknown log type: #{new_log_type
|
|
127
|
+
raise "unknown log type: #{new_log_type}, use one of: #{LOG_TYPES.join(', ')}"
|
|
127
128
|
end
|
|
128
129
|
@logger.level = current_severity_integer
|
|
129
130
|
@logger_type = new_log_type
|
data/lib/aspera/nagios.rb
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'aspera/log'
|
|
4
|
+
require 'aspera/assert'
|
|
3
5
|
require 'date'
|
|
4
6
|
|
|
5
7
|
module Aspera
|
|
@@ -21,8 +23,11 @@ module Aspera
|
|
|
21
23
|
class << self
|
|
22
24
|
# process results of a analysis and display status and exit with code
|
|
23
25
|
def process(data)
|
|
24
|
-
|
|
25
|
-
|
|
26
|
+
Aspera.assert_type(data, Array)
|
|
27
|
+
Aspera.assert(!data.empty?){'data is empty'}
|
|
28
|
+
%w[status component message].each do |c|
|
|
29
|
+
Aspera.assert(data.first.key?(c)){"result must have #{c}"}
|
|
30
|
+
end
|
|
26
31
|
res_errors = data.reject{|s|s['status'].eql?('ok')}
|
|
27
32
|
# keep only errors in case of problem, other ok are assumed so
|
|
28
33
|
data = res_errors unless res_errors.empty?
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'aspera/ascp/installation'
|
|
4
|
+
require 'aspera/agent/direct'
|
|
5
|
+
require 'aspera/log'
|
|
6
|
+
require 'webrick'
|
|
7
|
+
require 'json'
|
|
8
|
+
|
|
9
|
+
module Aspera
|
|
10
|
+
# this class answers the Faspex /send API and creates a package on Aspera on Cloud
|
|
11
|
+
class NodeSimulatorServlet < WEBrick::HTTPServlet::AbstractServlet
|
|
12
|
+
PATH_TRANSFERS = '/ops/transfers'
|
|
13
|
+
PATH_ONE_TRANSFER = %r{/ops/transfers/(.+)$}
|
|
14
|
+
# @param app_api [Api::AoC]
|
|
15
|
+
# @param app_context [String]
|
|
16
|
+
def initialize(server, credentials, transfer)
|
|
17
|
+
super(server)
|
|
18
|
+
@credentials = credentials
|
|
19
|
+
@xfer_manager = Agent::Direct.new
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def do_POST(request, response)
|
|
23
|
+
case request.path
|
|
24
|
+
when PATH_TRANSFERS
|
|
25
|
+
job_id = @xfer_manager.start_transfer(JSON.parse(request.body))
|
|
26
|
+
session = @xfer_manager.sessions_by_job(job_id).first
|
|
27
|
+
result = session[:ts].clone
|
|
28
|
+
result['id'] = job_id
|
|
29
|
+
set_json_response(response, result)
|
|
30
|
+
else
|
|
31
|
+
set_json_response(response, [{error: 'Bad request'}], code: 400)
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def do_GET(request, response)
|
|
36
|
+
case request.path
|
|
37
|
+
when '/info'
|
|
38
|
+
info = Ascp::Installation.instance.ascp_info
|
|
39
|
+
set_json_response(response, {
|
|
40
|
+
application: 'node',
|
|
41
|
+
current_time: Time.now.utc.iso8601(0),
|
|
42
|
+
version: info['ascp_version'].gsub(/ .*$/, ''),
|
|
43
|
+
license_expiration_date: info['expiration_date'],
|
|
44
|
+
license_max_rate: info['maximum_bandwidth'],
|
|
45
|
+
os: %x(uname -srv).chomp,
|
|
46
|
+
aej_status: 'disconnected',
|
|
47
|
+
async_reporting: 'yes',
|
|
48
|
+
transfer_activity_reporting: 'yes',
|
|
49
|
+
transfer_user: 'xfer',
|
|
50
|
+
docroot: 'file:////data/aoc/eudemo-sedemo',
|
|
51
|
+
node_id: '2bbdcc39-f789-4d47-8163-6767fc14f421',
|
|
52
|
+
cluster_id: '6dae2844-d1a9-47a5-916d-9b3eac3ea466',
|
|
53
|
+
acls: [],
|
|
54
|
+
access_key_configuration_capabilities: {
|
|
55
|
+
transfer: %w[
|
|
56
|
+
cipher
|
|
57
|
+
policy
|
|
58
|
+
target_rate_cap_kbps
|
|
59
|
+
target_rate_kbps
|
|
60
|
+
preserve_timestamps
|
|
61
|
+
content_protection_secret
|
|
62
|
+
aggressiveness
|
|
63
|
+
token_encryption_key
|
|
64
|
+
byok_enabled
|
|
65
|
+
bandwidth_flow_network_rc_module
|
|
66
|
+
file_checksum_type],
|
|
67
|
+
server: %w[
|
|
68
|
+
activity_event_logging
|
|
69
|
+
activity_file_event_logging
|
|
70
|
+
recursive_counts
|
|
71
|
+
aej_logging
|
|
72
|
+
wss_enabled
|
|
73
|
+
activity_transfer_ignore_skipped_files
|
|
74
|
+
activity_files_max
|
|
75
|
+
access_key_credentials_encryption_type
|
|
76
|
+
discovery
|
|
77
|
+
auto_delete
|
|
78
|
+
allow
|
|
79
|
+
deny]
|
|
80
|
+
},
|
|
81
|
+
capabilities: [
|
|
82
|
+
{name: 'sync', value: true},
|
|
83
|
+
{name: 'watchfolder', value: true},
|
|
84
|
+
{name: 'symbolic_links', value: true},
|
|
85
|
+
{name: 'move_file', value: true},
|
|
86
|
+
{name: 'move_directory', value: true},
|
|
87
|
+
{name: 'filelock', value: false},
|
|
88
|
+
{name: 'ssh_fingerprint', value: false},
|
|
89
|
+
{name: 'aej_version', value: '1.0'},
|
|
90
|
+
{name: 'page', value: true},
|
|
91
|
+
{name: 'file_id_version', value: '2.0'},
|
|
92
|
+
{name: 'auto_delete', value: false}],
|
|
93
|
+
settings: [
|
|
94
|
+
{name: 'content_protection_required', value: false},
|
|
95
|
+
{name: 'content_protection_strong_pass_required', value: false},
|
|
96
|
+
{name: 'filelock_restriction', value: 'none'},
|
|
97
|
+
{name: 'ssh_fingerprint', value: nil},
|
|
98
|
+
{name: 'wss_enabled', value: false},
|
|
99
|
+
{name: 'wss_port', value: 443}
|
|
100
|
+
]})
|
|
101
|
+
when PATH_TRANSFERS
|
|
102
|
+
result = @xfer_manager.sessions.map { |session| job_to_transfer(session) }
|
|
103
|
+
set_json_response(response, result)
|
|
104
|
+
when PATH_ONE_TRANSFER
|
|
105
|
+
job_id = request.path.match(PATH_ONE_TRANSFER)[1]
|
|
106
|
+
set_json_response(response, job_to_transfer(@xfer_manager.sessions_by_job(job_id).first))
|
|
107
|
+
else
|
|
108
|
+
set_json_response(response, [{error: 'Unknown request'}], code: 400)
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def set_json_response(response, json, code: 200)
|
|
113
|
+
response.status = code
|
|
114
|
+
response['Content-Type'] = 'application/json'
|
|
115
|
+
response.body = json.to_json
|
|
116
|
+
Log.log.trace1{Log.dump('response', json)}
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def job_to_transfer(job)
|
|
120
|
+
session = {
|
|
121
|
+
id: 'bafc72b8-366c-4501-8095-47208183d6b8',
|
|
122
|
+
client_node_id: '',
|
|
123
|
+
server_node_id: '2bbdcc39-f789-4d47-8163-6767fc14f421',
|
|
124
|
+
client_ip_address: '192.168.0.100',
|
|
125
|
+
server_ip_address: '5.10.114.4',
|
|
126
|
+
status: 'running',
|
|
127
|
+
retry_timeout: 3600,
|
|
128
|
+
retry_count: 0,
|
|
129
|
+
start_time_usec: 1701094040000000,
|
|
130
|
+
end_time_usec: nil,
|
|
131
|
+
elapsed_usec: 405312,
|
|
132
|
+
bytes_transferred: 26,
|
|
133
|
+
bytes_written: 26,
|
|
134
|
+
bytes_lost: 0,
|
|
135
|
+
files_completed: 1,
|
|
136
|
+
directories_completed: 0,
|
|
137
|
+
target_rate_kbps: 500000,
|
|
138
|
+
min_rate_kbps: 0,
|
|
139
|
+
calc_rate_kbps: 9900,
|
|
140
|
+
network_delay_usec: 40000,
|
|
141
|
+
avg_rate_kbps: 0.51,
|
|
142
|
+
error_code: 0,
|
|
143
|
+
error_desc: '',
|
|
144
|
+
source_statistics: {
|
|
145
|
+
args_scan_attempted: 1,
|
|
146
|
+
args_scan_completed: 1,
|
|
147
|
+
paths_scan_attempted: 1,
|
|
148
|
+
paths_scan_failed: 0,
|
|
149
|
+
paths_scan_skipped: 0,
|
|
150
|
+
paths_scan_excluded: 0,
|
|
151
|
+
dirs_scan_completed: 0,
|
|
152
|
+
files_scan_completed: 1,
|
|
153
|
+
dirs_xfer_attempted: 0,
|
|
154
|
+
dirs_xfer_fail: 0,
|
|
155
|
+
files_xfer_attempted: 1,
|
|
156
|
+
files_xfer_fail: 0,
|
|
157
|
+
files_xfer_noxfer: 0
|
|
158
|
+
},
|
|
159
|
+
precalc: {
|
|
160
|
+
enabled: true,
|
|
161
|
+
status: 'ready',
|
|
162
|
+
bytes_expected: 0,
|
|
163
|
+
directories_expected: 0,
|
|
164
|
+
files_expected: 0,
|
|
165
|
+
files_excluded: 0,
|
|
166
|
+
files_special: 0,
|
|
167
|
+
files_failed: 1
|
|
168
|
+
}}
|
|
169
|
+
return {
|
|
170
|
+
id: '609a667d-642e-4290-9312-b4d20d3c0159',
|
|
171
|
+
status: 'running',
|
|
172
|
+
start_spec: job[:ts],
|
|
173
|
+
sessions: [session],
|
|
174
|
+
bytes_transferred: 26,
|
|
175
|
+
bytes_written: 26,
|
|
176
|
+
bytes_lost: 0,
|
|
177
|
+
avg_rate_kbps: 0.51,
|
|
178
|
+
files_completed: 1,
|
|
179
|
+
files_skipped: 0,
|
|
180
|
+
directories_completed: 0,
|
|
181
|
+
start_time_usec: 1701094040000000,
|
|
182
|
+
end_time_usec: 1701094040405312,
|
|
183
|
+
elapsed_usec: 405312,
|
|
184
|
+
error_code: 0,
|
|
185
|
+
error_desc: '',
|
|
186
|
+
precalc: {
|
|
187
|
+
status: 'ready',
|
|
188
|
+
bytes_expected: 0,
|
|
189
|
+
files_expected: 0,
|
|
190
|
+
directories_expected: 0,
|
|
191
|
+
files_special: 0,
|
|
192
|
+
files_failed: 1
|
|
193
|
+
},
|
|
194
|
+
files: [{
|
|
195
|
+
id: 'd1b5c112-82b75425-860745fc-93851671-64541bdd',
|
|
196
|
+
path: '/workspaces/45071/packages/bYA_ilq73g.asp-package/contents/data_file.bin',
|
|
197
|
+
start_time_usec: 1701094040000000,
|
|
198
|
+
elapsed_usec: 105616,
|
|
199
|
+
end_time_usec: 1701094040001355,
|
|
200
|
+
status: 'completed',
|
|
201
|
+
error_code: 0,
|
|
202
|
+
error_desc: '',
|
|
203
|
+
size: 26,
|
|
204
|
+
type: 'file',
|
|
205
|
+
checksum_type: 'none',
|
|
206
|
+
checksum: nil,
|
|
207
|
+
start_byte: 0,
|
|
208
|
+
bytes_written: 26,
|
|
209
|
+
session_id: 'bafc72b8-366c-4501-8095-47208183d6b8'}]
|
|
210
|
+
}
|
|
211
|
+
end
|
|
212
|
+
end # NodeSimulatorServlet
|
|
213
|
+
end # Aspera
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'aspera/oauth/factory'
|
|
4
|
+
require 'aspera/log'
|
|
5
|
+
require 'aspera/assert'
|
|
6
|
+
require 'aspera/id_generator'
|
|
7
|
+
require 'date'
|
|
8
|
+
|
|
9
|
+
module Aspera
|
|
10
|
+
module OAuth
|
|
11
|
+
# Implement OAuth 2 for the REST client and generate a bearer token
|
|
12
|
+
# call get_authorization() to get a token.
|
|
13
|
+
# bearer tokens are kept in memory and also in a file cache for later re-use
|
|
14
|
+
# if a token is expired (api returns 4xx), call again get_authorization(refresh: true)
|
|
15
|
+
# https://tools.ietf.org/html/rfc6749
|
|
16
|
+
class Base
|
|
17
|
+
# scope can be modified after creation
|
|
18
|
+
attr_writer :scope
|
|
19
|
+
|
|
20
|
+
# [M]=mandatory [D]=has default value [O]=Optional/nil
|
|
21
|
+
# @param base_url [M] URL of authentication API
|
|
22
|
+
# @param auth [O] basic auth parameters
|
|
23
|
+
# @param client_id [O]
|
|
24
|
+
# @param client_secret [O]
|
|
25
|
+
# @param scope [O]
|
|
26
|
+
# @param path_token [D] API end point to create a token
|
|
27
|
+
# @param token_field [D] field in result that contains the token
|
|
28
|
+
def initialize(
|
|
29
|
+
base_url:,
|
|
30
|
+
auth: nil,
|
|
31
|
+
client_id: nil,
|
|
32
|
+
client_secret: nil,
|
|
33
|
+
scope: nil,
|
|
34
|
+
path_token: 'token', # default endpoint for /token to generate token
|
|
35
|
+
token_field: 'access_token' # field with token in result of call to path_token
|
|
36
|
+
)
|
|
37
|
+
Aspera.assert_type(base_url, String)
|
|
38
|
+
Aspera.assert(respond_to?(:create_token), 'create_token method must be defined', exception_class: InternalError)
|
|
39
|
+
@base_url = base_url
|
|
40
|
+
@path_token = path_token
|
|
41
|
+
@token_field = token_field
|
|
42
|
+
@client_id = client_id
|
|
43
|
+
@client_secret = client_secret
|
|
44
|
+
@scope = scope
|
|
45
|
+
@identifiers = []
|
|
46
|
+
@identifiers.push(auth[:username]) if auth.is_a?(Hash) && auth.key?(:username)
|
|
47
|
+
# this is the OAuth API
|
|
48
|
+
@api = Rest.new(
|
|
49
|
+
base_url: @base_url,
|
|
50
|
+
redirect_max: 2,
|
|
51
|
+
auth: auth)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# helper method to create token as per RFC
|
|
55
|
+
def create_token_call(www_params)
|
|
56
|
+
Log.log.debug{'Generating a new token'.bg_green}
|
|
57
|
+
return @api.call(
|
|
58
|
+
operation: 'POST',
|
|
59
|
+
subpath: @path_token,
|
|
60
|
+
headers: {'Accept' => 'application/json'},
|
|
61
|
+
www_body_params: www_params)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# @return Hash with optional general parameters
|
|
65
|
+
def optional_scope_client_id(add_secret: false)
|
|
66
|
+
call_params = {}
|
|
67
|
+
call_params[:scope] = @scope unless @scope.nil?
|
|
68
|
+
call_params[:client_id] = @client_id unless @client_id.nil?
|
|
69
|
+
call_params[:client_secret] = @client_secret if add_secret && !@client_id.nil?
|
|
70
|
+
return call_params
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# OAuth v2 token generation
|
|
74
|
+
# @param use_refresh_token set to true to force refresh or re-generation (if previous failed)
|
|
75
|
+
def get_authorization(use_refresh_token: false, use_cache: true)
|
|
76
|
+
# generate token unique identifier for persistency (memory/disk cache)
|
|
77
|
+
token_id = IdGenerator.from_list(Factory.id(
|
|
78
|
+
@base_url,
|
|
79
|
+
@grant_method,
|
|
80
|
+
@identifiers,
|
|
81
|
+
@scope
|
|
82
|
+
))
|
|
83
|
+
|
|
84
|
+
# get token_data from cache (or nil), token_data is what is returned by /token
|
|
85
|
+
token_data = Factory.instance.persist_mgr.get(token_id) if use_cache
|
|
86
|
+
token_data = JSON.parse(token_data) unless token_data.nil?
|
|
87
|
+
# Optional optimization: check if node token is expired based on decoded content then force refresh if close enough
|
|
88
|
+
# might help in case the transfer agent cannot refresh himself
|
|
89
|
+
# `direct` agent is equipped with refresh code
|
|
90
|
+
if !use_refresh_token && !token_data.nil?
|
|
91
|
+
decoded_token = OAuth::Factory.instance.decode_token(token_data[@token_field])
|
|
92
|
+
Log.log.debug{Log.dump('decoded_token', decoded_token)} unless decoded_token.nil?
|
|
93
|
+
if decoded_token.is_a?(Hash)
|
|
94
|
+
expires_at_sec =
|
|
95
|
+
if decoded_token['expires_at'].is_a?(String) then DateTime.parse(decoded_token['expires_at']).to_time
|
|
96
|
+
elsif decoded_token['exp'].is_a?(Integer) then Time.at(decoded_token['exp'])
|
|
97
|
+
end
|
|
98
|
+
# force refresh if we see a token too close from expiration
|
|
99
|
+
use_refresh_token = true if expires_at_sec.is_a?(Time) && (expires_at_sec - Time.now) < OAuth::Factory.instance.globals[:token_expiration_guard_sec]
|
|
100
|
+
Log.log.debug{"Expiration: #{expires_at_sec} / #{use_refresh_token}"}
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# an API was already called, but failed, we need to regenerate or refresh
|
|
105
|
+
if use_refresh_token
|
|
106
|
+
if token_data.is_a?(Hash) && token_data.key?('refresh_token') && !token_data['refresh_token'].eql?('not_supported')
|
|
107
|
+
# save possible refresh token, before deleting the cache
|
|
108
|
+
refresh_token = token_data['refresh_token']
|
|
109
|
+
end
|
|
110
|
+
# delete cache
|
|
111
|
+
Factory.instance.persist_mgr.delete(token_id)
|
|
112
|
+
token_data = nil
|
|
113
|
+
# lets try the existing refresh token
|
|
114
|
+
if !refresh_token.nil?
|
|
115
|
+
Log.log.info{"refresh=[#{refresh_token}]".bg_green}
|
|
116
|
+
# try to refresh
|
|
117
|
+
# note: AoC admin token has no refresh, and lives by default 1800secs
|
|
118
|
+
resp = create_token_call(optional_scope_client_id.merge(grant_type: 'refresh_token', refresh_token: refresh_token))
|
|
119
|
+
if resp[:http].code.start_with?('2')
|
|
120
|
+
# save only if success
|
|
121
|
+
json_data = resp[:http].body
|
|
122
|
+
token_data = JSON.parse(json_data)
|
|
123
|
+
Factory.instance.persist_mgr.put(token_id, json_data)
|
|
124
|
+
else
|
|
125
|
+
Log.log.debug{"refresh failed: #{resp[:http].body}".bg_red}
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# no cache, nor refresh: generate a token
|
|
131
|
+
if token_data.nil?
|
|
132
|
+
resp = create_token
|
|
133
|
+
json_data = resp[:http].body
|
|
134
|
+
token_data = JSON.parse(json_data)
|
|
135
|
+
Factory.instance.persist_mgr.put(token_id, json_data)
|
|
136
|
+
end # if ! in_cache
|
|
137
|
+
Aspera.assert(token_data.key?(@token_field)){"API error: No such field in answer: #{@token_field}"}
|
|
138
|
+
# ok we shall have a token here
|
|
139
|
+
return OAuth::Factory.bearer_build(token_data[@token_field])
|
|
140
|
+
end
|
|
141
|
+
end # OAuth
|
|
142
|
+
end
|
|
143
|
+
end
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'singleton'
|
|
4
|
+
require 'aspera/assert'
|
|
5
|
+
require 'base64'
|
|
6
|
+
module Aspera
|
|
7
|
+
module OAuth
|
|
8
|
+
class Factory
|
|
9
|
+
include Singleton
|
|
10
|
+
# a prefix for persistency of tokens (simplify garbage collect)
|
|
11
|
+
PERSIST_CATEGORY_TOKEN = 'token'
|
|
12
|
+
# prefix for bearer token when in header
|
|
13
|
+
BEARER_PREFIX = 'Bearer '
|
|
14
|
+
|
|
15
|
+
private_constant :PERSIST_CATEGORY_TOKEN, :BEARER_PREFIX
|
|
16
|
+
|
|
17
|
+
class << self
|
|
18
|
+
def bearer_build(token)
|
|
19
|
+
return BEARER_PREFIX + token
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def bearer_extract(token)
|
|
23
|
+
Aspera.assert(bearer?(token)){'not a bearer token, wrong prefix'}
|
|
24
|
+
return token[BEARER_PREFIX.length..-1]
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def bearer?(token)
|
|
28
|
+
return token.start_with?(BEARER_PREFIX)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def id(*params)
|
|
32
|
+
return [PERSIST_CATEGORY_TOKEN, *params].flatten
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def class_to_id(creator_class)
|
|
36
|
+
return creator_class.name.split('::').last.capital_to_snake.to_sym
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def initialize
|
|
43
|
+
# persistency manager
|
|
44
|
+
@persist = nil
|
|
45
|
+
# token creation methods
|
|
46
|
+
@token_type_classes = {}
|
|
47
|
+
@decoders = []
|
|
48
|
+
@globals = {
|
|
49
|
+
# remove 5 minutes to account for time offset between client and server (TODO: configurable?)
|
|
50
|
+
jwt_accepted_offset_sec: 300,
|
|
51
|
+
# one hour validity (TODO: configurable?)
|
|
52
|
+
jwt_expiry_offset_sec: 3600,
|
|
53
|
+
# tokens older than 30 minutes will be discarded from cache
|
|
54
|
+
token_cache_expiry_sec: 1800,
|
|
55
|
+
# tokens valid for less than this duration will be regenerated
|
|
56
|
+
token_expiration_guard_sec: 120
|
|
57
|
+
}
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
public
|
|
61
|
+
|
|
62
|
+
attr_reader :globals
|
|
63
|
+
|
|
64
|
+
def persist_mgr=(manager)
|
|
65
|
+
@persist = manager
|
|
66
|
+
# cleanup expired tokens
|
|
67
|
+
@persist.garbage_collect(PERSIST_CATEGORY_TOKEN, @globals[:token_cache_expiry_sec])
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def persist_mgr
|
|
71
|
+
if @persist.nil?
|
|
72
|
+
# use OAuth::Factory.instance.persist_mgr=PersistencyFolder.new)
|
|
73
|
+
Log.log.debug('Not using persistency')
|
|
74
|
+
# create NULL persistency class
|
|
75
|
+
@persist = Class.new do
|
|
76
|
+
def get(_x); nil; end; def delete(_x); nil; end; def put(_x, _y); nil; end; def garbage_collect(_x, _y); nil; end # rubocop:disable Layout/EmptyLineBetweenDefs, Style/Semicolon, Layout/LineLength
|
|
77
|
+
end.new
|
|
78
|
+
end
|
|
79
|
+
return @persist
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# delete all existing tokens
|
|
83
|
+
def flush_tokens
|
|
84
|
+
persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN, nil)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# register a bearer token decoder, mainly to inspect expiry date
|
|
88
|
+
def register_decoder(method)
|
|
89
|
+
@decoders.push(method)
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# decode token using all registered decoders
|
|
93
|
+
def decode_token(token)
|
|
94
|
+
@decoders.each do |decoder|
|
|
95
|
+
result = decoder.call(token) rescue nil
|
|
96
|
+
return result unless result.nil?
|
|
97
|
+
end
|
|
98
|
+
return nil
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# register a token creation method
|
|
102
|
+
# @param id creation type from field :grant_method in constructor
|
|
103
|
+
# @param lambda_create called to create token
|
|
104
|
+
# @param id_create called to generate unique id for token, for cache
|
|
105
|
+
def register_token_creator(creator_class)
|
|
106
|
+
Aspera.assert_type(creator_class, Class)
|
|
107
|
+
id = self.class.class_to_id(creator_class)
|
|
108
|
+
Log.log.debug{"registering token creator #{id}"}
|
|
109
|
+
@token_type_classes[id] = creator_class
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# @return one of the registered creators for the given create type
|
|
113
|
+
def create(**parameters)
|
|
114
|
+
Aspera.assert_type(parameters, Hash)
|
|
115
|
+
id = parameters[:grant_method]
|
|
116
|
+
Aspera.assert(@token_type_classes.key?(id)){"token grant method unknown: '#{id}'"}
|
|
117
|
+
create_parameters = parameters.reject { |k, _v| k.eql?(:grant_method) }
|
|
118
|
+
@token_type_classes[id].new(**create_parameters)
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
# JSON Web Signature (JWS) compact serialization: https://datatracker.ietf.org/doc/html/rfc7515
|
|
122
|
+
Factory.instance.register_decoder(lambda { |token| parts = token.split('.'); Aspera.assert(parts.length.eql?(3)){'not aoc token'}; JSON.parse(Base64.decode64(parts[1]))}) # rubocop:disable Style/Semicolon, Layout/LineLength
|
|
123
|
+
end
|
|
124
|
+
end
|