aspera-cli 4.15.0 → 4.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +29 -3
  4. data/CHANGELOG.md +375 -280
  5. data/CONTRIBUTING.md +71 -18
  6. data/README.md +1978 -1656
  7. data/bin/ascli +13 -31
  8. data/bin/asession +32 -22
  9. data/examples/dascli +2 -2
  10. data/lib/aspera/agent/alpha.rb +117 -0
  11. data/lib/aspera/agent/base.rb +61 -0
  12. data/lib/aspera/{fasp/agent_connect.rb → agent/connect.rb} +13 -11
  13. data/lib/aspera/{fasp/agent_direct.rb → agent/direct.rb} +116 -116
  14. data/lib/aspera/{fasp/agent_httpgw.rb → agent/httpgw.rb} +21 -19
  15. data/lib/aspera/{fasp/agent_node.rb → agent/node.rb} +21 -33
  16. data/lib/aspera/agent/trsdk.rb +188 -0
  17. data/lib/aspera/api/aoc.rb +586 -0
  18. data/lib/aspera/api/ats.rb +46 -0
  19. data/lib/aspera/api/cos_node.rb +95 -0
  20. data/lib/aspera/api/node.rb +344 -0
  21. data/lib/aspera/ascmd.rb +47 -14
  22. data/lib/aspera/{fasp → ascp}/installation.rb +54 -15
  23. data/lib/aspera/{fasp → ascp}/management.rb +14 -14
  24. data/lib/aspera/{fasp → ascp}/products.rb +1 -1
  25. data/lib/aspera/assert.rb +45 -0
  26. data/lib/aspera/cli/basic_auth_plugin.rb +11 -10
  27. data/lib/aspera/cli/extended_value.rb +5 -5
  28. data/lib/aspera/cli/formatter.rb +27 -14
  29. data/lib/aspera/cli/hints.rb +7 -6
  30. data/lib/aspera/cli/main.rb +49 -29
  31. data/lib/aspera/cli/manager.rb +46 -36
  32. data/lib/aspera/cli/plugin.rb +34 -20
  33. data/lib/aspera/cli/plugin_factory.rb +61 -0
  34. data/lib/aspera/cli/plugins/alee.rb +7 -7
  35. data/lib/aspera/cli/plugins/aoc.rb +168 -132
  36. data/lib/aspera/cli/plugins/ats.rb +33 -33
  37. data/lib/aspera/cli/plugins/bss.rb +3 -4
  38. data/lib/aspera/cli/plugins/config.rb +250 -272
  39. data/lib/aspera/cli/plugins/console.rb +8 -6
  40. data/lib/aspera/cli/plugins/cos.rb +20 -19
  41. data/lib/aspera/cli/plugins/faspex.rb +71 -60
  42. data/lib/aspera/cli/plugins/faspex5.rb +212 -133
  43. data/lib/aspera/cli/plugins/node.rb +83 -75
  44. data/lib/aspera/cli/plugins/orchestrator.rb +36 -44
  45. data/lib/aspera/cli/plugins/preview.rb +33 -31
  46. data/lib/aspera/cli/plugins/server.rb +33 -32
  47. data/lib/aspera/cli/plugins/shares.rb +39 -33
  48. data/lib/aspera/cli/sync_actions.rb +9 -9
  49. data/lib/aspera/cli/transfer_agent.rb +45 -25
  50. data/lib/aspera/cli/transfer_progress.rb +2 -3
  51. data/lib/aspera/cli/version.rb +1 -1
  52. data/lib/aspera/colors.rb +5 -0
  53. data/lib/aspera/command_line_builder.rb +16 -14
  54. data/lib/aspera/coverage.rb +21 -0
  55. data/lib/aspera/data_repository.rb +33 -2
  56. data/lib/aspera/environment.rb +5 -4
  57. data/lib/aspera/faspex_gw.rb +13 -11
  58. data/lib/aspera/faspex_postproc.rb +6 -5
  59. data/lib/aspera/id_generator.rb +4 -2
  60. data/lib/aspera/json_rpc.rb +10 -8
  61. data/lib/aspera/keychain/encrypted_hash.rb +46 -11
  62. data/lib/aspera/keychain/macos_security.rb +29 -22
  63. data/lib/aspera/log.rb +5 -4
  64. data/lib/aspera/nagios.rb +7 -2
  65. data/lib/aspera/node_simulator.rb +213 -0
  66. data/lib/aspera/oauth/base.rb +143 -0
  67. data/lib/aspera/oauth/factory.rb +124 -0
  68. data/lib/aspera/oauth/generic.rb +34 -0
  69. data/lib/aspera/oauth/jwt.rb +51 -0
  70. data/lib/aspera/oauth/url_json.rb +31 -0
  71. data/lib/aspera/oauth/web.rb +50 -0
  72. data/lib/aspera/oauth.rb +5 -328
  73. data/lib/aspera/open_application.rb +7 -7
  74. data/lib/aspera/persistency_action_once.rb +13 -14
  75. data/lib/aspera/persistency_folder.rb +3 -2
  76. data/lib/aspera/preview/file_types.rb +53 -267
  77. data/lib/aspera/preview/generator.rb +7 -5
  78. data/lib/aspera/preview/terminal.rb +17 -7
  79. data/lib/aspera/preview/utils.rb +8 -7
  80. data/lib/aspera/proxy_auto_config.rb +6 -3
  81. data/lib/aspera/rest.rb +187 -140
  82. data/lib/aspera/rest_error_analyzer.rb +1 -0
  83. data/lib/aspera/rest_errors_aspera.rb +5 -3
  84. data/lib/aspera/resumer.rb +77 -0
  85. data/lib/aspera/secret_hider.rb +5 -2
  86. data/lib/aspera/ssh.rb +15 -8
  87. data/lib/aspera/temp_file_manager.rb +1 -1
  88. data/lib/aspera/{fasp → transfer}/error.rb +3 -3
  89. data/lib/aspera/{fasp → transfer}/error_info.rb +1 -1
  90. data/lib/aspera/{fasp → transfer}/faux_file.rb +1 -1
  91. data/lib/aspera/{fasp → transfer}/parameters.rb +95 -120
  92. data/lib/aspera/{fasp/transfer_spec.rb → transfer/spec.rb} +23 -19
  93. data/lib/aspera/{fasp/parameters.yaml → transfer/spec.yaml} +4 -99
  94. data/lib/aspera/transfer/sync.rb +273 -0
  95. data/lib/aspera/{fasp → transfer}/uri.rb +10 -9
  96. data/lib/aspera/web_server_simple.rb +12 -3
  97. data.tar.gz.sig +0 -0
  98. metadata +92 -68
  99. metadata.gz.sig +0 -0
  100. data/lib/aspera/aoc.rb +0 -606
  101. data/lib/aspera/ats_api.rb +0 -47
  102. data/lib/aspera/cos_node.rb +0 -93
  103. data/lib/aspera/fasp/agent_aspera.rb +0 -126
  104. data/lib/aspera/fasp/agent_base.rb +0 -48
  105. data/lib/aspera/fasp/agent_trsdk.rb +0 -146
  106. data/lib/aspera/fasp/resume_policy.rb +0 -77
  107. data/lib/aspera/node.rb +0 -338
  108. data/lib/aspera/sync.rb +0 -219
@@ -2,6 +2,9 @@
2
2
 
3
3
  # https://github.com/fastlane-community/security
4
4
  require 'aspera/cli/info'
5
+ require 'aspera/log'
6
+ require 'aspera/assert'
7
+ require 'open3'
5
8
 
6
9
  # enhance the gem to support other key chains
7
10
  module Aspera
@@ -9,6 +12,8 @@ module Aspera
9
12
  module MacosSecurity
10
13
  # keychain based on macOS keychain, using `security` command line
11
14
  class Keychain
15
+ # https://www.unix.com/man-page/osx/1/security/
16
+ SECURITY_UTILITY = 'security'
12
17
  DOMAINS = %i[user system common dynamic].freeze
13
18
  LIST_OPTIONS = {
14
19
  domain: :c
@@ -36,25 +41,27 @@ module Aspera
36
41
  url = options&.delete(:url)
37
42
  if !url.nil?
38
43
  uri = URI.parse(url)
39
- raise 'only https' unless uri.scheme.eql?('https')
44
+ Aspera.assert(uri.scheme.eql?('https')){'only https'}
40
45
  options[:protocol] = 'htps' # cspell: disable-line
41
46
  raise 'host required in URL' if uri.host.nil?
42
47
  options[:server] = uri.host
43
48
  options[:path] = uri.path unless ['', '/'].include?(uri.path)
44
49
  options[:port] = uri.port unless uri.port.eql?(443) && !url.include?(':443/')
45
50
  end
46
- cmd = ['security', command]
51
+ command_line = [SECURITY_UTILITY, command]
47
52
  options&.each do |k, v|
48
- raise "unknown option: #{k}" unless supported.key?(k)
53
+ Aspera.assert(supported.key?(k)){"unknown option: #{k}"}
49
54
  next if v.nil?
50
- cmd.push("-#{supported[k]}")
51
- cmd.push(v.shellescape) unless v.empty?
55
+ command_line.push("-#{supported[k]}")
56
+ command_line.push(v.shellescape) unless v.empty?
52
57
  end
53
- cmd.push(last_opt) unless last_opt.nil?
54
- Log.log.debug{"executing>>#{cmd.join(' ')}"}
55
- result = %x(#{cmd.join(' ')} 2>&1)
56
- Log.log.debug{"result>>[#{result}]"}
57
- return result
58
+ command_line.push(last_opt) unless last_opt.nil?
59
+ Log.log.debug{"executing>>#{command_line.join(' ')}"}
60
+ stdout, stderr, status = Open3.capture3(*command_line)
61
+ Log.log.debug{"status=#{status}, stderr=#{stderr}"}
62
+ Log.log.trace1{"stdout=#{stdout}"}
63
+ raise "#{SECURITY_UTILITY} failed: #{status.exitstatus} : #{stderr}" unless status.success?
64
+ return stdout
58
65
  end
59
66
 
60
67
  def key_chains(output)
@@ -70,7 +77,7 @@ module Aspera
70
77
  end
71
78
 
72
79
  def list(options={})
73
- raise ArgumentError, "Invalid domain #{options[:domain]}, expected one of: #{DOMAINS}" unless options[:domain].nil? || DOMAINS.include?(options[:domain])
80
+ Aspera.assert_values(options[:domain], DOMAINS, exception_class: ArgumentError){'domain'} unless options[:domain].nil?
74
81
  key_chains(execute('list-key_chains', options, LIST_OPTIONS))
75
82
  end
76
83
 
@@ -89,15 +96,15 @@ module Aspera
89
96
  end
90
97
 
91
98
  def password(operation, pass_type, options)
92
- raise "wrong operation: #{operation}" unless %i[add find delete].include?(operation)
93
- raise "wrong pass_type: #{pass_type}" unless %i[generic internet].include?(pass_type)
94
- raise 'options shall be Hash' unless options.is_a?(Hash)
99
+ Aspera.assert_values(operation, %i[add find delete]){'operation'}
100
+ Aspera.assert_values(pass_type, %i[generic internet]){'pass_type'}
101
+ Aspera.assert_type(options, Hash)
95
102
  missing = (operation.eql?(:add) ? %i[account service password] : %i[label]) - options.keys
96
- raise "missing options: #{missing}" unless missing.empty?
103
+ Aspera.assert(missing.empty?){"missing options: #{missing}"}
97
104
  options[:getpass] = '' if operation.eql?(:find)
98
105
  output = self.class.execute("#{operation}-#{pass_type}-password", options, ADD_PASS_OPTIONS, @path)
99
106
  raise output.gsub(/^.*: /, '') if output.start_with?('security: ')
100
- return nil unless operation.eql?(:find)
107
+ return unless operation.eql?(:find)
101
108
  attributes = {}
102
109
  output.split("\n").each do |line|
103
110
  case line
@@ -127,18 +134,18 @@ module Aspera
127
134
  end
128
135
 
129
136
  def set(options)
130
- raise 'options shall be Hash' unless options.is_a?(Hash)
137
+ Aspera.assert_type(options, Hash){'options'}
131
138
  unsupported = options.keys - %i[label username password url description]
132
- raise "unsupported options: #{unsupported}" unless unsupported.empty?
139
+ Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}"}
133
140
  @keychain.password(
134
141
  :add, :generic, service: options[:label],
135
142
  account: options[:username] || 'none', password: options[:password], comment: options[:description])
136
143
  end
137
144
 
138
145
  def get(options)
139
- raise 'options shall be Hash' unless options.is_a?(Hash)
146
+ Aspera.assert_type(options, Hash){'options'}
140
147
  unsupported = options.keys - %i[label]
141
- raise "unsupported options: #{unsupported}" unless unsupported.empty?
148
+ Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}"}
142
149
  info = @keychain.password(:find, :generic, label: options[:label])
143
150
  raise 'not found' if info.nil?
144
151
  result = options.clone
@@ -153,9 +160,9 @@ module Aspera
153
160
  end
154
161
 
155
162
  def delete(options)
156
- raise 'options shall be Hash' unless options.is_a?(Hash)
163
+ Aspera.assert_type(options, Hash){'options'}
157
164
  unsupported = options.keys - %i[label]
158
- raise "unsupported options: #{unsupported}" unless unsupported.empty?
165
+ Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}"}
159
166
  raise 'delete not implemented, use macos keychain app'
160
167
  end
161
168
  end
data/lib/aspera/log.rb CHANGED
@@ -3,6 +3,7 @@
3
3
  require 'aspera/colors'
4
4
  require 'aspera/secret_hider'
5
5
  require 'aspera/environment'
6
+ require 'aspera/assert'
6
7
  require 'logger'
7
8
  require 'pp'
8
9
  require 'json'
@@ -13,7 +14,7 @@ class Logger
13
14
  TRACE_MAX = 2
14
15
  # add custom level to logger severity
15
16
  module Severity
16
- 1.upto(TRACE_MAX).each { |level| const_set("TRACE#{level}", - level)}
17
+ 1.upto(TRACE_MAX).each { |level| const_set(:"TRACE#{level}", - level)}
17
18
  end
18
19
  # quick access to label
19
20
  SEVERITY_LABEL = Severity.constants.each_with_object({}) { |name, hash| hash[Severity.const_get(name)] = name}
@@ -39,6 +40,7 @@ class Logger
39
40
  end
40
41
  EOM
41
42
  end
43
+ # declare methods for all levels
42
44
  Logger::Severity.constants.each { |severity| make_methods(severity) }
43
45
  end
44
46
 
@@ -97,8 +99,7 @@ module Aspera
97
99
  Logger::Severity.constants.each do |name|
98
100
  return name.downcase.to_sym if @logger.level.eql?(Logger::Severity.const_get(name))
99
101
  end
100
- # should not happen
101
- raise "INTERNAL ERROR: unexpected level #{@logger.level}"
102
+ Aspera.error_unexpected_value(@logger.level){'log level'}
102
103
  end
103
104
 
104
105
  # change underlying logger, but keep log level
@@ -123,7 +124,7 @@ module Aspera
123
124
  end
124
125
  @logger = Syslog::Logger.new(@program_name, Syslog::LOG_LOCAL2)
125
126
  else
126
- raise "unknown log type: #{new_log_type.class} #{new_log_type}"
127
+ raise "unknown log type: #{new_log_type}, use one of: #{LOG_TYPES.join(', ')}"
127
128
  end
128
129
  @logger.level = current_severity_integer
129
130
  @logger_type = new_log_type
data/lib/aspera/nagios.rb CHANGED
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'aspera/log'
4
+ require 'aspera/assert'
3
5
  require 'date'
4
6
 
5
7
  module Aspera
@@ -21,8 +23,11 @@ module Aspera
21
23
  class << self
22
24
  # process results of a analysis and display status and exit with code
23
25
  def process(data)
24
- raise 'INTERNAL ERROR, result must be list and not empty' unless data.is_a?(Array) && !data.empty?
25
- %w[status component message].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.key?(c)}
26
+ Aspera.assert_type(data, Array)
27
+ Aspera.assert(!data.empty?){'data is empty'}
28
+ %w[status component message].each do |c|
29
+ Aspera.assert(data.first.key?(c)){"result must have #{c}"}
30
+ end
26
31
  res_errors = data.reject{|s|s['status'].eql?('ok')}
27
32
  # keep only errors in case of problem, other ok are assumed so
28
33
  data = res_errors unless res_errors.empty?
@@ -0,0 +1,213 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aspera/ascp/installation'
4
+ require 'aspera/agent/direct'
5
+ require 'aspera/log'
6
+ require 'webrick'
7
+ require 'json'
8
+
9
+ module Aspera
10
+ # this class answers the Faspex /send API and creates a package on Aspera on Cloud
11
+ class NodeSimulatorServlet < WEBrick::HTTPServlet::AbstractServlet
12
+ PATH_TRANSFERS = '/ops/transfers'
13
+ PATH_ONE_TRANSFER = %r{/ops/transfers/(.+)$}
14
+ # @param app_api [Api::AoC]
15
+ # @param app_context [String]
16
+ def initialize(server, credentials, transfer)
17
+ super(server)
18
+ @credentials = credentials
19
+ @xfer_manager = Agent::Direct.new
20
+ end
21
+
22
+ def do_POST(request, response)
23
+ case request.path
24
+ when PATH_TRANSFERS
25
+ job_id = @xfer_manager.start_transfer(JSON.parse(request.body))
26
+ session = @xfer_manager.sessions_by_job(job_id).first
27
+ result = session[:ts].clone
28
+ result['id'] = job_id
29
+ set_json_response(response, result)
30
+ else
31
+ set_json_response(response, [{error: 'Bad request'}], code: 400)
32
+ end
33
+ end
34
+
35
+ def do_GET(request, response)
36
+ case request.path
37
+ when '/info'
38
+ info = Ascp::Installation.instance.ascp_info
39
+ set_json_response(response, {
40
+ application: 'node',
41
+ current_time: Time.now.utc.iso8601(0),
42
+ version: info['ascp_version'].gsub(/ .*$/, ''),
43
+ license_expiration_date: info['expiration_date'],
44
+ license_max_rate: info['maximum_bandwidth'],
45
+ os: %x(uname -srv).chomp,
46
+ aej_status: 'disconnected',
47
+ async_reporting: 'yes',
48
+ transfer_activity_reporting: 'yes',
49
+ transfer_user: 'xfer',
50
+ docroot: 'file:////data/aoc/eudemo-sedemo',
51
+ node_id: '2bbdcc39-f789-4d47-8163-6767fc14f421',
52
+ cluster_id: '6dae2844-d1a9-47a5-916d-9b3eac3ea466',
53
+ acls: [],
54
+ access_key_configuration_capabilities: {
55
+ transfer: %w[
56
+ cipher
57
+ policy
58
+ target_rate_cap_kbps
59
+ target_rate_kbps
60
+ preserve_timestamps
61
+ content_protection_secret
62
+ aggressiveness
63
+ token_encryption_key
64
+ byok_enabled
65
+ bandwidth_flow_network_rc_module
66
+ file_checksum_type],
67
+ server: %w[
68
+ activity_event_logging
69
+ activity_file_event_logging
70
+ recursive_counts
71
+ aej_logging
72
+ wss_enabled
73
+ activity_transfer_ignore_skipped_files
74
+ activity_files_max
75
+ access_key_credentials_encryption_type
76
+ discovery
77
+ auto_delete
78
+ allow
79
+ deny]
80
+ },
81
+ capabilities: [
82
+ {name: 'sync', value: true},
83
+ {name: 'watchfolder', value: true},
84
+ {name: 'symbolic_links', value: true},
85
+ {name: 'move_file', value: true},
86
+ {name: 'move_directory', value: true},
87
+ {name: 'filelock', value: false},
88
+ {name: 'ssh_fingerprint', value: false},
89
+ {name: 'aej_version', value: '1.0'},
90
+ {name: 'page', value: true},
91
+ {name: 'file_id_version', value: '2.0'},
92
+ {name: 'auto_delete', value: false}],
93
+ settings: [
94
+ {name: 'content_protection_required', value: false},
95
+ {name: 'content_protection_strong_pass_required', value: false},
96
+ {name: 'filelock_restriction', value: 'none'},
97
+ {name: 'ssh_fingerprint', value: nil},
98
+ {name: 'wss_enabled', value: false},
99
+ {name: 'wss_port', value: 443}
100
+ ]})
101
+ when PATH_TRANSFERS
102
+ result = @xfer_manager.sessions.map { |session| job_to_transfer(session) }
103
+ set_json_response(response, result)
104
+ when PATH_ONE_TRANSFER
105
+ job_id = request.path.match(PATH_ONE_TRANSFER)[1]
106
+ set_json_response(response, job_to_transfer(@xfer_manager.sessions_by_job(job_id).first))
107
+ else
108
+ set_json_response(response, [{error: 'Unknown request'}], code: 400)
109
+ end
110
+ end
111
+
112
+ def set_json_response(response, json, code: 200)
113
+ response.status = code
114
+ response['Content-Type'] = 'application/json'
115
+ response.body = json.to_json
116
+ Log.log.trace1{Log.dump('response', json)}
117
+ end
118
+
119
+ def job_to_transfer(job)
120
+ session = {
121
+ id: 'bafc72b8-366c-4501-8095-47208183d6b8',
122
+ client_node_id: '',
123
+ server_node_id: '2bbdcc39-f789-4d47-8163-6767fc14f421',
124
+ client_ip_address: '192.168.0.100',
125
+ server_ip_address: '5.10.114.4',
126
+ status: 'running',
127
+ retry_timeout: 3600,
128
+ retry_count: 0,
129
+ start_time_usec: 1701094040000000,
130
+ end_time_usec: nil,
131
+ elapsed_usec: 405312,
132
+ bytes_transferred: 26,
133
+ bytes_written: 26,
134
+ bytes_lost: 0,
135
+ files_completed: 1,
136
+ directories_completed: 0,
137
+ target_rate_kbps: 500000,
138
+ min_rate_kbps: 0,
139
+ calc_rate_kbps: 9900,
140
+ network_delay_usec: 40000,
141
+ avg_rate_kbps: 0.51,
142
+ error_code: 0,
143
+ error_desc: '',
144
+ source_statistics: {
145
+ args_scan_attempted: 1,
146
+ args_scan_completed: 1,
147
+ paths_scan_attempted: 1,
148
+ paths_scan_failed: 0,
149
+ paths_scan_skipped: 0,
150
+ paths_scan_excluded: 0,
151
+ dirs_scan_completed: 0,
152
+ files_scan_completed: 1,
153
+ dirs_xfer_attempted: 0,
154
+ dirs_xfer_fail: 0,
155
+ files_xfer_attempted: 1,
156
+ files_xfer_fail: 0,
157
+ files_xfer_noxfer: 0
158
+ },
159
+ precalc: {
160
+ enabled: true,
161
+ status: 'ready',
162
+ bytes_expected: 0,
163
+ directories_expected: 0,
164
+ files_expected: 0,
165
+ files_excluded: 0,
166
+ files_special: 0,
167
+ files_failed: 1
168
+ }}
169
+ return {
170
+ id: '609a667d-642e-4290-9312-b4d20d3c0159',
171
+ status: 'running',
172
+ start_spec: job[:ts],
173
+ sessions: [session],
174
+ bytes_transferred: 26,
175
+ bytes_written: 26,
176
+ bytes_lost: 0,
177
+ avg_rate_kbps: 0.51,
178
+ files_completed: 1,
179
+ files_skipped: 0,
180
+ directories_completed: 0,
181
+ start_time_usec: 1701094040000000,
182
+ end_time_usec: 1701094040405312,
183
+ elapsed_usec: 405312,
184
+ error_code: 0,
185
+ error_desc: '',
186
+ precalc: {
187
+ status: 'ready',
188
+ bytes_expected: 0,
189
+ files_expected: 0,
190
+ directories_expected: 0,
191
+ files_special: 0,
192
+ files_failed: 1
193
+ },
194
+ files: [{
195
+ id: 'd1b5c112-82b75425-860745fc-93851671-64541bdd',
196
+ path: '/workspaces/45071/packages/bYA_ilq73g.asp-package/contents/data_file.bin',
197
+ start_time_usec: 1701094040000000,
198
+ elapsed_usec: 105616,
199
+ end_time_usec: 1701094040001355,
200
+ status: 'completed',
201
+ error_code: 0,
202
+ error_desc: '',
203
+ size: 26,
204
+ type: 'file',
205
+ checksum_type: 'none',
206
+ checksum: nil,
207
+ start_byte: 0,
208
+ bytes_written: 26,
209
+ session_id: 'bafc72b8-366c-4501-8095-47208183d6b8'}]
210
+ }
211
+ end
212
+ end # NodeSimulatorServlet
213
+ end # Aspera
@@ -0,0 +1,143 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aspera/oauth/factory'
4
+ require 'aspera/log'
5
+ require 'aspera/assert'
6
+ require 'aspera/id_generator'
7
+ require 'date'
8
+
9
+ module Aspera
10
+ module OAuth
11
+ # Implement OAuth 2 for the REST client and generate a bearer token
12
+ # call get_authorization() to get a token.
13
+ # bearer tokens are kept in memory and also in a file cache for later re-use
14
+ # if a token is expired (api returns 4xx), call again get_authorization(refresh: true)
15
+ # https://tools.ietf.org/html/rfc6749
16
+ class Base
17
+ # scope can be modified after creation
18
+ attr_writer :scope
19
+
20
+ # [M]=mandatory [D]=has default value [O]=Optional/nil
21
+ # @param base_url [M] URL of authentication API
22
+ # @param auth [O] basic auth parameters
23
+ # @param client_id [O]
24
+ # @param client_secret [O]
25
+ # @param scope [O]
26
+ # @param path_token [D] API end point to create a token
27
+ # @param token_field [D] field in result that contains the token
28
+ def initialize(
29
+ base_url:,
30
+ auth: nil,
31
+ client_id: nil,
32
+ client_secret: nil,
33
+ scope: nil,
34
+ path_token: 'token', # default endpoint for /token to generate token
35
+ token_field: 'access_token' # field with token in result of call to path_token
36
+ )
37
+ Aspera.assert_type(base_url, String)
38
+ Aspera.assert(respond_to?(:create_token), 'create_token method must be defined', exception_class: InternalError)
39
+ @base_url = base_url
40
+ @path_token = path_token
41
+ @token_field = token_field
42
+ @client_id = client_id
43
+ @client_secret = client_secret
44
+ @scope = scope
45
+ @identifiers = []
46
+ @identifiers.push(auth[:username]) if auth.is_a?(Hash) && auth.key?(:username)
47
+ # this is the OAuth API
48
+ @api = Rest.new(
49
+ base_url: @base_url,
50
+ redirect_max: 2,
51
+ auth: auth)
52
+ end
53
+
54
+ # helper method to create token as per RFC
55
+ def create_token_call(www_params)
56
+ Log.log.debug{'Generating a new token'.bg_green}
57
+ return @api.call(
58
+ operation: 'POST',
59
+ subpath: @path_token,
60
+ headers: {'Accept' => 'application/json'},
61
+ www_body_params: www_params)
62
+ end
63
+
64
+ # @return Hash with optional general parameters
65
+ def optional_scope_client_id(add_secret: false)
66
+ call_params = {}
67
+ call_params[:scope] = @scope unless @scope.nil?
68
+ call_params[:client_id] = @client_id unless @client_id.nil?
69
+ call_params[:client_secret] = @client_secret if add_secret && !@client_id.nil?
70
+ return call_params
71
+ end
72
+
73
+ # OAuth v2 token generation
74
+ # @param use_refresh_token set to true to force refresh or re-generation (if previous failed)
75
+ def get_authorization(use_refresh_token: false, use_cache: true)
76
+ # generate token unique identifier for persistency (memory/disk cache)
77
+ token_id = IdGenerator.from_list(Factory.id(
78
+ @base_url,
79
+ @grant_method,
80
+ @identifiers,
81
+ @scope
82
+ ))
83
+
84
+ # get token_data from cache (or nil), token_data is what is returned by /token
85
+ token_data = Factory.instance.persist_mgr.get(token_id) if use_cache
86
+ token_data = JSON.parse(token_data) unless token_data.nil?
87
+ # Optional optimization: check if node token is expired based on decoded content then force refresh if close enough
88
+ # might help in case the transfer agent cannot refresh himself
89
+ # `direct` agent is equipped with refresh code
90
+ if !use_refresh_token && !token_data.nil?
91
+ decoded_token = OAuth::Factory.instance.decode_token(token_data[@token_field])
92
+ Log.log.debug{Log.dump('decoded_token', decoded_token)} unless decoded_token.nil?
93
+ if decoded_token.is_a?(Hash)
94
+ expires_at_sec =
95
+ if decoded_token['expires_at'].is_a?(String) then DateTime.parse(decoded_token['expires_at']).to_time
96
+ elsif decoded_token['exp'].is_a?(Integer) then Time.at(decoded_token['exp'])
97
+ end
98
+ # force refresh if we see a token too close from expiration
99
+ use_refresh_token = true if expires_at_sec.is_a?(Time) && (expires_at_sec - Time.now) < OAuth::Factory.instance.globals[:token_expiration_guard_sec]
100
+ Log.log.debug{"Expiration: #{expires_at_sec} / #{use_refresh_token}"}
101
+ end
102
+ end
103
+
104
+ # an API was already called, but failed, we need to regenerate or refresh
105
+ if use_refresh_token
106
+ if token_data.is_a?(Hash) && token_data.key?('refresh_token') && !token_data['refresh_token'].eql?('not_supported')
107
+ # save possible refresh token, before deleting the cache
108
+ refresh_token = token_data['refresh_token']
109
+ end
110
+ # delete cache
111
+ Factory.instance.persist_mgr.delete(token_id)
112
+ token_data = nil
113
+ # lets try the existing refresh token
114
+ if !refresh_token.nil?
115
+ Log.log.info{"refresh=[#{refresh_token}]".bg_green}
116
+ # try to refresh
117
+ # note: AoC admin token has no refresh, and lives by default 1800secs
118
+ resp = create_token_call(optional_scope_client_id.merge(grant_type: 'refresh_token', refresh_token: refresh_token))
119
+ if resp[:http].code.start_with?('2')
120
+ # save only if success
121
+ json_data = resp[:http].body
122
+ token_data = JSON.parse(json_data)
123
+ Factory.instance.persist_mgr.put(token_id, json_data)
124
+ else
125
+ Log.log.debug{"refresh failed: #{resp[:http].body}".bg_red}
126
+ end
127
+ end
128
+ end
129
+
130
+ # no cache, nor refresh: generate a token
131
+ if token_data.nil?
132
+ resp = create_token
133
+ json_data = resp[:http].body
134
+ token_data = JSON.parse(json_data)
135
+ Factory.instance.persist_mgr.put(token_id, json_data)
136
+ end # if ! in_cache
137
+ Aspera.assert(token_data.key?(@token_field)){"API error: No such field in answer: #{@token_field}"}
138
+ # ok we shall have a token here
139
+ return OAuth::Factory.bearer_build(token_data[@token_field])
140
+ end
141
+ end # OAuth
142
+ end
143
+ end
@@ -0,0 +1,124 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'singleton'
4
+ require 'aspera/assert'
5
+ require 'base64'
6
+ module Aspera
7
+ module OAuth
8
+ class Factory
9
+ include Singleton
10
+ # a prefix for persistency of tokens (simplify garbage collect)
11
+ PERSIST_CATEGORY_TOKEN = 'token'
12
+ # prefix for bearer token when in header
13
+ BEARER_PREFIX = 'Bearer '
14
+
15
+ private_constant :PERSIST_CATEGORY_TOKEN, :BEARER_PREFIX
16
+
17
+ class << self
18
+ def bearer_build(token)
19
+ return BEARER_PREFIX + token
20
+ end
21
+
22
+ def bearer_extract(token)
23
+ Aspera.assert(bearer?(token)){'not a bearer token, wrong prefix'}
24
+ return token[BEARER_PREFIX.length..-1]
25
+ end
26
+
27
+ def bearer?(token)
28
+ return token.start_with?(BEARER_PREFIX)
29
+ end
30
+
31
+ def id(*params)
32
+ return [PERSIST_CATEGORY_TOKEN, *params].flatten
33
+ end
34
+
35
+ def class_to_id(creator_class)
36
+ return creator_class.name.split('::').last.capital_to_snake.to_sym
37
+ end
38
+ end
39
+
40
+ private
41
+
42
+ def initialize
43
+ # persistency manager
44
+ @persist = nil
45
+ # token creation methods
46
+ @token_type_classes = {}
47
+ @decoders = []
48
+ @globals = {
49
+ # remove 5 minutes to account for time offset between client and server (TODO: configurable?)
50
+ jwt_accepted_offset_sec: 300,
51
+ # one hour validity (TODO: configurable?)
52
+ jwt_expiry_offset_sec: 3600,
53
+ # tokens older than 30 minutes will be discarded from cache
54
+ token_cache_expiry_sec: 1800,
55
+ # tokens valid for less than this duration will be regenerated
56
+ token_expiration_guard_sec: 120
57
+ }
58
+ end
59
+
60
+ public
61
+
62
+ attr_reader :globals
63
+
64
+ def persist_mgr=(manager)
65
+ @persist = manager
66
+ # cleanup expired tokens
67
+ @persist.garbage_collect(PERSIST_CATEGORY_TOKEN, @globals[:token_cache_expiry_sec])
68
+ end
69
+
70
+ def persist_mgr
71
+ if @persist.nil?
72
+ # use OAuth::Factory.instance.persist_mgr=PersistencyFolder.new)
73
+ Log.log.debug('Not using persistency')
74
+ # create NULL persistency class
75
+ @persist = Class.new do
76
+ def get(_x); nil; end; def delete(_x); nil; end; def put(_x, _y); nil; end; def garbage_collect(_x, _y); nil; end # rubocop:disable Layout/EmptyLineBetweenDefs, Style/Semicolon, Layout/LineLength
77
+ end.new
78
+ end
79
+ return @persist
80
+ end
81
+
82
+ # delete all existing tokens
83
+ def flush_tokens
84
+ persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN, nil)
85
+ end
86
+
87
+ # register a bearer token decoder, mainly to inspect expiry date
88
+ def register_decoder(method)
89
+ @decoders.push(method)
90
+ end
91
+
92
+ # decode token using all registered decoders
93
+ def decode_token(token)
94
+ @decoders.each do |decoder|
95
+ result = decoder.call(token) rescue nil
96
+ return result unless result.nil?
97
+ end
98
+ return nil
99
+ end
100
+
101
+ # register a token creation method
102
+ # @param id creation type from field :grant_method in constructor
103
+ # @param lambda_create called to create token
104
+ # @param id_create called to generate unique id for token, for cache
105
+ def register_token_creator(creator_class)
106
+ Aspera.assert_type(creator_class, Class)
107
+ id = self.class.class_to_id(creator_class)
108
+ Log.log.debug{"registering token creator #{id}"}
109
+ @token_type_classes[id] = creator_class
110
+ end
111
+
112
+ # @return one of the registered creators for the given create type
113
+ def create(**parameters)
114
+ Aspera.assert_type(parameters, Hash)
115
+ id = parameters[:grant_method]
116
+ Aspera.assert(@token_type_classes.key?(id)){"token grant method unknown: '#{id}'"}
117
+ create_parameters = parameters.reject { |k, _v| k.eql?(:grant_method) }
118
+ @token_type_classes[id].new(**create_parameters)
119
+ end
120
+ end
121
+ # JSON Web Signature (JWS) compact serialization: https://datatracker.ietf.org/doc/html/rfc7515
122
+ Factory.instance.register_decoder(lambda { |token| parts = token.split('.'); Aspera.assert(parts.length.eql?(3)){'not aoc token'}; JSON.parse(Base64.decode64(parts[1]))}) # rubocop:disable Style/Semicolon, Layout/LineLength
123
+ end
124
+ end