aspera-cli 4.22.0 → 4.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +405 -364
  4. data/CONTRIBUTING.md +86 -29
  5. data/README.md +1856 -961
  6. data/bin/ascli +2 -1
  7. data/bin/asession +4 -4
  8. data/lib/aspera/agent/base.rb +4 -0
  9. data/lib/aspera/agent/connect.rb +20 -18
  10. data/lib/aspera/agent/desktop.rb +14 -11
  11. data/lib/aspera/agent/direct.rb +39 -31
  12. data/lib/aspera/agent/httpgw.rb +2 -2
  13. data/lib/aspera/agent/node.rb +9 -11
  14. data/lib/aspera/agent/transferd.rb +18 -11
  15. data/lib/aspera/api/aoc.rb +53 -43
  16. data/lib/aspera/api/cos_node.rb +7 -5
  17. data/lib/aspera/api/httpgw.rb +23 -22
  18. data/lib/aspera/api/node.rb +104 -22
  19. data/lib/aspera/ascmd.rb +35 -21
  20. data/lib/aspera/ascp/installation.rb +43 -43
  21. data/lib/aspera/ascp/management.rb +5 -4
  22. data/lib/aspera/assert.rb +55 -24
  23. data/lib/aspera/cli/basic_auth_plugin.rb +8 -7
  24. data/lib/aspera/cli/error.rb +1 -1
  25. data/lib/aspera/cli/extended_value.rb +28 -29
  26. data/lib/aspera/cli/formatter.rb +191 -168
  27. data/lib/aspera/cli/hints.rb +38 -4
  28. data/lib/aspera/cli/main.rb +139 -108
  29. data/lib/aspera/cli/manager.rb +51 -31
  30. data/lib/aspera/cli/plugin.rb +149 -78
  31. data/lib/aspera/cli/plugin_factory.rb +2 -2
  32. data/lib/aspera/cli/plugins/aoc.rb +217 -88
  33. data/lib/aspera/cli/plugins/ats.rb +15 -13
  34. data/lib/aspera/cli/plugins/config.rb +105 -227
  35. data/lib/aspera/cli/plugins/console.rb +49 -18
  36. data/lib/aspera/cli/plugins/cos.rb +4 -4
  37. data/lib/aspera/cli/plugins/faspex.rb +45 -51
  38. data/lib/aspera/cli/plugins/faspex5.rb +162 -163
  39. data/lib/aspera/cli/plugins/faspio.rb +6 -5
  40. data/lib/aspera/cli/plugins/httpgw.rb +2 -2
  41. data/lib/aspera/cli/plugins/node.rb +233 -247
  42. data/lib/aspera/cli/plugins/orchestrator.rb +10 -14
  43. data/lib/aspera/cli/plugins/preview.rb +26 -29
  44. data/lib/aspera/cli/plugins/server.rb +29 -28
  45. data/lib/aspera/cli/plugins/shares.rb +40 -28
  46. data/lib/aspera/cli/sync_actions.rb +101 -80
  47. data/lib/aspera/cli/transfer_agent.rb +55 -58
  48. data/lib/aspera/cli/transfer_progress.rb +29 -20
  49. data/lib/aspera/cli/version.rb +1 -1
  50. data/lib/aspera/cli/wizard.rb +160 -0
  51. data/lib/aspera/colors.rb +13 -8
  52. data/lib/aspera/command_line_builder.rb +28 -22
  53. data/lib/aspera/command_line_converter.rb +31 -0
  54. data/lib/aspera/data_repository.rb +1 -0
  55. data/lib/aspera/environment.rb +144 -100
  56. data/lib/aspera/faspex_gw.rb +1 -1
  57. data/lib/aspera/faspex_postproc.rb +3 -2
  58. data/lib/aspera/hash_ext.rb +1 -1
  59. data/lib/aspera/id_generator.rb +10 -10
  60. data/lib/aspera/keychain/base.rb +18 -0
  61. data/lib/aspera/keychain/encrypted_hash.rb +6 -12
  62. data/lib/aspera/keychain/factory.rb +9 -3
  63. data/lib/aspera/keychain/hashicorp_vault.rb +9 -6
  64. data/lib/aspera/keychain/macos_security.rb +13 -13
  65. data/lib/aspera/log.rb +70 -20
  66. data/lib/aspera/nagios.rb +5 -6
  67. data/lib/aspera/node_simulator.rb +12 -7
  68. data/lib/aspera/oauth/base.rb +6 -2
  69. data/lib/aspera/oauth/factory.rb +25 -18
  70. data/lib/aspera/oauth/jwt.rb +13 -1
  71. data/lib/aspera/oauth/url_json.rb +3 -3
  72. data/lib/aspera/oauth/web.rb +5 -3
  73. data/lib/aspera/persistency_folder.rb +2 -2
  74. data/lib/aspera/preview/file_types.rb +43 -35
  75. data/lib/aspera/preview/generator.rb +26 -13
  76. data/lib/aspera/preview/terminal.rb +10 -7
  77. data/lib/aspera/preview/utils.rb +11 -9
  78. data/lib/aspera/products/connect.rb +2 -1
  79. data/lib/aspera/products/desktop.rb +1 -1
  80. data/lib/aspera/products/other.rb +2 -2
  81. data/lib/aspera/products/transferd.rb +8 -6
  82. data/lib/aspera/proxy_auto_config.rb +1 -1
  83. data/lib/aspera/rest.rb +46 -28
  84. data/lib/aspera/rest_call_error.rb +1 -1
  85. data/lib/aspera/rest_error_analyzer.rb +1 -0
  86. data/lib/aspera/resumer.rb +1 -1
  87. data/lib/aspera/secret_hider.rb +46 -40
  88. data/lib/aspera/ssh.rb +14 -4
  89. data/lib/aspera/sync/args.schema.yaml +102 -0
  90. data/lib/aspera/sync/conf.schema.yaml +701 -0
  91. data/lib/aspera/sync/database.rb +83 -0
  92. data/lib/aspera/{transfer/sync.rb → sync/operations.rb} +145 -68
  93. data/lib/aspera/temp_file_manager.rb +4 -2
  94. data/lib/aspera/timer_limiter.rb +7 -5
  95. data/lib/aspera/transfer/error.rb +1 -1
  96. data/lib/aspera/transfer/error_info.rb +1 -2
  97. data/lib/aspera/transfer/faux_file.rb +11 -10
  98. data/lib/aspera/transfer/parameters.rb +6 -5
  99. data/lib/aspera/transfer/spec.rb +15 -1
  100. data/lib/aspera/transfer/spec.schema.yaml +316 -293
  101. data/lib/aspera/transfer/spec_doc.rb +34 -16
  102. data/lib/aspera/transfer/uri.rb +5 -5
  103. data/lib/aspera/uri_reader.rb +14 -10
  104. data/lib/aspera/web_auth.rb +2 -2
  105. data/lib/aspera/web_server_simple.rb +2 -2
  106. data.tar.gz.sig +0 -0
  107. metadata +15 -15
  108. metadata.gz.sig +0 -0
  109. data/examples/dascli +0 -30
  110. data/examples/get_proto_file.rb +0 -8
  111. data/examples/proxy.pac +0 -60
  112. data/lib/aspera/transfer/convert.rb +0 -29
  113. data/lib/aspera/transfer/sync_instance.schema.yaml +0 -13
  114. data/lib/aspera/transfer/sync_session.schema.yaml +0 -79
@@ -4,20 +4,21 @@ require 'aspera/hash_ext'
4
4
  require 'aspera/environment'
5
5
  require 'aspera/log'
6
6
  require 'aspera/assert'
7
+ require 'aspera/keychain/base'
7
8
  require 'symmetric_encryption/core'
8
9
  require 'yaml'
9
10
 
10
11
  module Aspera
11
12
  module Keychain
12
13
  # Manage secrets in a simple Hash
13
- class EncryptedHash
14
+ class EncryptedHash < Base
14
15
  LEGACY_CIPHER_NAME = 'aes-256-cbc'
15
16
  DEFAULT_CIPHER_NAME = 'aes-256-cbc'
16
17
  FILE_TYPE = 'encrypted_hash_vault'
17
- CONTENT_KEYS = %i[label username password url description].freeze
18
18
  FILE_KEYS = %w[version type cipher data].sort.freeze
19
- private_constant :LEGACY_CIPHER_NAME, :DEFAULT_CIPHER_NAME, :FILE_TYPE, :CONTENT_KEYS, :FILE_KEYS
19
+ private_constant :LEGACY_CIPHER_NAME, :DEFAULT_CIPHER_NAME, :FILE_TYPE, :FILE_KEYS
20
20
  def initialize(file:, password:)
21
+ super()
21
22
  Aspera.assert_type(file, String){'path to vault file'}
22
23
  @path = file
23
24
  @all_secrets = {}
@@ -38,9 +39,7 @@ module Aspera
38
39
  end
39
40
  # setting password also creates the cipher
40
41
  @cipher = cipher(password)
41
- if !vault_encrypted_data.nil?
42
- @all_secrets = YAML.load_stream(@cipher.decrypt(vault_encrypted_data)).first
43
- end
42
+ @all_secrets = YAML.load_stream(@cipher.decrypt(vault_encrypted_data)).first if !vault_encrypted_data.nil?
44
43
  end
45
44
 
46
45
  def info
@@ -63,12 +62,7 @@ module Aspera
63
62
  # set a secret
64
63
  # @param options [Hash] with keys :label, :username, :password, :url, :description
65
64
  def set(options)
66
- Aspera.assert_type(options, Hash){'options'}
67
- unsupported = options.keys - CONTENT_KEYS
68
- Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}"}
69
- options.each_pair do |k, v|
70
- Aspera.assert_type(v, String){k.to_s}
71
- end
65
+ validate_set(options)
72
66
  label = options.delete(:label)
73
67
  raise "secret #{label} already exist, delete first" if @all_secrets.key?(label)
74
68
  @all_secrets[label] = options.symbolize_keys
@@ -6,6 +6,11 @@ module Aspera
6
6
  class Factory
7
7
  LIST = %i[file system vault].freeze
8
8
  class << self
9
+ # Create a vault instance
10
+ # @param info [Hash] vault options
11
+ # @param name [String] name of the vault
12
+ # @param folder [String] folder to store the vault (if needed)
13
+ # @param password [String] password to open the vault
9
14
  def create(info, name, folder, password)
10
15
  Aspera.assert_type(info, Hash)
11
16
  Aspera.assert(info.values.all?(String)){'vault info shall have only string values'}
@@ -14,7 +19,7 @@ module Aspera
14
19
  Aspera.assert_values(vault_type, LIST.map(&:to_s)){'vault.type'}
15
20
  case vault_type
16
21
  when 'file'
17
- info[:file] ||= 'vault.bin'
22
+ info[:file] = name || 'vault.bin'
18
23
  info[:file] = File.join(folder, info[:file]) unless File.absolute_path?(info[:file])
19
24
  Aspera.assert(!password.nil?){'please provide password'}
20
25
  info[:password] = password
@@ -22,15 +27,16 @@ module Aspera
22
27
  require 'aspera/keychain/encrypted_hash'
23
28
  @vault = Keychain::EncryptedHash.new(**info)
24
29
  when 'system'
25
- case Environment.os
30
+ case Environment.instance.os
26
31
  when Environment::OS_MACOS
27
32
  info[:name] ||= name
28
33
  @vault = Keychain::MacosSystem.new(**info)
29
34
  else
30
- raise 'not implemented for this OS'
35
+ raise Error, 'not implemented for this OS'
31
36
  end
32
37
  when 'vault'
33
38
  require 'aspera/keychain/hashicorp_vault'
39
+ info[:token] ||= password
34
40
  @vault = Keychain::HashicorpVault.new(**info)
35
41
  else Aspera.error_unexpected_value(vault_type)
36
42
  end
@@ -3,17 +3,19 @@
3
3
  require 'aspera/environment'
4
4
  require 'aspera/log'
5
5
  require 'aspera/assert'
6
+ require 'aspera/keychain/base'
6
7
  require 'vault'
7
8
 
8
9
  module Aspera
9
10
  module Keychain
10
11
  # Manage secrets in a Hashicorp Vault
11
- class HashicorpVault
12
- SECRET_PATH = 'secret/data/'
12
+ class HashicorpVault < Base
13
+ STORE_PATH = 'secret/data/'
13
14
 
14
- private_constant :SECRET_PATH
15
+ private_constant :STORE_PATH
15
16
 
16
17
  def initialize(url:, token:)
18
+ super()
17
19
  Vault.configure do |config|
18
20
  config.address = url
19
21
  config.token = token
@@ -28,7 +30,7 @@ module Aspera
28
30
  end
29
31
 
30
32
  def list
31
- metadata_path = SECRET_PATH.sub('/data/', '/metadata/')
33
+ metadata_path = STORE_PATH.sub('/data/', '/metadata/')
32
34
  return Vault.logical.list(metadata_path).filter_map do |label|
33
35
  get(label: label).merge(label: label)
34
36
  end
@@ -37,6 +39,7 @@ module Aspera
37
39
  # Set a secret
38
40
  # @param options [Hash] with keys :label, :username, :password, :url, :description
39
41
  def set(options)
42
+ validate_set(options)
40
43
  label = options.fetch(:label)
41
44
  data = {
42
45
  username: options[:username],
@@ -51,7 +54,7 @@ module Aspera
51
54
  secret = Vault.logical.read(path(label))
52
55
  if secret.nil?
53
56
  raise "Secret '#{label}' not found" if exception
54
- return nil
57
+ return
55
58
  end
56
59
  return secret.data[:data]
57
60
  end
@@ -64,7 +67,7 @@ module Aspera
64
67
  private
65
68
 
66
69
  def path(label)
67
- "#{SECRET_PATH}#{label}"
70
+ "#{STORE_PATH}#{label}"
68
71
  end
69
72
  end
70
73
  end
@@ -5,6 +5,7 @@ require 'aspera/cli/info'
5
5
  require 'aspera/log'
6
6
  require 'aspera/assert'
7
7
  require 'aspera/environment'
8
+ require 'aspera/keychain/base'
8
9
 
9
10
  # enhance the gem to support other key chains
10
11
  module Aspera
@@ -37,13 +38,13 @@ module Aspera
37
38
  getpass: :g
38
39
  }.freeze
39
40
  class << self
40
- def execute(command, options=nil, supported=nil, last_opt=nil)
41
+ def execute(command, options = nil, supported = nil, last_opt = nil)
41
42
  url = options&.delete(:url)
42
43
  if !url.nil?
43
44
  uri = URI.parse(url)
44
45
  Aspera.assert(uri.scheme.eql?('https')){'only https'}
45
46
  options[:protocol] = 'htps' # cspell: disable-line
46
- raise 'host required in URL' if uri.host.nil?
47
+ raise Error, 'host required in URL' if uri.host.nil?
47
48
  options[:server] = uri.host
48
49
  options[:path] = uri.path unless ['', '/'].include?(uri.path)
49
50
  options[:port] = uri.port unless uri.port.eql?(443) && !url.include?(':443/')
@@ -71,8 +72,8 @@ module Aspera
71
72
  key_chains(execute('login-keychain')).first
72
73
  end
73
74
 
74
- def list(options={})
75
- Aspera.assert_values(options[:domain], DOMAINS, exception_class: ArgumentError){'domain'} unless options[:domain].nil?
75
+ def list(options = {})
76
+ Aspera.assert_values(options[:domain], DOMAINS, type: ArgumentError){'domain'} unless options[:domain].nil?
76
77
  key_chains(execute('list-keychains', options, LIST_OPTIONS))
77
78
  end
78
79
 
@@ -122,9 +123,9 @@ module Aspera
122
123
  end
123
124
  end
124
125
 
125
- class MacosSystem
126
- OPTIONS = %i[label username password url description].freeze
126
+ class MacosSystem < Base
127
127
  def initialize(name: nil)
128
+ super()
128
129
  @keychain_name = name.nil? ? 'default keychain' : name
129
130
  @keychain = name.nil? ? MacosSecurity::Keychain.default : MacosSecurity::Keychain.by_name(name)
130
131
  raise "no such keychain #{name}" if @keychain.nil?
@@ -138,16 +139,15 @@ module Aspera
138
139
 
139
140
  def list
140
141
  # the only way to list is `dump-keychain` which triggers security alert
141
- raise 'list not implemented, use macos keychain app'
142
+ raise Error, 'list not implemented, use macos keychain app'
142
143
  end
143
144
 
144
145
  def set(options)
145
- Aspera.assert_type(options, Hash){'options'}
146
- unsupported = options.keys - OPTIONS
147
- Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}, use #{OPTIONS.join(', ')}"}
146
+ validate_set(options)
148
147
  @keychain.password(
149
148
  :add, :generic, service: options[:label],
150
- account: options[:username] || 'none', password: options[:password], comment: options[:description])
149
+ account: options[:username] || 'none', password: options[:password], comment: options[:description]
150
+ )
151
151
  end
152
152
 
153
153
  def get(options)
@@ -155,7 +155,7 @@ module Aspera
155
155
  unsupported = options.keys - %i[label]
156
156
  Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}"}
157
157
  info = @keychain.password(:find, :generic, label: options[:label])
158
- raise 'not found' if info.nil?
158
+ raise Error, 'not found' if info.nil?
159
159
  result = options.clone
160
160
  result[:secret] = info['password']
161
161
  result[:description] = info['icmt'] # cspell: disable-line
@@ -166,7 +166,7 @@ module Aspera
166
166
  Aspera.assert_type(options, Hash){'options'}
167
167
  unsupported = options.keys - %i[label]
168
168
  Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}"}
169
- raise 'delete not implemented, use macos keychain app'
169
+ raise Error, 'delete not implemented, use macos keychain app'
170
170
  end
171
171
  end
172
172
  end
data/lib/aspera/log.rb CHANGED
@@ -48,30 +48,57 @@ module Aspera
48
48
  # Singleton object for logging
49
49
  class Log
50
50
  include Singleton
51
+
51
52
  # Where logs are sent to
52
53
  LOG_TYPES = %i[stderr stdout syslog].freeze
53
- @@format = :json # rubocop:disable Style/ClassVars
54
+ DEFAULT_FORMATTER = ->(s, _d, _p, m){"#{Log.color_level(s)} #{m}\n"}
55
+ FORMATTERS = {
56
+ standard: Logger::Formatter.new,
57
+ default: DEFAULT_FORMATTER
58
+ }.freeze
54
59
  # Class methods
55
60
  class << self
61
+ def color_level(level)
62
+ case level
63
+ when :TRACE2 then 'TR2'.dim
64
+ when :TRACE1 then 'TR1'.blue
65
+ when :DEBUG then 'DBG'.cyan
66
+ when :INFO then 'INF'.green
67
+ when :WARN then 'WRN'.bg_brown.black
68
+ when :ERROR then 'ERR'.bg_red.blink
69
+ when :FATAL then 'FTL'.magenta
70
+ when :UNKNOWN then 'UKN'.blink
71
+ else Aspera.error_unexpected_value(level){'log level'}
72
+ end
73
+ end
74
+
56
75
  # levels are :debug,:info,:warn,:error,fatal,:unknown
57
76
  def levels; Logger::Severity.constants.sort{ |a, b| Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b)}.map{ |c| c.downcase.to_sym}; end
58
77
 
59
78
  # get the logger object of singleton
60
79
  def log; instance.logger; end
61
80
 
62
- # dump object suitable for Log.log.debug
63
- # @param name string or symbol
64
- # @param format either pp or json format
65
- def dump(name, object)
66
- result =
67
- case @@format
68
- when :json
69
- JSON.pretty_generate(object) rescue PP.pp(object, +'')
70
- when :ruby
71
- PP.pp(object, +'')
72
- else error_unexpected_value(@@format){'dump format'}
73
- end
74
- "#{name.to_s.green} (#{@@format})=\n#{result}"
81
+ # Dump object (`Hash`) using specified level
82
+ #
83
+ # @param name [String, Symbol] Name of object dumped
84
+ # @param object [Hash, nil] Data to dump
85
+ # @param level [Symbol] Debug level
86
+ # @param block [Proc, nil] Give computed object
87
+ def dump(name, object = nil, level: :debug)
88
+ return unless instance.logger.send(:"#{level}?")
89
+ object = yield if block_given?
90
+ instance.logger.send(level, obj_dump(name, object))
91
+ end
92
+
93
+ def obj_dump(name, object)
94
+ dump_text = case instance.dump_format
95
+ when :json
96
+ JSON.pretty_generate(object) rescue PP.pp(object, +'')
97
+ when :ruby
98
+ PP.pp(object, +'')
99
+ else error_unexpected_value(instance.dump_format){'dump format'}
100
+ end
101
+ "#{name.to_s.green} (#{instance.dump_format})=\n#{dump_text}"
75
102
  end
76
103
 
77
104
  # Capture the output of $stderr and log it at debug level
@@ -86,13 +113,33 @@ module Aspera
86
113
  end
87
114
 
88
115
  attr_reader :logger_type, :logger
89
- attr_writer :program_name
116
+ attr_accessor :dump_format
117
+
118
+ def program_name=(value)
119
+ @program_name = value
120
+ self.logger_type = @logger_type
121
+ end
90
122
 
91
123
  # Set log level of underlying logger given symbol level
92
124
  def level=(new_level)
93
125
  @logger.level = Logger::Severity.const_get(new_level.to_sym.upcase)
94
126
  end
95
127
 
128
+ def formatter=(formatter)
129
+ if formatter.is_a?(String)
130
+ raise Error, "Unknown formatter #{formatter}, use one of: #{FORMATTERS.keys.join(', ')}" unless FORMATTERS.key?(formatter.to_sym)
131
+ formatter = FORMATTERS[formatter.to_sym]
132
+ elsif !formatter.respond_to?(:call) && !formatter.is_a?(Logger::Formatter)
133
+ raise Error, 'Formatter must be a String, a Logger::Formatter or a Proc'
134
+ end
135
+ # Update formatter with password hiding
136
+ @logger.formatter = SecretHider.instance.log_formatter(formatter)
137
+ end
138
+
139
+ def formatter
140
+ @logger.formatter
141
+ end
142
+
96
143
  # Get symbol of debug level of underlying logger
97
144
  def level
98
145
  Logger::Severity.constants.each do |name|
@@ -108,9 +155,9 @@ module Aspera
108
155
  current_severity_integer = Logger::Severity::WARN if current_severity_integer.nil?
109
156
  case new_log_type
110
157
  when :stderr
111
- @logger = Logger.new($stderr)
158
+ @logger = Logger.new($stderr, progname: @program_name, formatter: DEFAULT_FORMATTER)
112
159
  when :stdout
113
- @logger = Logger.new($stdout)
160
+ @logger = Logger.new($stdout, progname: @program_name, formatter: DEFAULT_FORMATTER)
114
161
  when :syslog
115
162
  require 'syslog/logger'
116
163
  # the syslog class automatically creates methods from the severity names
@@ -121,13 +168,14 @@ module Aspera
121
168
  Logger::Severity.constants.each do |severity|
122
169
  Syslog::Logger.make_methods(severity.downcase)
123
170
  end
171
+ # Use `local2` facility, like other Aspera components
124
172
  @logger = Syslog::Logger.new(@program_name, Syslog::LOG_LOCAL2)
125
173
  else error_unexpected_value(new_log_type){"log type (#{LOG_TYPES.join(', ')})"}
126
174
  end
127
175
  @logger.level = current_severity_integer
128
176
  @logger_type = new_log_type
129
- # Update formatter with password hiding
130
- @logger.formatter = SecretHider.log_formatter(@logger.formatter)
177
+ # add secret hider to default logger
178
+ self.formatter = @logger.formatter
131
179
  end
132
180
 
133
181
  private
@@ -135,8 +183,10 @@ module Aspera
135
183
  def initialize
136
184
  @logger = nil
137
185
  @program_name = 'aspera'
186
+ @dump_format = :json
187
+ @logger_type = :stderr
138
188
  # This sets @logger and @logger_type (self needed to call method instead of local var)
139
- self.logger_type = :stderr
189
+ self.logger_type = @logger_type
140
190
  end
141
191
  end
142
192
  end
data/lib/aspera/nagios.rb CHANGED
@@ -36,9 +36,8 @@ module Aspera
36
36
  # build message: if multiple components: concatenate
37
37
  # message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
38
38
  message = data
39
- .map{ |i| i['component']}
40
- .uniq
41
- .map{ |comp| comp + ':' + data.select{ |d| d['component'].eql?(comp)}.map{ |d| d['message']}.join(',')}
39
+ .group_by{ |d| d['component']}
40
+ .map{ |comp, items| "#{comp}:#{items.map{ |d| d['message']}.join(',')}"}
42
41
  .join(', ')
43
42
  .tr("\n", ' ')
44
43
  status = data.first['status'].upcase
@@ -58,8 +57,8 @@ module Aspera
58
57
  # compare remote time with local time
59
58
  def check_time_offset(remote_date, component)
60
59
  # check date if specified : 2015-10-13T07:32:01Z
61
- remote_time = DateTime.strptime(remote_date)
62
- diff_time = (remote_time - DateTime.now).abs
60
+ remote_time = Time.strptime(remote_date)
61
+ diff_time = (remote_time - Time.now).abs
63
62
  diff_rounded = diff_time.round(-2)
64
63
  Log.log.debug{"DATE: #{remote_date} #{remote_time} diff=#{diff_rounded}"}
65
64
  msg = "offset #{diff_rounded} sec"
@@ -79,7 +78,7 @@ module Aspera
79
78
 
80
79
  # translate for display
81
80
  def result
82
- raise 'missing result' if @data.empty?
81
+ Aspera.assert(!@data.empty?){'missing result'}
83
82
  {type: :object_list, data: @data.map{ |i| {'status' => LEVELS[i[:code]].to_s, 'component' => i[:comp], 'message' => i[:msg]}}}
84
83
  end
85
84
  end
@@ -119,9 +119,10 @@ module Aspera
119
119
  checksum: nil,
120
120
  start_byte: 0,
121
121
  bytes_written: 26,
122
- session_id: 'bafc72b8-366c-4501-8095-47208183d6b8'}]
122
+ session_id: 'bafc72b8-366c-4501-8095-47208183d6b8'
123
+ }]
123
124
  }
124
- Log.log.trace2{Log.dump(:job, result)}
125
+ Log.dump(:job, result, level: :trace2)
125
126
  return result
126
127
  end
127
128
 
@@ -276,7 +277,8 @@ module Aspera
276
277
  token_encryption_key
277
278
  byok_enabled
278
279
  bandwidth_flow_network_rc_module
279
- file_checksum_type],
280
+ file_checksum_type
281
+ ],
280
282
  server: %w[
281
283
  activity_event_logging
282
284
  activity_file_event_logging
@@ -289,7 +291,8 @@ module Aspera
289
291
  discovery
290
292
  auto_delete
291
293
  allow
292
- deny]
294
+ deny
295
+ ]
293
296
  },
294
297
  capabilities: [
295
298
  {name: 'sync', value: true},
@@ -302,7 +305,8 @@ module Aspera
302
305
  {name: 'aej_version', value: '1.0'},
303
306
  {name: 'page', value: true},
304
307
  {name: 'file_id_version', value: '2.0'},
305
- {name: 'auto_delete', value: false}],
308
+ {name: 'auto_delete', value: false}
309
+ ],
306
310
  settings: [
307
311
  {name: 'content_protection_required', value: false},
308
312
  {name: 'content_protection_strong_pass_required', value: false},
@@ -310,7 +314,8 @@ module Aspera
310
314
  {name: 'ssh_fingerprint', value: nil},
311
315
  {name: 'wss_enabled', value: false},
312
316
  {name: 'wss_port', value: 443}
313
- ]})
317
+ ]
318
+ })
314
319
  when PATH_TRANSFERS
315
320
  set_json_response(request, response, @simulator.all_sessions)
316
321
  when PATH_ONE_TRANSFER
@@ -325,7 +330,7 @@ module Aspera
325
330
  response.status = code
326
331
  response['Content-Type'] = Rest::MIME_JSON
327
332
  response.body = json.to_json
328
- Log.log.trace1{Log.dump("response for #{request.request_method} #{request.path}", json)}
333
+ Log.log.trace1{Log.obj_dump("response for #{request.request_method} #{request.path}", json)}
329
334
  end
330
335
  end
331
336
  end
@@ -31,9 +31,11 @@ module Aspera
31
31
  cache_ids: nil,
32
32
  **rest_params
33
33
  )
34
- Aspera.assert(respond_to?(:create_token), 'create_token method must be defined', exception_class: InternalError)
34
+ Aspera.assert(respond_to?(:create_token), 'create_token method must be defined', type: InternalError)
35
35
  # this is the OAuth API
36
36
  @api = Rest.new(**rest_params)
37
+ @scope = nil
38
+ @token_cache_id = nil
37
39
  @path_token = path_token
38
40
  @token_field = token_field
39
41
  @client_id = client_id
@@ -54,6 +56,8 @@ module Aspera
54
56
  @token_cache_id = Factory.cache_id(@api.base_url, self.class, @base_cache_ids, @scope)
55
57
  end
56
58
 
59
+ attr_reader :scope, :api, :path_token
60
+
57
61
  # helper method to create token as per RFC
58
62
  def create_token_call(creation_params)
59
63
  Log.log.debug{'Generating a new token'.bg_green}
@@ -82,7 +86,7 @@ module Aspera
82
86
 
83
87
  # @return value suitable for Authorization header
84
88
  def authorization(**kwargs)
85
- return OAuth::Factory.bearer_build(token(**kwargs))
89
+ return OAuth::Factory.bearer_authorization(token(**kwargs))
86
90
  end
87
91
 
88
92
  # get an OAuth v2 token (generated, cached, refreshed)
@@ -9,34 +9,40 @@ module Aspera
9
9
  # Factory to create tokens and manage their cache
10
10
  class Factory
11
11
  include Singleton
12
+
12
13
  # a prefix for persistency of tokens (simplify garbage collect)
13
14
  PERSIST_CATEGORY_TOKEN = 'token'
14
- # prefix for bearer token when in header
15
- BEARER_PREFIX = 'Bearer '
15
+ # prefix for bearer authorization when in header
16
+ SPACE_BEARER_AUTH_SCHEME = 'Bearer '
16
17
  TOKEN_FIELD = 'access_token'
17
18
 
18
- private_constant :PERSIST_CATEGORY_TOKEN, :BEARER_PREFIX
19
+ private_constant :PERSIST_CATEGORY_TOKEN, :SPACE_BEARER_AUTH_SCHEME
19
20
 
20
21
  class << self
21
- def bearer_build(token)
22
- return "#{BEARER_PREFIX}#{token}"
22
+ # @param token [String] The token alone
23
+ # @return [String] Value suitable for Authorization header
24
+ def bearer_authorization(token)
25
+ return "#{SPACE_BEARER_AUTH_SCHEME}#{token}"
23
26
  end
24
27
 
25
- def bearer?(token)
26
- return token.start_with?(BEARER_PREFIX)
28
+ # @return true if the authorization contains a bearer token , i.e. auth scheme is bearer
29
+ def bearer_auth?(authorization)
30
+ return authorization.start_with?(SPACE_BEARER_AUTH_SCHEME)
27
31
  end
28
32
 
29
- def bearer_extract(token)
30
- Aspera.assert(bearer?(token)){'not a bearer token, wrong prefix'}
31
- return token[BEARER_PREFIX.length..-1]
33
+ # Extract only token from Authorization (remove scheme)
34
+ def bearer_token(authorization)
35
+ Aspera.assert(bearer_auth?(authorization)){'not a bearer token, wrong prefix scheme'}
36
+ return authorization[SPACE_BEARER_AUTH_SCHEME.length..-1]
32
37
  end
33
38
 
34
- # @return a cache identifier
39
+ # @return a unique cache identifier
35
40
  def cache_id(url, creator_class, *params)
36
41
  return IdGenerator.from_list([
37
42
  PERSIST_CATEGORY_TOKEN,
38
43
  url,
39
- Factory.class_to_id(creator_class)] +
44
+ Factory.class_to_id(creator_class)
45
+ ] +
40
46
  params)
41
47
  end
42
48
 
@@ -53,6 +59,7 @@ module Aspera
53
59
  @persist = nil
54
60
  # token creation methods
55
61
  @token_type_classes = {}
62
+ # list of lambda
56
63
  @decoders = []
57
64
  # default parameters, others can be added by handlers
58
65
  @parameters = {
@@ -79,7 +86,7 @@ module Aspera
79
86
  Log.log.debug('Not using persistency')
80
87
  # create NULL persistency class
81
88
  @persist = Class.new do
82
- def get(_x); nil; end; def delete(_x); nil; end; def put(_x, _y); nil; end; def garbage_collect(_x, _y); nil; end # rubocop:disable Layout/EmptyLineBetweenDefs, Style/Semicolon, Layout/LineLength
89
+ def get(_x); nil; end; def delete(_x); nil; end; def put(_x, _y); nil; end; def garbage_collect(_x, _y); nil; end # rubocop:disable Style/Semicolon
83
90
  end.new
84
91
  end
85
92
  return @persist
@@ -106,17 +113,17 @@ module Aspera
106
113
  # @return [Hash] token internal information , including Date object for `expiration_date`
107
114
  def get_token_info(id)
108
115
  token_raw_string = persist_mgr.get(id)
109
- return nil if token_raw_string.nil?
116
+ return if token_raw_string.nil?
110
117
  token_data = JSON.parse(token_raw_string)
111
118
  Aspera.assert_type(token_data, Hash)
112
119
  decoded_token = decode_token(token_data[TOKEN_FIELD])
113
120
  info = {data: token_data}
114
- Log.log.debug{Log.dump('decoded_token', decoded_token)}
121
+ Log.dump(:decoded_token, decoded_token)
115
122
  if decoded_token.is_a?(Hash)
116
123
  info[:decoded] = decoded_token
117
124
  # TODO: move date decoding to token decoder ?
118
125
  expiration_date =
119
- if decoded_token['expires_at'].is_a?(String) then DateTime.parse(decoded_token['expires_at']).to_time
126
+ if decoded_token['expires_at'].is_a?(String) then Time.parse(decoded_token['expires_at']).to_time
120
127
  elsif decoded_token['exp'].is_a?(Integer) then Time.at(decoded_token['exp'])
121
128
  end
122
129
  unless expiration_date.nil?
@@ -139,7 +146,7 @@ module Aspera
139
146
  result = decoder.call(token) rescue nil
140
147
  return result unless result.nil?
141
148
  end
142
- return nil
149
+ return
143
150
  end
144
151
 
145
152
  # register a token creation method
@@ -163,6 +170,6 @@ module Aspera
163
170
  end
164
171
  end
165
172
  # JSON Web Signature (JWS) compact serialization: https://datatracker.ietf.org/doc/html/rfc7515
166
- Factory.instance.register_decoder(lambda{ |token| parts = token.split('.'); Aspera.assert(parts.length.eql?(3)){'not JWS token'}; JSON.parse(Base64.decode64(parts[1]))}) # rubocop:disable Style/Semicolon, Layout/LineLength
173
+ Factory.instance.register_decoder(lambda{ |token| parts = token.split('.'); Aspera.assert(parts.length.eql?(3)){'not JWS token'}; JSON.parse(Base64.decode64(parts[1]))}) # rubocop:disable Style/Semicolon
167
174
  end
168
175
  end
@@ -3,6 +3,7 @@
3
3
  require 'aspera/oauth/base'
4
4
  require 'aspera/assert'
5
5
  require 'securerandom'
6
+ require 'openssl'
6
7
  module Aspera
7
8
  module OAuth
8
9
  # remove 5 minutes to account for time offset between client and server (TODO: configurable?)
@@ -13,6 +14,17 @@ module Aspera
13
14
  # https://tools.ietf.org/html/rfc7523
14
15
  # https://tools.ietf.org/html/rfc7519
15
16
  class Jwt < Base
17
+ class << self
18
+ def generate_rsa_private_key(path:, length: DEFAULT_PRIV_KEY_LENGTH)
19
+ priv_key = OpenSSL::PKey::RSA.new(length)
20
+ File.write(path, priv_key.to_s)
21
+ File.write("#{path}.pub", priv_key.public_key.to_s)
22
+ Environment.restrict_file_access(path)
23
+ Environment.restrict_file_access("#{path}.pub")
24
+ nil
25
+ end
26
+ end
27
+ DEFAULT_PRIV_KEY_LENGTH = 4096
16
28
  GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
17
29
  # @param private_key_obj private key object
18
30
  # @param payload payload to be included in the JWT
@@ -45,7 +57,7 @@ module Aspera
45
57
  iat: seconds_since_epoch - OAuth::Factory.instance.parameters[:jwt_accepted_offset_sec] + 1, # issued at
46
58
  jti: SecureRandom.uuid # JWT id
47
59
  }.merge(@additional_payload)
48
- Log.log.debug{Log.dump(:jwt_payload, jwt_payload)}
60
+ Log.dump(:jwt_payload, jwt_payload)
49
61
  Log.log.debug{"private=[#{@private_key_obj}]"}
50
62
  assertion = JWT.encode(jwt_payload, @private_key_obj, 'RS256', @headers)
51
63
  Log.log.debug{"assertion=[#{assertion}]"}
@@ -19,10 +19,10 @@ module Aspera
19
19
  end
20
20
 
21
21
  def create_token
22
- @api.call(
22
+ api.call(
23
23
  operation: 'POST',
24
- subpath: @path_token,
25
- query: @query.merge(scope: @scope), # scope is here because it may change over time (node)
24
+ subpath: path_token,
25
+ query: @query.merge(scope: scope), # scope is here because it may change over time (node)
26
26
  content_type: Rest::MIME_JSON,
27
27
  body: @body,
28
28
  headers: {'Accept' => Rest::MIME_JSON}